diff --git a/components/platform/CMakeLists.txt b/components/platform/CMakeLists.txt index 8b9fb67..4fe1e1b 100644 --- a/components/platform/CMakeLists.txt +++ b/components/platform/CMakeLists.txt @@ -52,11 +52,15 @@ idf_component_register( ) set(platform_lvgl_animation_src_dir "${CMAKE_CURRENT_SOURCE_DIR}/assets/lvgl_animations") +set(platform_audio_prompt_src_dir "${CMAKE_CURRENT_SOURCE_DIR}/assets/audio_prompts") set(platform_lvgl_animation_image_dir "${CMAKE_CURRENT_BINARY_DIR}/lvgl_animations_image") set(platform_lvgl_animation_stage_stamp "${CMAKE_CURRENT_BINARY_DIR}/lvgl_animations_image.stamp") file(GLOB_RECURSE platform_lvgl_animation_bin_files CONFIGURE_DEPENDS "${platform_lvgl_animation_src_dir}/*.bin" ) +file(GLOB_RECURSE platform_audio_prompt_files CONFIGURE_DEPENDS + "${platform_audio_prompt_src_dir}/*.opus" +) set(platform_lvgl_animation_stage_commands) foreach(animation_file IN LISTS platform_lvgl_animation_bin_files) @@ -69,6 +73,15 @@ foreach(animation_file IN LISTS platform_lvgl_animation_bin_files) "${platform_lvgl_animation_image_dir}/${animation_rel_path}" ) endforeach() +foreach(audio_file IN LISTS platform_audio_prompt_files) + file(RELATIVE_PATH audio_rel_path "${platform_audio_prompt_src_dir}" "${audio_file}") + list(APPEND platform_lvgl_animation_stage_commands + COMMAND ${CMAKE_COMMAND} -E make_directory "${platform_lvgl_animation_image_dir}/audio_prompts" + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "${audio_file}" + "${platform_lvgl_animation_image_dir}/audio_prompts/${audio_rel_path}" + ) +endforeach() add_custom_command( OUTPUT "${platform_lvgl_animation_stage_stamp}" @@ -76,7 +89,7 @@ add_custom_command( COMMAND ${CMAKE_COMMAND} -E make_directory "${platform_lvgl_animation_image_dir}" ${platform_lvgl_animation_stage_commands} COMMAND ${CMAKE_COMMAND} -E touch "${platform_lvgl_animation_stage_stamp}" - DEPENDS ${platform_lvgl_animation_bin_files} + DEPENDS ${platform_lvgl_animation_bin_files} ${platform_audio_prompt_files} ) littlefs_create_partition_image( diff --git a/components/platform/assets/audio_prompts/battery_checking.opus b/components/platform/assets/audio_prompts/battery_checking.opus new file mode 100644 index 0000000..6ac1a43 Binary files /dev/null and b/components/platform/assets/audio_prompts/battery_checking.opus differ diff --git a/components/platform/assets/audio_prompts/battery_low_20.opus b/components/platform/assets/audio_prompts/battery_low_20.opus new file mode 100644 index 0000000..da516f8 Binary files /dev/null and b/components/platform/assets/audio_prompts/battery_low_20.opus differ diff --git a/components/platform/assets/audio_prompts/battery_low_energy_warning.opus b/components/platform/assets/audio_prompts/battery_low_energy_warning.opus new file mode 100644 index 0000000..7302037 Binary files /dev/null and b/components/platform/assets/audio_prompts/battery_low_energy_warning.opus differ diff --git a/components/platform/assets/audio_prompts/boot.opus b/components/platform/assets/audio_prompts/boot.opus new file mode 100644 index 0000000..0220b27 Binary files /dev/null and b/components/platform/assets/audio_prompts/boot.opus differ diff --git a/components/platform/assets/audio_prompts/button_interrupt_listening.opus b/components/platform/assets/audio_prompts/button_interrupt_listening.opus new file mode 100644 index 0000000..0c55f56 Binary files /dev/null and b/components/platform/assets/audio_prompts/button_interrupt_listening.opus differ diff --git a/components/platform/assets/audio_prompts/charge_complete.opus b/components/platform/assets/audio_prompts/charge_complete.opus new file mode 100644 index 0000000..6f1a97e Binary files /dev/null and b/components/platform/assets/audio_prompts/charge_complete.opus differ diff --git a/components/platform/assets/audio_prompts/daily_first_wake.opus b/components/platform/assets/audio_prompts/daily_first_wake.opus new file mode 100644 index 0000000..99eb5dd Binary files /dev/null and b/components/platform/assets/audio_prompts/daily_first_wake.opus differ diff --git a/components/platform/assets/audio_prompts/daily_wake.opus b/components/platform/assets/audio_prompts/daily_wake.opus new file mode 100644 index 0000000..b583ac0 Binary files /dev/null and b/components/platform/assets/audio_prompts/daily_wake.opus differ diff --git a/components/platform/assets/audio_prompts/enter_provisioning.opus b/components/platform/assets/audio_prompts/enter_provisioning.opus new file mode 100644 index 0000000..5f4d03e Binary files /dev/null and b/components/platform/assets/audio_prompts/enter_provisioning.opus differ diff --git a/components/platform/assets/audio_prompts/intro_first_setup_success.opus b/components/platform/assets/audio_prompts/intro_first_setup_success.opus new file mode 100644 index 0000000..9fb23e5 Binary files /dev/null and b/components/platform/assets/audio_prompts/intro_first_setup_success.opus differ diff --git a/components/platform/assets/audio_prompts/luck_flow_lost_help_find.opus b/components/platform/assets/audio_prompts/luck_flow_lost_help_find.opus new file mode 100644 index 0000000..9ddcfda Binary files /dev/null and b/components/platform/assets/audio_prompts/luck_flow_lost_help_find.opus differ diff --git a/components/platform/assets/audio_prompts/network_connect_failed.opus b/components/platform/assets/audio_prompts/network_connect_failed.opus new file mode 100644 index 0000000..dcdd076 Binary files /dev/null and b/components/platform/assets/audio_prompts/network_connect_failed.opus differ diff --git a/components/platform/assets/audio_prompts/network_connect_success.opus b/components/platform/assets/audio_prompts/network_connect_success.opus new file mode 100644 index 0000000..78ccd77 Binary files /dev/null and b/components/platform/assets/audio_prompts/network_connect_success.opus differ diff --git a/components/platform/assets/audio_prompts/network_connecting.opus b/components/platform/assets/audio_prompts/network_connecting.opus new file mode 100644 index 0000000..3e3e816 Binary files /dev/null and b/components/platform/assets/audio_prompts/network_connecting.opus differ diff --git a/components/platform/assets/audio_prompts/network_interrupted_repairing.opus b/components/platform/assets/audio_prompts/network_interrupted_repairing.opus new file mode 100644 index 0000000..6c88d67 Binary files /dev/null and b/components/platform/assets/audio_prompts/network_interrupted_repairing.opus differ diff --git a/components/platform/assets/audio_prompts/network_unstable.opus b/components/platform/assets/audio_prompts/network_unstable.opus new file mode 100644 index 0000000..f8e2425 Binary files /dev/null and b/components/platform/assets/audio_prompts/network_unstable.opus differ diff --git a/components/platform/assets/audio_prompts/server_abnormal_interrupted.opus b/components/platform/assets/audio_prompts/server_abnormal_interrupted.opus new file mode 100644 index 0000000..96d5fbf Binary files /dev/null and b/components/platform/assets/audio_prompts/server_abnormal_interrupted.opus differ diff --git a/components/platform/assets/audio_prompts/server_connect_failed.opus b/components/platform/assets/audio_prompts/server_connect_failed.opus new file mode 100644 index 0000000..44637d7 Binary files /dev/null and b/components/platform/assets/audio_prompts/server_connect_failed.opus differ diff --git a/components/platform/assets/audio_prompts/server_connect_success.opus b/components/platform/assets/audio_prompts/server_connect_success.opus new file mode 100644 index 0000000..c8bcb3e Binary files /dev/null and b/components/platform/assets/audio_prompts/server_connect_success.opus differ diff --git a/components/platform/assets/audio_prompts/server_repair_wait_open.opus b/components/platform/assets/audio_prompts/server_repair_wait_open.opus new file mode 100644 index 0000000..4024bc2 Binary files /dev/null and b/components/platform/assets/audio_prompts/server_repair_wait_open.opus differ diff --git a/components/platform/assets/audio_prompts/server_repair_wait_reopen.opus b/components/platform/assets/audio_prompts/server_repair_wait_reopen.opus new file mode 100644 index 0000000..a86f38a Binary files /dev/null and b/components/platform/assets/audio_prompts/server_repair_wait_reopen.opus differ diff --git a/components/platform/assets/audio_prompts/shutdown.opus b/components/platform/assets/audio_prompts/shutdown.opus new file mode 100644 index 0000000..71ab8f0 Binary files /dev/null and b/components/platform/assets/audio_prompts/shutdown.opus differ diff --git a/components/platform/assets/audio_prompts/shutdown_normal_goodbye.opus b/components/platform/assets/audio_prompts/shutdown_normal_goodbye.opus new file mode 100644 index 0000000..4d02a77 Binary files /dev/null and b/components/platform/assets/audio_prompts/shutdown_normal_goodbye.opus differ diff --git a/components/platform/assets/audio_prompts/shutdown_power_empty.opus b/components/platform/assets/audio_prompts/shutdown_power_empty.opus new file mode 100644 index 0000000..6cb2016 Binary files /dev/null and b/components/platform/assets/audio_prompts/shutdown_power_empty.opus differ diff --git a/components/platform/assets/audio_prompts/signal_restored_business_resume.opus b/components/platform/assets/audio_prompts/signal_restored_business_resume.opus new file mode 100644 index 0000000..31b3a6a Binary files /dev/null and b/components/platform/assets/audio_prompts/signal_restored_business_resume.opus differ diff --git a/components/platform/assets/audio_prompts/standby_5min.opus b/components/platform/assets/audio_prompts/standby_5min.opus new file mode 100644 index 0000000..bdf009e Binary files /dev/null and b/components/platform/assets/audio_prompts/standby_5min.opus differ diff --git a/components/platform/assets/audio_prompts/system_update_complete.opus b/components/platform/assets/audio_prompts/system_update_complete.opus new file mode 100644 index 0000000..73a1758 Binary files /dev/null and b/components/platform/assets/audio_prompts/system_update_complete.opus differ diff --git a/components/platform/assets/audio_prompts/system_upgrading.opus b/components/platform/assets/audio_prompts/system_upgrading.opus new file mode 100644 index 0000000..37299f6 Binary files /dev/null and b/components/platform/assets/audio_prompts/system_upgrading.opus differ diff --git a/components/platform/assets/audio_prompts/welcome_luck_ready.opus b/components/platform/assets/audio_prompts/welcome_luck_ready.opus new file mode 100644 index 0000000..4ae1444 Binary files /dev/null and b/components/platform/assets/audio_prompts/welcome_luck_ready.opus differ diff --git a/components/platform/assets/audio_prompts/wifi_password_wrong.opus b/components/platform/assets/audio_prompts/wifi_password_wrong.opus new file mode 100644 index 0000000..ad78fd7 Binary files /dev/null and b/components/platform/assets/audio_prompts/wifi_password_wrong.opus differ diff --git a/components/platform/src/multimodal/audio_pipeline.c b/components/platform/src/multimodal/audio_pipeline.c index dc15ed6..3d23ab1 100644 --- a/components/platform/src/multimodal/audio_pipeline.c +++ b/components/platform/src/multimodal/audio_pipeline.c @@ -296,7 +296,8 @@ typedef struct { bool uplink_send_queue_with_caps; TaskHandle_t uplink_task; TaskHandle_t uplink_send_task; - volatile bool uplink_stop_requested; + volatile bool uplink_recording_active; + volatile bool uplink_task_exit_requested; multimodal_audio_uplink_cb_t uplink_cb; void *uplink_cb_ctx; bool opus_ready; diff --git a/components/platform/src/multimodal/audio_pipeline/audio_pipeline_runtime_impl.c b/components/platform/src/multimodal/audio_pipeline/audio_pipeline_runtime_impl.c index 5fa4ae2..7a40774 100644 --- a/components/platform/src/multimodal/audio_pipeline/audio_pipeline_runtime_impl.c +++ b/components/platform/src/multimodal/audio_pipeline/audio_pipeline_runtime_impl.c @@ -79,24 +79,46 @@ static void reset_uplink_debug_stats(void) #endif } -static bool audio_get_uplink_stop_requested(const multimodal_audio_ctx_t *ctx) +static bool audio_get_uplink_recording_active(const multimodal_audio_ctx_t *ctx) { - bool stop_requested = true; + bool active = false; portENTER_CRITICAL(&s_audio_runtime_lock); if (ctx != NULL) { - stop_requested = ctx->uplink_stop_requested; + active = ctx->uplink_recording_active; } portEXIT_CRITICAL(&s_audio_runtime_lock); - return stop_requested; + return active; } -static void audio_set_uplink_stop_requested(multimodal_audio_ctx_t *ctx, bool stop_requested) +static void audio_set_uplink_recording_active(multimodal_audio_ctx_t *ctx, bool active) { portENTER_CRITICAL(&s_audio_runtime_lock); if (ctx != NULL) { - ctx->uplink_stop_requested = stop_requested; + ctx->uplink_recording_active = active; + } + portEXIT_CRITICAL(&s_audio_runtime_lock); +} + +static bool audio_get_uplink_task_exit_requested(const multimodal_audio_ctx_t *ctx) +{ + bool exit_requested = true; + + portENTER_CRITICAL(&s_audio_runtime_lock); + if (ctx != NULL) { + exit_requested = ctx->uplink_task_exit_requested; + } + portEXIT_CRITICAL(&s_audio_runtime_lock); + + return exit_requested; +} + +static void audio_set_uplink_task_exit_requested(multimodal_audio_ctx_t *ctx, bool exit_requested) +{ + portENTER_CRITICAL(&s_audio_runtime_lock); + if (ctx != NULL) { + ctx->uplink_task_exit_requested = exit_requested; } portEXIT_CRITICAL(&s_audio_runtime_lock); } @@ -512,6 +534,7 @@ static void notify_downstream_playback_stopped(void); static esp_err_t prime_downstream_output_path(multimodal_audio_ctx_t *ctx); static esp_err_t handle_downstream_opus_sync(const uint8_t *data, size_t len); static esp_err_t ensure_downstream_playback_task(multimodal_audio_ctx_t *ctx); +static esp_err_t ensure_uplink_worker_tasks(multimodal_audio_ctx_t *ctx); static esp_err_t stop_downstream_idle_mute_task(multimodal_audio_ctx_t *ctx, int timeout_ms); #define MULTIMODAL_AUDIO_DOWNSTREAM_STATS_INTERVAL 200U @@ -620,25 +643,14 @@ static void uplink_send_task(void *arg) (unsigned)get_uplink_send_queue_len(), (unsigned)CONFIG_MULTIMODAL_AUDIO_UPLINK_MAX_PACKET_BYTES); - while (true) { - bool stop_requested = audio_get_uplink_stop_requested(ctx); + while (!audio_get_uplink_task_exit_requested(ctx)) { if (!ctx->uplink_send_queue) { - if (stop_requested) { - break; - } vTaskDelay(pdMS_TO_TICKS(20)); continue; } uplink_packet_t packet = {0}; - if (stop_requested && uxQueueMessagesWaiting(ctx->uplink_send_queue) == 0U) { - break; - } - if (xQueueReceive(ctx->uplink_send_queue, &packet, pdMS_TO_TICKS(100)) != pdTRUE) { - if (stop_requested) { - break; - } continue; } if (packet.len == 0U) { @@ -697,7 +709,12 @@ static void uplink_capture_task(void *arg) (unsigned)ctx->fs.channel, (unsigned)ctx->mic_capture_channel); - while (!audio_get_uplink_stop_requested(ctx)) { + while (!audio_get_uplink_task_exit_requested(ctx)) { + if (!audio_get_uplink_recording_active(ctx)) { + ulTaskNotifyTake(pdTRUE, pdMS_TO_TICKS(100)); + continue; + } + if (!ctx->initialized || !ctx->codec_dev || !ctx->opus_enc) { vTaskDelay(pdMS_TO_TICKS(20)); continue; @@ -736,6 +753,69 @@ static void uplink_capture_task(void *arg) delete_audio_task_with_caps(NULL); } +static esp_err_t ensure_uplink_worker_tasks(multimodal_audio_ctx_t *ctx) +{ + if (ctx == NULL) { + return ESP_ERR_INVALID_ARG; + } + if (!ctx->initialized || !ctx->codec_dev || !ctx->opus_enc || + !ctx->uplink_codec_buf || !ctx->uplink_mono_buf || !ctx->uplink_opus_buf) { + return ESP_ERR_INVALID_STATE; + } + + if (ctx->uplink_send_queue == NULL) { + bool queue_with_caps = false; + UBaseType_t queue_len = get_uplink_send_queue_len(); + ctx->uplink_send_queue = create_uplink_send_queue(queue_len, &queue_with_caps); + if (ctx->uplink_send_queue == NULL) { + ESP_LOGW(TAG, + "Create uplink send queue failed (len=%u item=%u)", + (unsigned)queue_len, + (unsigned)sizeof(uplink_packet_t)); + return ESP_ERR_NO_MEM; + } + ctx->uplink_send_queue_with_caps = queue_with_caps; + } + + audio_set_uplink_task_exit_requested(ctx, false); + + TaskHandle_t capture_task = NULL; + TaskHandle_t send_task = NULL; + audio_snapshot_uplink_tasks(ctx, &capture_task, &send_task); + if (capture_task == NULL) { + BaseType_t created = create_audio_task_with_caps(uplink_capture_task, + "mm_audio_uplink", + get_uplink_task_stack_bytes(), + NULL, + 5, + &ctx->uplink_task, + true, + MULTIMODAL_AUDIO_CORE_ID); + if (created != pdPASS) { + audio_set_uplink_capture_task(ctx, NULL); + return ESP_ERR_NO_MEM; + } + } + + audio_snapshot_uplink_tasks(ctx, NULL, &send_task); + if (send_task == NULL) { + BaseType_t created = create_audio_task_with_caps(uplink_send_task, + "mm_audio_up_send", + get_uplink_send_task_stack_bytes(), + NULL, + 6, + &ctx->uplink_send_task, + true, + MULTIMODAL_AUDIO_CORE_ID); + if (created != pdPASS) { + audio_set_uplink_send_task(ctx, NULL); + return ESP_ERR_NO_MEM; + } + } + + return ESP_OK; +} + esp_err_t multimodal_audio_init(void) { #if !CONFIG_MULTIMODAL_AUDIO_ENABLE_CODEC_PATH @@ -793,8 +873,29 @@ esp_err_t multimodal_audio_init(void) audio_set_deinit_in_progress(&s_audio, false); audio_set_downstream_playback_stopping(&s_audio, false); audio_set_downstream_playback_stop_requested(&s_audio, false); + audio_set_uplink_recording_active(&s_audio, false); + audio_set_uplink_task_exit_requested(&s_audio, false); s_audio.initialized = true; audio_debug_dump_init(); + err = ensure_uplink_worker_tasks(&s_audio); + if (err == ESP_OK) { + err = ensure_downstream_playback_task(&s_audio); + } + if (err != ESP_OK) { + ESP_LOGW(TAG, "Create audio runtime tasks failed: %s", esp_err_to_name(err)); + s_audio.initialized = false; + audio_debug_dump_deinit(); + (void)multimodal_audio_stop_uplink(1000); + (void)multimodal_audio_stop_downstream_playback(); + cleanup_codec_path(&s_audio); + deinit_buses(&s_audio); + if (!reuse_prewarmed_opus) { + cleanup_opus_path(&s_audio); + memset(&s_audio, 0, sizeof(s_audio)); + } + last_err = err; + continue; + } ESP_LOGI(TAG, "Codec initialized: %uHz %uch %ubit opus(frame=%ums bitrate=%ukbps)", (unsigned)s_audio.fs.sample_rate, @@ -838,6 +939,12 @@ void multimodal_audio_deinit(void) audio_set_downstream_playback_stopping(&s_audio, true); audio_debug_dump_deinit(); (void)multimodal_audio_stop_uplink(2000); + audio_set_uplink_task_exit_requested(&s_audio, true); + TaskHandle_t notify_capture_task = NULL; + audio_snapshot_uplink_tasks(&s_audio, ¬ify_capture_task, NULL); + if (notify_capture_task != NULL) { + xTaskNotifyGive(notify_capture_task); + } (void)multimodal_audio_stop_downstream_playback(); for (int i = 0; i < 100 && audio_has_uplink_tasks(&s_audio); ++i) { vTaskDelay(pdMS_TO_TICKS(20)); @@ -894,140 +1001,25 @@ esp_err_t multimodal_audio_start_uplink(multimodal_audio_uplink_cb_t cb, void *c return ESP_ERR_INVALID_STATE; } - if (audio_get_uplink_stop_requested(&s_audio)) { - ESP_LOGW(TAG, "Mic uplink stop in progress"); - return ESP_ERR_INVALID_STATE; - } - if (audio_has_uplink_tasks(&s_audio)) { - return ESP_OK; - } - audio_set_uplink_callback(&s_audio, cb, cb_ctx); s_audio.uplink_mono_fill_bytes = 0U; memset(s_audio.uplink_mono_buf, 0, s_audio.uplink_mono_buf_capacity_bytes); - audio_set_uplink_stop_requested(&s_audio, false); reset_uplink_debug_stats(); - uint32_t uplink_stack_bytes = get_uplink_task_stack_bytes(); - uint32_t uplink_send_stack_bytes = get_uplink_send_task_stack_bytes(); - UBaseType_t uplink_send_queue_len = get_uplink_send_queue_len(); log_heap_snapshot("before_uplink_start"); - if (s_audio.uplink_send_queue == NULL) { - bool queue_with_caps = false; - s_audio.uplink_send_queue = create_uplink_send_queue(uplink_send_queue_len, &queue_with_caps); - if (s_audio.uplink_send_queue == NULL) { - size_t free_internal = heap_caps_get_free_size(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT); - size_t largest_internal = heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT); -#if SOC_SPIRAM_SUPPORTED - size_t free_spiram = heap_caps_get_free_size(MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); - size_t largest_spiram = heap_caps_get_largest_free_block(MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); - ESP_LOGW(TAG, - "Create uplink send queue failed (len=%u item=%u free_int=%u largest_int=%u free_spiram=%u largest_spiram=%u)", - (unsigned)uplink_send_queue_len, - (unsigned)sizeof(uplink_packet_t), - (unsigned)free_internal, - (unsigned)largest_internal, - (unsigned)free_spiram, - (unsigned)largest_spiram); -#else - ESP_LOGW(TAG, - "Create uplink send queue failed (len=%u item=%u free_int=%u largest_int=%u)", - (unsigned)uplink_send_queue_len, - (unsigned)sizeof(uplink_packet_t), - (unsigned)free_internal, - (unsigned)largest_internal); -#endif - audio_set_uplink_callback(&s_audio, NULL, NULL); - audio_set_uplink_stop_requested(&s_audio, false); - return ESP_ERR_NO_MEM; - } - s_audio.uplink_send_queue_with_caps = queue_with_caps; - } else { - reset_uplink_send_queue(&s_audio); - } - - BaseType_t created = create_audio_task_with_caps(uplink_capture_task, - "mm_audio_uplink", - uplink_stack_bytes, - NULL, - 5, - &s_audio.uplink_task, - true, - MULTIMODAL_AUDIO_CORE_ID); - if (created != pdPASS) { - audio_set_uplink_capture_task(&s_audio, NULL); - size_t free_internal = heap_caps_get_free_size(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT); - size_t largest_internal = heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT); -#if SOC_SPIRAM_SUPPORTED - size_t free_spiram = heap_caps_get_free_size(MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); - size_t largest_spiram = heap_caps_get_largest_free_block(MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); - ESP_LOGW(TAG, - "Create uplink capture task failed (stack=%uB free_int=%u largest_int=%u free_spiram=%u largest_spiram=%u)", - (unsigned)uplink_stack_bytes, - (unsigned)free_internal, - (unsigned)largest_internal, - (unsigned)free_spiram, - (unsigned)largest_spiram); -#else - ESP_LOGW(TAG, - "Create uplink capture task failed (stack=%uB free_int=%u largest_int=%u)", - (unsigned)uplink_stack_bytes, - (unsigned)free_internal, - (unsigned)largest_internal); -#endif - reset_uplink_send_queue(&s_audio); + esp_err_t err = ensure_uplink_worker_tasks(&s_audio); + if (err != ESP_OK) { audio_set_uplink_callback(&s_audio, NULL, NULL); - audio_set_uplink_stop_requested(&s_audio, false); - return ESP_ERR_NO_MEM; + return err; } - TaskHandle_t existing_send_task = NULL; - audio_snapshot_uplink_tasks(&s_audio, NULL, &existing_send_task); - if (existing_send_task == NULL) { - BaseType_t send_created = create_audio_task_with_caps(uplink_send_task, - "mm_audio_up_send", - uplink_send_stack_bytes, - NULL, - 6, - &s_audio.uplink_send_task, - true, - MULTIMODAL_AUDIO_CORE_ID); - if (send_created != pdPASS) { - audio_set_uplink_send_task(&s_audio, NULL); - audio_set_uplink_stop_requested(&s_audio, true); - for (int i = 0; i < 40 && audio_has_uplink_tasks(&s_audio); ++i) { - vTaskDelay(pdMS_TO_TICKS(20)); - } - - size_t free_internal = heap_caps_get_free_size(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT); - size_t largest_internal = heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT); -#if SOC_SPIRAM_SUPPORTED - size_t free_spiram = heap_caps_get_free_size(MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); - size_t largest_spiram = heap_caps_get_largest_free_block(MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); - ESP_LOGW(TAG, - "Create uplink send task failed (stack=%uB free_int=%u largest_int=%u free_spiram=%u largest_spiram=%u)", - (unsigned)uplink_send_stack_bytes, - (unsigned)free_internal, - (unsigned)largest_internal, - (unsigned)free_spiram, - (unsigned)largest_spiram); -#else - ESP_LOGW(TAG, - "Create uplink send task failed (stack=%uB free_int=%u largest_int=%u)", - (unsigned)uplink_send_stack_bytes, - (unsigned)free_internal, - (unsigned)largest_internal); -#endif - - reset_uplink_send_queue(&s_audio); - audio_set_uplink_callback(&s_audio, NULL, NULL); - if (!audio_has_uplink_tasks(&s_audio)) { - audio_set_uplink_stop_requested(&s_audio, false); - } - return ESP_ERR_NO_MEM; - } + reset_uplink_send_queue(&s_audio); + audio_set_uplink_recording_active(&s_audio, true); + TaskHandle_t capture_task = NULL; + audio_snapshot_uplink_tasks(&s_audio, &capture_task, NULL); + if (capture_task != NULL) { + xTaskNotifyGive(capture_task); } - log_heap_snapshot("after_uplink_start"); return ESP_OK; } @@ -1038,63 +1030,18 @@ esp_err_t multimodal_audio_stop_uplink(int timeout_ms) return ESP_ERR_INVALID_ARG; } - if (!audio_has_uplink_tasks(&s_audio)) { - reset_uplink_send_queue(&s_audio); - audio_set_uplink_callback(&s_audio, NULL, NULL); - s_audio.uplink_mono_fill_bytes = 0U; - audio_set_uplink_stop_requested(&s_audio, false); - return ESP_OK; - } - - audio_set_uplink_stop_requested(&s_audio, true); - if (timeout_ms == 0) { - audio_set_uplink_callback(&s_audio, NULL, NULL); - return ESP_OK; - } - - int effective_timeout_ms = timeout_ms; - int waited_ms = 0; - while (audio_has_uplink_tasks(&s_audio) && waited_ms < effective_timeout_ms) { - vTaskDelay(pdMS_TO_TICKS(20)); - waited_ms += 20; - } - TaskHandle_t capture_task = NULL; - TaskHandle_t send_task = NULL; - audio_snapshot_uplink_tasks(&s_audio, &capture_task, &send_task); - esp_err_t result = ESP_OK; - if (capture_task != NULL || send_task != NULL) { - ESP_LOGW(TAG, - "Mic uplink stop timeout, force delete (capture=%p send=%p)", - (void *)capture_task, - (void *)send_task); - result = ESP_ERR_TIMEOUT; - if (capture_task != NULL) { - delete_audio_task_with_caps(capture_task); - audio_set_uplink_capture_task(&s_audio, NULL); - } - if (send_task != NULL) { - delete_audio_task_with_caps(send_task); - audio_set_uplink_send_task(&s_audio, NULL); - } - } - + (void)timeout_ms; + audio_set_uplink_recording_active(&s_audio, false); reset_uplink_send_queue(&s_audio); audio_set_uplink_callback(&s_audio, NULL, NULL); s_audio.uplink_mono_fill_bytes = 0U; - audio_set_uplink_stop_requested(&s_audio, false); log_heap_snapshot("after_uplink_stop"); - return result; + return ESP_OK; } bool multimodal_audio_is_uplink_running(void) { - TaskHandle_t capture_task = NULL; - TaskHandle_t send_task = NULL; - - audio_snapshot_uplink_tasks(&s_audio, &capture_task, &send_task); - return capture_task != NULL && - send_task != NULL && - !audio_get_uplink_stop_requested(&s_audio); + return audio_get_uplink_recording_active(&s_audio); } static uint32_t snapshot_downstream_playback_generation(void) @@ -1664,9 +1611,6 @@ static void downstream_playback_task(void *arg) if (received != pdTRUE || packet.data == NULL || packet.len == 0U) { if (snapshot_downstream_response_end_received()) { maybe_notify_downstream_playback_completed_after_drain("queue_empty"); - if (!snapshot_downstream_response_active()) { - break; - } } vTaskDelay(pdMS_TO_TICKS(10)); continue; @@ -1923,10 +1867,15 @@ esp_err_t multimodal_audio_stop_downstream_playback(void) (void)advance_downstream_playback_generation(); notify_downstream_playback_stopped(); - bool keep_stopping = audio_get_deinit_in_progress(ctx); - esp_err_t task_stop_err = stop_downstream_playback_task(ctx, 1000, keep_stopping); - if (task_stop_err != ESP_OK && task_stop_err != ESP_ERR_INVALID_STATE) { - ESP_LOGW(TAG, "Stop downstream playback task failed: %s", esp_err_to_name(task_stop_err)); + if (audio_get_deinit_in_progress(ctx)) { + esp_err_t task_stop_err = stop_downstream_playback_task(ctx, 1000, true); + if (task_stop_err != ESP_OK && task_stop_err != ESP_ERR_INVALID_STATE) { + ESP_LOGW(TAG, "Stop downstream playback task failed: %s", esp_err_to_name(task_stop_err)); + } + } else { + drain_downstream_playback_queue(ctx); + audio_set_downstream_playback_stop_requested(ctx, false); + audio_set_downstream_playback_stopping(ctx, false); } if (!ctx->initialized || ctx->codec_dev == NULL) { diff --git a/components/platform/src/platform_behavior_stubs.c b/components/platform/src/platform_behavior_stubs.c index 7b9a73f..b518640 100644 --- a/components/platform/src/platform_behavior_stubs.c +++ b/components/platform/src/platform_behavior_stubs.c @@ -4,11 +4,20 @@ * SPDX-License-Identifier: Apache-2.0 */ +#include #include +#include +#include +#include #include "driver/gpio.h" #include "esp_err.h" #include "esp_log.h" +#include "esp_heap_caps.h" +#include "esp_littlefs.h" +#include "freertos/FreeRTOS.h" +#include "freertos/task.h" +#include "multimodal/audio_pipeline.h" #include "power_manager.h" #include "sdkconfig.h" #include "voice_prompt.h" @@ -21,8 +30,19 @@ #error "CONFIG_TQ_POWER_HOLD_ACTIVE_LEVEL must be defined in Kconfig" #endif +#define VOICE_PROMPT_DEFAULT_BASE_PATH "/littlefs/audio_prompts" +#define VOICE_PROMPT_LITTLEFS_BASE_PATH "/littlefs" +#define VOICE_PROMPT_LITTLEFS_PARTITION "gifpack" +#define VOICE_PROMPT_FRAME_HEADER_BYTES 4U +#define VOICE_PROMPT_MAX_OPUS_PACKET_BYTES 2048U +#define VOICE_PROMPT_OPUS_FRAME_MS 60U +#define VOICE_PROMPT_DRAIN_MARGIN_MS 120U + static const char *TAG = "power_manager"; +static const char *VOICE_TAG = "voice_prompt"; static bool s_power_hold_configured; +static bool s_voice_prompt_littlefs_mount_attempted; +static esp_err_t s_voice_prompt_littlefs_mount_result = ESP_ERR_INVALID_STATE; static uint64_t power_manager_gpio_bit_mask(int gpio_num) { @@ -91,8 +111,163 @@ void platform_power_hold_init(void) power_manager_set_keep_awake(true); } +static esp_err_t voice_prompt_mount_littlefs(void) +{ + if (s_voice_prompt_littlefs_mount_attempted) { + return s_voice_prompt_littlefs_mount_result; + } + + esp_vfs_littlefs_conf_t conf = { + .base_path = VOICE_PROMPT_LITTLEFS_BASE_PATH, + .partition_label = VOICE_PROMPT_LITTLEFS_PARTITION, + .partition = NULL, + .format_if_mount_failed = false, + .dont_mount = false, + }; + + esp_err_t err = esp_vfs_littlefs_register(&conf); + if (err == ESP_ERR_INVALID_STATE && + esp_littlefs_mounted(VOICE_PROMPT_LITTLEFS_PARTITION)) { + err = ESP_OK; + } + if (err == ESP_OK) { + size_t total_bytes = 0; + size_t used_bytes = 0; + esp_err_t info_err = esp_littlefs_info(VOICE_PROMPT_LITTLEFS_PARTITION, + &total_bytes, + &used_bytes); + if (info_err == ESP_OK) { + ESP_LOGI(VOICE_TAG, + "LittleFS mounted at %s, used=%u total=%u", + VOICE_PROMPT_LITTLEFS_BASE_PATH, + (unsigned)used_bytes, + (unsigned)total_bytes); + } + } else { + ESP_LOGW(VOICE_TAG, + "Mount LittleFS failed: %s", + esp_err_to_name(err)); + } + + s_voice_prompt_littlefs_mount_attempted = true; + s_voice_prompt_littlefs_mount_result = err; + return err; +} + esp_err_t voice_prompt_play_file(const char *path) { - (void)path; - return ESP_ERR_NOT_SUPPORTED; + if (path == NULL || path[0] == '\0') { + return ESP_ERR_INVALID_ARG; + } + + esp_err_t mount_err = voice_prompt_mount_littlefs(); + if (mount_err != ESP_OK) { + return mount_err; + } + + char resolved_path[160] = {0}; + if (path[0] == '/') { + int written = snprintf(resolved_path, sizeof(resolved_path), "%s", path); + if (written < 0 || written >= (int)sizeof(resolved_path)) { + return ESP_ERR_INVALID_ARG; + } + } else { + int written = snprintf(resolved_path, + sizeof(resolved_path), + "%s/%s", + VOICE_PROMPT_DEFAULT_BASE_PATH, + path); + if (written < 0 || written >= (int)sizeof(resolved_path)) { + return ESP_ERR_INVALID_ARG; + } + } + + FILE *file = fopen(resolved_path, "rb"); + if (file == NULL) { + ESP_LOGW(VOICE_TAG, "Open prompt failed: %s", resolved_path); + return ESP_ERR_NOT_FOUND; + } + + esp_err_t err = multimodal_audio_begin_downstream_response(); + if (err != ESP_OK) { + ESP_LOGW(VOICE_TAG, "Begin prompt playback failed: %s", esp_err_to_name(err)); + fclose(file); + return err; + } + + uint8_t *packet = (uint8_t *)heap_caps_malloc(VOICE_PROMPT_MAX_OPUS_PACKET_BYTES, + MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); + if (packet == NULL) { + ESP_LOGW(VOICE_TAG, + "Allocate prompt opus packet buffer in PSRAM failed: %u bytes", + (unsigned)VOICE_PROMPT_MAX_OPUS_PACKET_BYTES); + fclose(file); + (void)multimodal_audio_finish_downstream_response(); + return ESP_ERR_NO_MEM; + } + + uint32_t packet_count = 0; + size_t total_bytes = 0; + + while (true) { + uint8_t header[VOICE_PROMPT_FRAME_HEADER_BYTES] = {0}; + size_t header_read = fread(header, 1, sizeof(header), file); + if (header_read == 0U) { + break; + } + if (header_read != sizeof(header)) { + ESP_LOGW(VOICE_TAG, "Truncated prompt frame header: %s", resolved_path); + err = ESP_ERR_INVALID_SIZE; + break; + } + + uint32_t packet_len = ((uint32_t)header[0] << 24) | + ((uint32_t)header[1] << 16) | + ((uint32_t)header[2] << 8) | + (uint32_t)header[3]; + if (packet_len == 0U || packet_len > VOICE_PROMPT_MAX_OPUS_PACKET_BYTES) { + ESP_LOGW(VOICE_TAG, + "Invalid prompt opus packet size: %u in %s", + (unsigned)packet_len, + resolved_path); + err = ESP_ERR_INVALID_SIZE; + break; + } + + size_t payload_read = fread(packet, 1, packet_len, file); + if (payload_read != packet_len) { + ESP_LOGW(VOICE_TAG, "Truncated prompt opus packet: %s", resolved_path); + err = ESP_ERR_INVALID_SIZE; + break; + } + + err = multimodal_audio_handle_downstream_opus(packet, packet_len); + if (err != ESP_OK) { + ESP_LOGW(VOICE_TAG, + "Queue prompt opus packet failed: %s", + esp_err_to_name(err)); + break; + } + packet_count++; + total_bytes += packet_len; + } + + free(packet); + fclose(file); + esp_err_t finish_err = multimodal_audio_finish_downstream_response(); + if (err == ESP_OK) { + err = finish_err; + } + + if (err == ESP_OK) { + uint32_t drain_ms = packet_count * VOICE_PROMPT_OPUS_FRAME_MS + + VOICE_PROMPT_DRAIN_MARGIN_MS; + vTaskDelay(pdMS_TO_TICKS(drain_ms)); + ESP_LOGI(VOICE_TAG, + "Prompt queued: %s packets=%u bytes=%u", + resolved_path, + (unsigned)packet_count, + (unsigned)total_bytes); + } + return err; } diff --git a/defaultAudio/battery_checking.opus b/defaultAudio/battery_checking.opus new file mode 100644 index 0000000..6ac1a43 Binary files /dev/null and b/defaultAudio/battery_checking.opus differ diff --git a/defaultAudio/battery_low_20.opus b/defaultAudio/battery_low_20.opus new file mode 100644 index 0000000..da516f8 Binary files /dev/null and b/defaultAudio/battery_low_20.opus differ diff --git a/defaultAudio/battery_low_energy_warning.opus b/defaultAudio/battery_low_energy_warning.opus new file mode 100644 index 0000000..7302037 Binary files /dev/null and b/defaultAudio/battery_low_energy_warning.opus differ diff --git a/defaultAudio/boot.opus b/defaultAudio/boot.opus new file mode 100644 index 0000000..0220b27 Binary files /dev/null and b/defaultAudio/boot.opus differ diff --git a/defaultAudio/button_interrupt_listening.opus b/defaultAudio/button_interrupt_listening.opus new file mode 100644 index 0000000..0c55f56 Binary files /dev/null and b/defaultAudio/button_interrupt_listening.opus differ diff --git a/defaultAudio/charge_complete.opus b/defaultAudio/charge_complete.opus new file mode 100644 index 0000000..6f1a97e Binary files /dev/null and b/defaultAudio/charge_complete.opus differ diff --git a/defaultAudio/daily_first_wake.opus b/defaultAudio/daily_first_wake.opus new file mode 100644 index 0000000..99eb5dd Binary files /dev/null and b/defaultAudio/daily_first_wake.opus differ diff --git a/defaultAudio/daily_wake.opus b/defaultAudio/daily_wake.opus new file mode 100644 index 0000000..b583ac0 Binary files /dev/null and b/defaultAudio/daily_wake.opus differ diff --git a/defaultAudio/enter_provisioning.opus b/defaultAudio/enter_provisioning.opus new file mode 100644 index 0000000..5f4d03e Binary files /dev/null and b/defaultAudio/enter_provisioning.opus differ diff --git a/defaultAudio/intro_first_setup_success.opus b/defaultAudio/intro_first_setup_success.opus new file mode 100644 index 0000000..9fb23e5 Binary files /dev/null and b/defaultAudio/intro_first_setup_success.opus differ diff --git a/defaultAudio/luck_flow_lost_help_find.opus b/defaultAudio/luck_flow_lost_help_find.opus new file mode 100644 index 0000000..9ddcfda Binary files /dev/null and b/defaultAudio/luck_flow_lost_help_find.opus differ diff --git a/defaultAudio/network_connect_failed.opus b/defaultAudio/network_connect_failed.opus new file mode 100644 index 0000000..dcdd076 Binary files /dev/null and b/defaultAudio/network_connect_failed.opus differ diff --git a/defaultAudio/network_connect_success.opus b/defaultAudio/network_connect_success.opus new file mode 100644 index 0000000..78ccd77 Binary files /dev/null and b/defaultAudio/network_connect_success.opus differ diff --git a/defaultAudio/network_connecting.opus b/defaultAudio/network_connecting.opus new file mode 100644 index 0000000..3e3e816 Binary files /dev/null and b/defaultAudio/network_connecting.opus differ diff --git a/defaultAudio/network_interrupted_repairing.opus b/defaultAudio/network_interrupted_repairing.opus new file mode 100644 index 0000000..6c88d67 Binary files /dev/null and b/defaultAudio/network_interrupted_repairing.opus differ diff --git a/defaultAudio/network_unstable.opus b/defaultAudio/network_unstable.opus new file mode 100644 index 0000000..f8e2425 Binary files /dev/null and b/defaultAudio/network_unstable.opus differ diff --git a/defaultAudio/server_abnormal_interrupted.opus b/defaultAudio/server_abnormal_interrupted.opus new file mode 100644 index 0000000..96d5fbf Binary files /dev/null and b/defaultAudio/server_abnormal_interrupted.opus differ diff --git a/defaultAudio/server_connect_failed.opus b/defaultAudio/server_connect_failed.opus new file mode 100644 index 0000000..44637d7 Binary files /dev/null and b/defaultAudio/server_connect_failed.opus differ diff --git a/defaultAudio/server_connect_success.opus b/defaultAudio/server_connect_success.opus new file mode 100644 index 0000000..c8bcb3e Binary files /dev/null and b/defaultAudio/server_connect_success.opus differ diff --git a/defaultAudio/server_repair_wait_open.opus b/defaultAudio/server_repair_wait_open.opus new file mode 100644 index 0000000..4024bc2 Binary files /dev/null and b/defaultAudio/server_repair_wait_open.opus differ diff --git a/defaultAudio/server_repair_wait_reopen.opus b/defaultAudio/server_repair_wait_reopen.opus new file mode 100644 index 0000000..a86f38a Binary files /dev/null and b/defaultAudio/server_repair_wait_reopen.opus differ diff --git a/defaultAudio/shutdown.opus b/defaultAudio/shutdown.opus new file mode 100644 index 0000000..71ab8f0 Binary files /dev/null and b/defaultAudio/shutdown.opus differ diff --git a/defaultAudio/shutdown_normal_goodbye.opus b/defaultAudio/shutdown_normal_goodbye.opus new file mode 100644 index 0000000..4d02a77 Binary files /dev/null and b/defaultAudio/shutdown_normal_goodbye.opus differ diff --git a/defaultAudio/shutdown_power_empty.opus b/defaultAudio/shutdown_power_empty.opus new file mode 100644 index 0000000..6cb2016 Binary files /dev/null and b/defaultAudio/shutdown_power_empty.opus differ diff --git a/defaultAudio/signal_restored_business_resume.opus b/defaultAudio/signal_restored_business_resume.opus new file mode 100644 index 0000000..31b3a6a Binary files /dev/null and b/defaultAudio/signal_restored_business_resume.opus differ diff --git a/defaultAudio/standby_5min.opus b/defaultAudio/standby_5min.opus new file mode 100644 index 0000000..bdf009e Binary files /dev/null and b/defaultAudio/standby_5min.opus differ diff --git a/defaultAudio/system_update_complete.opus b/defaultAudio/system_update_complete.opus new file mode 100644 index 0000000..73a1758 Binary files /dev/null and b/defaultAudio/system_update_complete.opus differ diff --git a/defaultAudio/system_upgrading.opus b/defaultAudio/system_upgrading.opus new file mode 100644 index 0000000..37299f6 Binary files /dev/null and b/defaultAudio/system_upgrading.opus differ diff --git a/defaultAudio/welcome_luck_ready.opus b/defaultAudio/welcome_luck_ready.opus new file mode 100644 index 0000000..4ae1444 Binary files /dev/null and b/defaultAudio/welcome_luck_ready.opus differ diff --git a/defaultAudio/wifi_password_wrong.opus b/defaultAudio/wifi_password_wrong.opus new file mode 100644 index 0000000..ad78fd7 Binary files /dev/null and b/defaultAudio/wifi_password_wrong.opus differ diff --git a/docs/GX8006-WIFI-CAT1-AI对讲自由对讲方案.pdf b/docs/GX8006-WIFI-CAT1-AI对讲自由对讲方案.pdf new file mode 100644 index 0000000..bfaad07 Binary files /dev/null and b/docs/GX8006-WIFI-CAT1-AI对讲自由对讲方案.pdf differ diff --git a/docs/国芯微离线语音协议_v1.3.7(1).pdf b/docs/国芯微离线语音协议_v1.3.7(1).pdf new file mode 100644 index 0000000..ac062bb Binary files /dev/null and b/docs/国芯微离线语音协议_v1.3.7(1).pdf differ diff --git a/sdkconfig.defaults b/sdkconfig.defaults index b7c25a0..cde04a2 100644 --- a/sdkconfig.defaults +++ b/sdkconfig.defaults @@ -16,7 +16,7 @@ CONFIG_FREERTOS_USE_TRACE_FACILITY=y CONFIG_TQ_TASK_MONITOR_ENABLE=y # CONFIG_TQ_BOOT_SEQUENCE_DRY_RUN is not set CONFIG_TQ_BOOT_SOUND_TIMEOUT_MS=3000 -CONFIG_TQ_BOOT_SOUND_FILE="" +CONFIG_TQ_BOOT_SOUND_FILE="boot.opus" CONFIG_TQ_NETWORK_TASK_CORE=0 CONFIG_TQ_LOCAL_TASK_CORE=1 CONFIG_TQ_TASK_MONITOR_INTERVAL_MS=10000