From 617255bdabc5ef5473553e09822901d68d5c4471 Mon Sep 17 00:00:00 2001 From: chenjintao Date: Mon, 10 Aug 2026 17:40:12 +0800 Subject: [PATCH] feat(voice): add local wake response prompt Enable the packaged wake response prompt by default and add the MP3-to-device-Opus conversion utility. --- .../control_plane_multimodal_local_wake.c | 3 +- .../audio_prompts/local_wake_response.opus | Bin 0 -> 3680 bytes main/Kconfig.projbuild | 12 +- sdkconfig.defaults | 4 +- tools/check_architecture_guards.sh | 9 +- tools/convert_mp3_to_prompt_opus.py | 344 ++++++++++++++++++ 6 files changed, 358 insertions(+), 14 deletions(-) create mode 100644 components/platform/assets/audio_prompts/local_wake_response.opus create mode 100644 tools/convert_mp3_to_prompt_opus.py diff --git a/components/control_plane/src/multimodal/control_plane_multimodal_local_wake.c b/components/control_plane/src/multimodal/control_plane_multimodal_local_wake.c index 5b930c4..e6aa506 100644 --- a/components/control_plane/src/multimodal/control_plane_multimodal_local_wake.c +++ b/components/control_plane/src/multimodal/control_plane_multimodal_local_wake.c @@ -18,8 +18,7 @@ #endif #ifndef CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_FILE -#define CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_FILE \ - "/littlefs/audio_prompts/local_wake_response.opus" +#define CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_FILE "local_wake_response.opus" #endif #ifndef CONFIG_TQ_LOCAL_WAKE_RETRY_MS diff --git a/components/platform/assets/audio_prompts/local_wake_response.opus b/components/platform/assets/audio_prompts/local_wake_response.opus new file mode 100644 index 0000000000000000000000000000000000000000..f260815490b04107cf08ea520f832297c59692d8 GIT binary patch literal 3680 zcmeIzX*d*W8wc&4hqa#=0wnrVkC8QmrNVAWuJmo+T)VgC&1|0xdKTy%h0!|gc*DR*d0Hch-(a}RAb ze(Fru*wMa?`xUDhCW|8PjI!4H-x1K}phIcf^FG5*4YXj5pHIiEgKXR2wTh0$w^=JG z9H#-{Q=^e9QILMyh%oa@KqT@}xPB#E{%OJ4=+4ffF+OE!%Uz{Mw5q7w^P+NFhahSC z1$KM+L%|fy$-u7D8G3P7olrdK?Kk1j`A#cDEwKb~A54IRd4-FbT#ZfLeSYk)ddtO_ z-(PUx2F!&{1SSef{5V{9>C!kQGde(c?zk3`Hl~p;ClpBuyVlr}uLk0Mt0Z=3Nv&q^ z^z9m_UzDNOEt2^0%HBZP<~Sy6>a6s-9(?eX+<)#cdA;?4VWarbOU^AvHDn+z$=Ff2 zQ5;bY)hv62Pr1#wrILtG^OC)i+lKLA$ax9Jf+;PnnPbbF*Nk>m0a2Gb5_v?*pSUib=-VVV%F!!-ysT5C<%&!Dh(YZtJv zZ*J!)Jlpt(khpU`v-1pOWDWMBC!s@*bZ*-+)*I8eP~_N`+Rr2ZS@kjvI`)FLbyZ({w0T7zzQQep^GisJPw_bB6Mx83q?qtlOg(&O9 zk-vmh&)s59NEh$zf%b6N*%;+sbPRNuAC$o-yGhj?Fcun4LF7$o6a7SPfe-f}yJGi? zZk<>+C~Jyf+ix7sCVdj|(=}}U!wpzm!!?t;-@uE?-f~0lI7^90=w;A^5)VCek$pF0 zXicdJfO-ii1_S++4Bb_SXQBhn*9sl!n$z_ZE8iyCitasmS&|2$H4OYv->aFkT8>%S zsnCrmdGx8t%5hC1dgX)jPW#ht;b-o34Mf#rTg+b4Mbm0K->O`fqz_;|OY^K~I<8(a z-Rd_Rr1Y{_Q`RY)-U&8aJPD}ou)md4U9WVx;ma^Z1CPN3d(^MZWy#fi&l^Bb0RY>N zb;dbV|7@P=_TTeWL6L#O^a_@tJG@WVtSG47$Lk~o&>yO2*%r-_n-j(PD({v=PiGIrAu|)3|PQK7h`tV<2=OFY^4Up zKJnh4bnmxCM(p=45(k>-J{4T&YDDDNyu{i8NJ=P5-YksciDZEd}1TYp-Ne(+XiRZB{G$ z422%hh-mf9RM82_>D`+|SeqD_OoCe-qIVbxnMFmT%sfNVuTB+Q1FP)qt`8i zOC+uvK8Vy!kLONqamF)JrQLd{g4W?K2C_IUX|g@wmrQpOl~?p$r8PTddsbFJ zzhb+eCfqC|!B%HU;sD1yBba-2PncbkOu#eqmW%O_olD8&Q#o`Q^_}oz=9X*5P`|PN5_MKjWz6HbTdI1Z7SxBpXbuBJ;AuaW93mWp*$)i~h|RbZ2~Q#Iz#e5oc*{pvru+AYXF7uEPPMl00=G5>tV{JT3m z#^0QKFpdFRC{aI}V;-~@;Q4q%_R0e>H*^}scg_e@GkUMB%nV)g+M5?Y>Do3K`8w6U zwrmF_=6lfKZ>od53xJK(8z+eJJX@A2ei~^o{_Smb*x$tY)7Sp%{?k$YB?te_`!9*{ H&o%iU5N)Yj literal 0 HcmV?d00001 diff --git a/main/Kconfig.projbuild b/main/Kconfig.projbuild index 61fd282..917db02 100644 --- a/main/Kconfig.projbuild +++ b/main/Kconfig.projbuild @@ -889,20 +889,18 @@ menu "Example Configuration" config TQ_LOCAL_WAKE_RESPONSE_PROMPT_ENABLE bool "Enable local wake response prompt" depends on TQ_LOCAL_WAKE_ENABLE - default n + default y help Play a short acknowledgement after 喵掌柜 is detected and - before the normal single-turn dialog starts. Keep disabled - until the placeholder Opus asset has been supplied. + before the normal single-turn dialog starts. config TQ_LOCAL_WAKE_RESPONSE_PROMPT_FILE string "Local wake response prompt file" depends on TQ_LOCAL_WAKE_ENABLE - default "/littlefs/audio_prompts/local_wake_response.opus" + default "local_wake_response.opus" help - Reserved LittleFS path for the future wake acknowledgement. - Add local_wake_response.opus under - components/platform/assets/audio_prompts before enabling it. + Prompt filename relative to /littlefs/audio_prompts. Absolute + LittleFS paths are also supported. config TQ_LOCAL_WAKE_RETRY_MS int "Local wake startup retry delay (ms)" diff --git a/sdkconfig.defaults b/sdkconfig.defaults index 1f021c4..41e2dfb 100644 --- a/sdkconfig.defaults +++ b/sdkconfig.defaults @@ -60,8 +60,8 @@ CONFIG_TQ_PTT_MODE_CLICK_TO_TALK=y # CONFIG_TQ_PTT_MODE_HOLD_TO_TALK is not set CONFIG_TQ_PTT_CLICK_CAPTURE_TIMEOUT_MS=8000 CONFIG_TQ_LOCAL_WAKE_ENABLE=y -# CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_ENABLE is not set -CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_FILE="/littlefs/audio_prompts/local_wake_response.opus" +CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_ENABLE=y +CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_FILE="local_wake_response.opus" CONFIG_TQ_LOCAL_WAKE_RETRY_MS=5000 CONFIG_TQ_LOCAL_WAKE_STOP_WAIT_MS=1000 CONFIG_TQ_LOCAL_WAKE_TASK_STACK_BYTES=6144 diff --git a/tools/check_architecture_guards.sh b/tools/check_architecture_guards.sh index fc79092..1d36286 100644 --- a/tools/check_architecture_guards.sh +++ b/tools/check_architecture_guards.sh @@ -50,11 +50,14 @@ fi if ! rg -n 'TQ_LOCAL_WAKE_RESPONSE_PROMPT_ENABLE' \ main/Kconfig.projbuild \ components/control_plane/src/multimodal/control_plane_multimodal_local_wake.c >/dev/null 2>&1 || - ! rg -n '/littlefs/audio_prompts/local_wake_response\.opus' \ + ! rg -n 'CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_ENABLE=y' \ + sdkconfig.defaults >/dev/null 2>&1 || + ! rg -n 'local_wake_response\.opus' \ main/Kconfig.projbuild \ sdkconfig.defaults \ - components/control_plane/src/multimodal/control_plane_multimodal_local_wake.c >/dev/null 2>&1; then - fail "Local wake response prompt must keep its disabled asset placeholder contract" + components/control_plane/src/multimodal/control_plane_multimodal_local_wake.c >/dev/null 2>&1 || + [[ ! -f components/platform/assets/audio_prompts/local_wake_response.opus ]]; then + fail "Local wake response prompt must remain enabled with its packaged asset contract" fi if ! rg -n '#include "esp_afe_aec.h"' components/platform/src/multimodal/audio_pipeline.c >/dev/null 2>&1 || ! rg -n 'afe_aec_create\(' components/platform/src/multimodal/audio_pipeline/audio_pipeline_codec_impl.c >/dev/null 2>&1 || diff --git a/tools/convert_mp3_to_prompt_opus.py b/tools/convert_mp3_to_prompt_opus.py new file mode 100644 index 0000000..c103581 --- /dev/null +++ b/tools/convert_mp3_to_prompt_opus.py @@ -0,0 +1,344 @@ +#!/usr/bin/env python3 +"""Convert an audio file into the device's length-prefixed raw Opus format. + +The runtime prompt player does not consume an Ogg Opus file directly. It +expects a sequence of records with this layout: + + uint32_be packet_size + uint8_t raw_opus_packet[packet_size] + +FFmpeg is used only for decoding/resampling and Opus encoding. This script +then removes the Ogg container and writes the framing expected by the device. +""" + +from __future__ import annotations + +import argparse +import os +from pathlib import Path +import shutil +import struct +import subprocess +import sys +import tempfile +from typing import BinaryIO, Iterable, Iterator + + +SAMPLE_RATE_HZ = 16_000 +CHANNELS = 1 +FRAME_DURATION_MS = 60 +DEFAULT_BITRATE_KBPS = 24 +MAX_PACKET_BYTES = 2_048 +OGG_CAPTURE_PATTERN = b"OggS" + + +class ConversionError(RuntimeError): + """Raised when conversion input or output violates the prompt contract.""" + + +def _read_exact(stream: BinaryIO, size: int, description: str) -> bytes: + data = stream.read(size) + if len(data) != size: + raise ConversionError(f"truncated {description}") + return data + + +def iter_ogg_packets(path: Path) -> Iterator[bytes]: + """Yield packets from the single logical Ogg stream generated by FFmpeg.""" + + packet = bytearray() + stream_serial: int | None = None + expected_sequence: int | None = None + + with path.open("rb") as stream: + while True: + capture = stream.read(4) + if capture == b"": + break + if capture != OGG_CAPTURE_PATTERN: + raise ConversionError("invalid Ogg capture pattern") + + fixed_header = _read_exact(stream, 23, "Ogg page header") + version = fixed_header[0] + header_type = fixed_header[1] + serial = struct.unpack_from(" int: + """Return an Opus packet duration using the TOC rules from RFC 6716.""" + + if not packet: + raise ConversionError("empty Opus packet") + + toc = packet[0] + config = toc >> 3 + if config >= 16: + frame_duration_us = 2_500 << (config & 0x03) + elif config >= 12: + frame_duration_us = 10_000 << (config & 0x01) + elif (config & 0x03) == 0x03: + frame_duration_us = 60_000 + else: + frame_duration_us = 10_000 << (config & 0x03) + + frame_code = toc & 0x03 + if frame_code == 0: + frame_count = 1 + elif frame_code in (1, 2): + frame_count = 2 + else: + if len(packet) < 2: + raise ConversionError("truncated Opus packet frame-count byte") + frame_count = packet[1] & 0x3F + if frame_count == 0: + raise ConversionError("invalid zero-frame Opus packet") + + duration_us = frame_duration_us * frame_count + if duration_us > 120_000: + raise ConversionError( + f"Opus packet duration exceeds 120 ms: {duration_us / 1000:g} ms" + ) + return duration_us + + +def validate_raw_prompt_packets(packets: Iterable[bytes]) -> tuple[int, int]: + packet_count = 0 + payload_bytes = 0 + expected_duration_us = FRAME_DURATION_MS * 1_000 + for index, packet in enumerate(packets, start=1): + if len(packet) > MAX_PACKET_BYTES: + raise ConversionError( + f"packet {index} is {len(packet)} bytes; maximum is " + f"{MAX_PACKET_BYTES}" + ) + duration_us = opus_packet_duration_us(packet) + if duration_us != expected_duration_us: + raise ConversionError( + f"packet {index} duration is {duration_us / 1000:g} ms; " + f"expected {FRAME_DURATION_MS} ms" + ) + packet_count += 1 + payload_bytes += len(packet) + + if packet_count == 0: + raise ConversionError("no audio packets were produced") + return packet_count, payload_bytes + + +def read_length_prefixed_packets(path: Path) -> Iterator[bytes]: + with path.open("rb") as stream: + while True: + header = stream.read(4) + if header == b"": + break + if len(header) != 4: + raise ConversionError("truncated prompt packet header") + packet_size = struct.unpack(">I", header)[0] + if packet_size == 0 or packet_size > MAX_PACKET_BYTES: + raise ConversionError( + f"invalid prompt packet size: {packet_size}" + ) + yield _read_exact(stream, packet_size, "prompt packet payload") + + +def write_length_prefixed_packets(path: Path, packets: Iterable[bytes]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary_path: Path | None = None + try: + with tempfile.NamedTemporaryFile( + mode="wb", prefix=f".{path.name}.", dir=path.parent, delete=False + ) as output: + temporary_path = Path(output.name) + for packet in packets: + output.write(struct.pack(">I", len(packet))) + output.write(packet) + os.replace(temporary_path, path) + temporary_path = None + finally: + if temporary_path is not None: + temporary_path.unlink(missing_ok=True) + + +def encode_ogg_opus( + ffmpeg: str, input_path: Path, output_path: Path, bitrate_kbps: int +) -> None: + command = [ + ffmpeg, + "-hide_banner", + "-loglevel", + "error", + "-y", + "-i", + str(input_path), + "-map", + "0:a:0", + "-vn", + "-map_metadata", + "-1", + "-ac", + str(CHANNELS), + "-ar", + str(SAMPLE_RATE_HZ), + "-c:a", + "libopus", + "-application", + "voip", + "-frame_duration", + str(FRAME_DURATION_MS), + "-b:a", + f"{bitrate_kbps}k", + "-vbr", + "off", + "-compression_level", + "10", + str(output_path), + ] + try: + subprocess.run(command, check=True) + except FileNotFoundError as error: + raise ConversionError(f"ffmpeg executable not found: {ffmpeg}") from error + except subprocess.CalledProcessError as error: + raise ConversionError(f"ffmpeg failed with exit code {error.returncode}") from error + + +def convert( + input_path: Path, + output_path: Path, + ffmpeg: str, + bitrate_kbps: int, + overwrite: bool, +) -> tuple[int, int]: + if not input_path.is_file(): + raise ConversionError(f"input file does not exist: {input_path}") + if output_path.exists() and not overwrite: + raise ConversionError( + f"output already exists: {output_path}; pass --force to replace it" + ) + if not 6 <= bitrate_kbps <= 510: + raise ConversionError("bitrate must be between 6 and 510 kbps") + + resolved_ffmpeg = shutil.which(ffmpeg) + if resolved_ffmpeg is None: + raise ConversionError( + f"ffmpeg executable not found: {ffmpeg}; install FFmpeg or pass " + "--ffmpeg /absolute/path/to/ffmpeg" + ) + + with tempfile.TemporaryDirectory(prefix="tq_prompt_opus_") as temp_dir: + ogg_path = Path(temp_dir) / "encoded.ogg" + encode_ogg_opus( + resolved_ffmpeg, input_path, ogg_path, bitrate_kbps + ) + ogg_packets = list(iter_ogg_packets(ogg_path)) + + if len(ogg_packets) < 3 or not ogg_packets[0].startswith(b"OpusHead") or not ogg_packets[1].startswith(b"OpusTags"): + raise ConversionError("FFmpeg output is missing OpusHead/OpusTags packets") + + audio_packets = ogg_packets[2:] + packet_count, payload_bytes = validate_raw_prompt_packets(audio_packets) + write_length_prefixed_packets(output_path, audio_packets) + return packet_count, payload_bytes + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "Convert MP3 or another FFmpeg-supported audio file into the " + "length-prefixed raw Opus prompt format used by TQ_CatN." + ) + ) + parser.add_argument("input", nargs="?", type=Path, help="source audio file") + parser.add_argument("output", nargs="?", type=Path, help="destination .opus file") + parser.add_argument( + "--bitrate-kbps", + type=int, + default=DEFAULT_BITRATE_KBPS, + help=f"Opus bitrate in kbps (default: {DEFAULT_BITRATE_KBPS})", + ) + parser.add_argument( + "--ffmpeg", + default="ffmpeg", + help="FFmpeg executable name or absolute path (default: ffmpeg)", + ) + parser.add_argument( + "--force", action="store_true", help="replace an existing output file" + ) + parser.add_argument( + "--validate-only", + type=Path, + metavar="PROMPT_OPUS", + help="validate an existing device prompt without invoking FFmpeg", + ) + args = parser.parse_args() + if args.validate_only is None and (args.input is None or args.output is None): + parser.error("input and output are required unless --validate-only is used") + if args.validate_only is not None and (args.input is not None or args.output is not None): + parser.error("input/output cannot be combined with --validate-only") + return args + + +def main() -> int: + args = parse_args() + try: + if args.validate_only is not None: + packet_count, payload_bytes = validate_raw_prompt_packets( + read_length_prefixed_packets(args.validate_only) + ) + output_path = args.validate_only + action = "Validated" + else: + packet_count, payload_bytes = convert( + args.input, + args.output, + args.ffmpeg, + args.bitrate_kbps, + args.force, + ) + output_path = args.output + action = "Created" + except (ConversionError, OSError) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + + print( + f"{action} {output_path}: {packet_count} packets, " + f"{payload_bytes} Opus bytes, {SAMPLE_RATE_HZ} Hz mono, " + f"{FRAME_DURATION_MS} ms frames" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())