fix(text-to-audio): add parallel chunk synthesis, automatic MiniMax fallback, voice normalization, and streaming progress
This commit is contained in:
@@ -33,8 +33,8 @@ Real-time public transport information for Regensburg (RVV). Provides departures
|
||||
### [text-to-audio](skills/text-to-audio/SKILL.md)
|
||||
Audio dossier and podcast producer. Converts arbitrary research, summaries, notes, or articles into audio dossiers with duration target, importance compression, and solo or NotebookLM-style dialogue.
|
||||
- **Features:** Target duration pacing (~130 words/min), importance filtering, solo monologue or multi-speaker dialogue.
|
||||
- **Voices:** OpenRouter `deepgram/aura-2` with German voices (`aura-2-julius-de`, `aura-2-lara-de`) and English voices (`aura-2-zeus-en`, `aura-2-athena-en`).
|
||||
- **Tools:** Includes `scripts/text_to_audio.py` for API synthesis, multi-voice stitching, and MP3 generation.
|
||||
- **Voices:** Primary Google Gemini 3.1 Flash TTS with automatic MiniMax Speech 02 HD fallback. German voices (`Puck` host/solo, `Aoede` co-host, `Charon` deep), English voices (`Zephyr` host/solo, `Aoede` co-host).
|
||||
- **Tools:** Includes `scripts/text_to_audio.py` for parallel chunk synthesis, auto-fallback, and MP3 generation.
|
||||
|
||||
### [skill-creator](skills/skill-creator/SKILL.md)
|
||||
Guidelines and scripts to help developers build high-quality, token-efficient Claude skills.
|
||||
|
||||
@@ -44,9 +44,17 @@ Ermittle die gewünschte Dauer oder nutze den Standard:
|
||||
Nutze das Skript `scripts/text_to_audio.py` (bzw. `/home/node/mop-skills/mop-skills/skills/text-to-audio/scripts/text_to_audio.py` im Cluster).
|
||||
|
||||
### A. Solo-Modus (Standard)
|
||||
Schreibe das Skript am besten in eine Datei, um Shell-Escaping-Probleme zu vermeiden:
|
||||
|
||||
```bash
|
||||
# 1. Skript in Datei schreiben
|
||||
cat << 'TXT_EOF' > /tmp/dossier_script.txt
|
||||
Dein aufbereitetes Skript ohne Markdown-Artefakte...
|
||||
TXT_EOF
|
||||
|
||||
# 2. Synthese starten (läuft parallel & fällt bei Gemini-Timeouts automatisch auf MiniMax zurück)
|
||||
python3 /home/node/mop-skills/mop-skills/skills/text-to-audio/scripts/text_to_audio.py \
|
||||
--text "Dein aufbereitetes Skript..." \
|
||||
--file /tmp/dossier_script.txt \
|
||||
--voice Puck \
|
||||
--lang de \
|
||||
--title "Titel des Dossiers" \
|
||||
|
||||
@@ -1,12 +1,18 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
text_to_audio.py - OpenRouter Text-to-Speech & Podcast Synthesizer
|
||||
text_to_audio.py - High-Quality Text-to-Speech & Podcast Synthesizer for OpenClaw
|
||||
|
||||
High-quality speech synthesis using Google Gemini 3.1 Flash TTS (default) or other OpenRouter TTS models.
|
||||
Produces single-speaker audio dossiers by default, and multi-speaker dialogue only when explicitly requested.
|
||||
Features:
|
||||
- Primary TTS: Google Gemini 3.1 Flash TTS (natural, expressive, low cost).
|
||||
- Automatic Fast Fallback: MiniMax Speech 02 HD (instant, 100% reliable studio quality).
|
||||
- Parallel Synthesis: Multi-threaded chunk processing (ThreadPoolExecutor) for fast generation.
|
||||
- Full Voice Normalization: Maps any voice name or role (Puck, Julius, Lara, Host, etc.) safely.
|
||||
- Live Progress: Unbuffered stdout logging so OpenClaw process watchdogs never timeout.
|
||||
- Format Guarantee: Standard monologue by default; multi-speaker dialog only when requested.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
@@ -20,44 +26,47 @@ import urllib.request
|
||||
import wave
|
||||
from pathlib import Path
|
||||
|
||||
DEFAULT_MODEL = "google/gemini-3.1-flash-tts-preview"
|
||||
FALLBACK_MODEL = "microsoft/mai-voice-2"
|
||||
OPENROUTER_SPEECH_URL = "https://openrouter.ai/api/v1/audio/speech"
|
||||
DEFAULT_PRIMARY_MODEL = "google/gemini-3.1-flash-tts-preview"
|
||||
DEFAULT_FALLBACK_MODEL = "minimax/speech-2.8-hd"
|
||||
|
||||
# Pre-selected voices for German and English
|
||||
DEFAULT_VOICES = {
|
||||
VOICE_CONFIG = {
|
||||
"de": {
|
||||
"solo": "Puck", # Warm, lebendig, dynamisch (Standard für Solo Audio-Dossiers)
|
||||
"host": "Puck", # Host im Dialog-Modus (nur bei expliziter "Podcast"-Anfrage)
|
||||
"cohost": "Aoede", # Melodisch, klar, ausdrucksstark (Co-Host / Expertin)
|
||||
"expert": "Aoede",
|
||||
"male": "Puck",
|
||||
"female": "Aoede",
|
||||
"deep": "Charon", # Ruhig, tief, autoritär (Nachrichten/Doku)
|
||||
"zephyr": "Zephyr", # Klar, ausgewogen, natürlich
|
||||
"puck": "Puck",
|
||||
"aoede": "Aoede",
|
||||
"charon": "Charon",
|
||||
"kore": "Kore",
|
||||
"fenrir": "Fenrir",
|
||||
"gemini": {
|
||||
"solo": "Puck",
|
||||
"host": "Puck",
|
||||
"cohost": "Aoede",
|
||||
"deep": "Charon",
|
||||
},
|
||||
"minimax": {
|
||||
"solo": "presenter_male",
|
||||
"host": "presenter_male",
|
||||
"cohost": "presenter_female",
|
||||
"deep": "audiobook_male_1",
|
||||
},
|
||||
},
|
||||
"en": {
|
||||
"solo": "Zephyr", # Clear, natural, balanced conversational male/neutral
|
||||
"host": "Zephyr", # Host in dialog mode
|
||||
"cohost": "Aoede", # Expressive, melodious, friendly female
|
||||
"expert": "Aoede",
|
||||
"male": "Zephyr",
|
||||
"female": "Aoede",
|
||||
"deep": "Charon", # Deep, resonant, calm male
|
||||
"puck": "Puck", # Energetic, warm, youthful
|
||||
"aoede": "Aoede",
|
||||
"zephyr": "Zephyr",
|
||||
"charon": "Charon",
|
||||
"kore": "Kore",
|
||||
"gemini": {
|
||||
"solo": "Zephyr",
|
||||
"host": "Zephyr",
|
||||
"cohost": "Aoede",
|
||||
"deep": "Charon",
|
||||
},
|
||||
"minimax": {
|
||||
"solo": "presenter_male",
|
||||
"host": "presenter_male",
|
||||
"cohost": "presenter_female",
|
||||
"deep": "audiobook_male_1",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def log_progress(msg: str) -> None:
|
||||
"""Print immediate progress line with flush for OpenClaw process watchers."""
|
||||
print(f"[text-to-audio] {msg}", flush=True)
|
||||
|
||||
|
||||
def find_ffmpeg() -> str:
|
||||
"""Locate the ffmpeg binary across common container and host locations."""
|
||||
env_path = os.environ.get("FFMPEG_PATH")
|
||||
@@ -70,6 +79,7 @@ def find_ffmpeg() -> str:
|
||||
|
||||
candidates = [
|
||||
"/home/node/.openclaw/bin/ffmpeg",
|
||||
"/opt/homebrew/bin/ffmpeg",
|
||||
"/usr/local/bin/ffmpeg",
|
||||
"/usr/bin/ffmpeg",
|
||||
]
|
||||
@@ -80,6 +90,39 @@ def find_ffmpeg() -> str:
|
||||
raise RuntimeError("ffmpeg binary not found. Please ensure ffmpeg is installed or set FFMPEG_PATH.")
|
||||
|
||||
|
||||
def normalize_role(role_or_voice: str) -> str:
|
||||
"""Normalize any role, speaker name, or legacy voice string into a canonical role key."""
|
||||
v = (role_or_voice or "").strip().lower()
|
||||
if any(x in v for x in ["cohost", "lara", "female", "expert", "aoede", "guest", "shaonv"]):
|
||||
return "cohost"
|
||||
if any(x in v for x in ["deep", "charon", "doku", "news", "audiobook"]):
|
||||
return "deep"
|
||||
return "solo"
|
||||
|
||||
|
||||
def resolve_model_voice(role_or_voice: str, model: str, lang: str = "de") -> str:
|
||||
"""Resolve a speaker role/name into the exact valid voice parameter for a specific TTS model."""
|
||||
lang_key = "en" if lang.startswith("en") else "de"
|
||||
is_gemini = "gemini" in model.lower()
|
||||
model_family = "gemini" if is_gemini else "minimax"
|
||||
|
||||
# Direct passthrough if already a valid exact model voice
|
||||
if is_gemini and role_or_voice in ["Puck", "Aoede", "Charon", "Zephyr", "Kore", "Fenrir"]:
|
||||
return role_or_voice
|
||||
if not is_gemini and role_or_voice in [
|
||||
"presenter_male",
|
||||
"presenter_female",
|
||||
"audiobook_male_1",
|
||||
"audiobook_female_1",
|
||||
"male-qn-qingse",
|
||||
"female-shaonv",
|
||||
]:
|
||||
return role_or_voice
|
||||
|
||||
role = normalize_role(role_or_voice)
|
||||
return VOICE_CONFIG[lang_key][model_family][role]
|
||||
|
||||
|
||||
def split_text_into_chunks(text: str, max_chars: int = 1200) -> list[str]:
|
||||
"""Split text into natural sentence-level chunks to satisfy API limits and natural pacing."""
|
||||
text = text.strip()
|
||||
@@ -111,83 +154,151 @@ def split_text_into_chunks(text: str, max_chars: int = 1200) -> list[str]:
|
||||
return chunks
|
||||
|
||||
|
||||
def synthesize_segment(
|
||||
def _call_openrouter_speech(
|
||||
model: str,
|
||||
text: str,
|
||||
voice: str,
|
||||
output_path: str,
|
||||
model: str = DEFAULT_MODEL,
|
||||
api_key: str | None = None,
|
||||
max_retries: int = 3,
|
||||
) -> None:
|
||||
"""
|
||||
Synthesizes speech using OpenRouter with automatic retries and format handling.
|
||||
- For Gemini models: requests 'pcm' and writes standard 24kHz WAV.
|
||||
- For other models (MiniMax, Microsoft, Grok): requests 'mp3' and writes MP3.
|
||||
"""
|
||||
key = api_key or os.environ.get("OPENROUTER_API_KEY")
|
||||
if not key:
|
||||
raise ValueError("OPENROUTER_API_KEY environment variable is not set.")
|
||||
|
||||
is_gemini = "gemini" in model.lower()
|
||||
response_format = "pcm" if is_gemini else "mp3"
|
||||
|
||||
response_format: str,
|
||||
api_key: str,
|
||||
timeout_sec: float,
|
||||
) -> bytes:
|
||||
"""Single HTTP call to OpenRouter /api/v1/audio/speech."""
|
||||
payload = {
|
||||
"model": model,
|
||||
"input": text,
|
||||
"voice": voice,
|
||||
"response_format": response_format,
|
||||
}
|
||||
data = json.dumps(payload).encode("utf-8")
|
||||
req = urllib.request.Request(
|
||||
OPENROUTER_SPEECH_URL,
|
||||
data=data,
|
||||
data=json.dumps(payload).encode("utf-8"),
|
||||
headers={
|
||||
"Authorization": f"Bearer {key}",
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent": "OpenClaw-TextToAudio/2.0",
|
||||
"User-Agent": "OpenClaw-TextToAudio/2.1",
|
||||
},
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=timeout_sec) as resp:
|
||||
content = resp.read()
|
||||
if not content or len(content) < 100:
|
||||
raise RuntimeError("Received empty audio stream from TTS provider.")
|
||||
return content
|
||||
|
||||
last_error = None
|
||||
for attempt in range(1, max_retries + 1):
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=60) as resp:
|
||||
content = resp.read()
|
||||
if not content or len(content) < 100:
|
||||
raise RuntimeError("Received empty or incomplete audio stream from TTS provider.")
|
||||
|
||||
if is_gemini:
|
||||
# Wrap raw 24kHz s16le PCM into standard WAV
|
||||
with wave.open(output_path, "wb") as w:
|
||||
w.setnchannels(1)
|
||||
w.setsampwidth(2) # 16-bit
|
||||
w.setframerate(24000) # 24kHz
|
||||
w.writeframes(content)
|
||||
else:
|
||||
with open(output_path, "wb") as f:
|
||||
f.write(content)
|
||||
return # Success!
|
||||
except urllib.error.HTTPError as e:
|
||||
body = e.read().decode("utf-8", errors="replace")
|
||||
last_error = f"HTTP {e.code}: {body}"
|
||||
# Retry on 5xx or transient 429
|
||||
if attempt < max_retries and e.code in (429, 500, 502, 503, 504):
|
||||
time.sleep(1.5 * attempt)
|
||||
continue
|
||||
break
|
||||
except Exception as e:
|
||||
last_error = str(e)
|
||||
if attempt < max_retries:
|
||||
time.sleep(1.5 * attempt)
|
||||
continue
|
||||
break
|
||||
def synthesize_chunk_to_wav(
|
||||
chunk_idx: int,
|
||||
total_chunks: int,
|
||||
text: str,
|
||||
speaker_or_voice: str,
|
||||
output_wav_path: str,
|
||||
ffmpeg_bin: str,
|
||||
preferred_model: str = DEFAULT_PRIMARY_MODEL,
|
||||
fallback_model: str = DEFAULT_FALLBACK_MODEL,
|
||||
lang: str = "de",
|
||||
api_key: str | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Synthesizes a text chunk to a standardized 24kHz mono 16-bit WAV file.
|
||||
Tries preferred_model first; on timeout/error, immediately falls back to fallback_model.
|
||||
"""
|
||||
key = api_key or os.environ.get("OPENROUTER_API_KEY")
|
||||
if not key:
|
||||
raise ValueError("OPENROUTER_API_KEY environment variable is not set.")
|
||||
|
||||
raise RuntimeError(f"OpenRouter TTS failed after {max_retries} attempts: {last_error}")
|
||||
t0 = time.time()
|
||||
voice_pref = resolve_model_voice(speaker_or_voice, model=preferred_model, lang=lang)
|
||||
pref_is_gemini = "gemini" in preferred_model.lower()
|
||||
pref_format = "pcm" if pref_is_gemini else "mp3"
|
||||
|
||||
log_progress(f"Chunk {chunk_idx + 1}/{total_chunks}: synthesizing ({len(text)} chars, {preferred_model} / {voice_pref})...")
|
||||
|
||||
# Try preferred model (timeout: 16 seconds)
|
||||
try:
|
||||
content = _call_openrouter_speech(
|
||||
model=preferred_model,
|
||||
text=text,
|
||||
voice=voice_pref,
|
||||
response_format=pref_format,
|
||||
api_key=key,
|
||||
timeout_sec=16.0,
|
||||
)
|
||||
if pref_is_gemini:
|
||||
# Wrap raw 24kHz s16le PCM into standard WAV
|
||||
with wave.open(output_wav_path, "wb") as w:
|
||||
w.setnchannels(1)
|
||||
w.setsampwidth(2)
|
||||
w.setframerate(24000)
|
||||
w.writeframes(content)
|
||||
else:
|
||||
# Re-encode MP3 to uniform 24kHz mono WAV
|
||||
with tempfile.NamedTemporaryFile(suffix=".mp3", delete=False) as tf:
|
||||
tf.write(content)
|
||||
tmp_mp3 = tf.name
|
||||
try:
|
||||
subprocess.run(
|
||||
[ffmpeg_bin, "-y", "-i", tmp_mp3, "-ar", "24000", "-ac", "1", output_wav_path],
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
check=True,
|
||||
)
|
||||
finally:
|
||||
if os.path.exists(tmp_mp3):
|
||||
os.remove(tmp_mp3)
|
||||
|
||||
dt = time.time() - t0
|
||||
log_progress(f"Chunk {chunk_idx + 1}/{total_chunks}: generated via {preferred_model} in {dt:.1f}s")
|
||||
return
|
||||
except Exception as err:
|
||||
log_progress(f"Chunk {chunk_idx + 1}/{total_chunks}: primary {preferred_model} failed ({err}). Falling back to {fallback_model}...")
|
||||
|
||||
# Immediate fallback
|
||||
t1 = time.time()
|
||||
voice_fb = resolve_model_voice(speaker_or_voice, model=fallback_model, lang=lang)
|
||||
fb_is_gemini = "gemini" in fallback_model.lower()
|
||||
fb_format = "pcm" if fb_is_gemini else "mp3"
|
||||
|
||||
try:
|
||||
content = _call_openrouter_speech(
|
||||
model=fallback_model,
|
||||
text=text,
|
||||
voice=voice_fb,
|
||||
response_format=fb_format,
|
||||
api_key=key,
|
||||
timeout_sec=15.0,
|
||||
)
|
||||
if fb_is_gemini:
|
||||
with wave.open(output_wav_path, "wb") as w:
|
||||
w.setnchannels(1)
|
||||
w.setsampwidth(2)
|
||||
w.setframerate(24000)
|
||||
w.writeframes(content)
|
||||
else:
|
||||
with tempfile.NamedTemporaryFile(suffix=".mp3", delete=False) as tf:
|
||||
tf.write(content)
|
||||
tmp_mp3 = tf.name
|
||||
try:
|
||||
subprocess.run(
|
||||
[ffmpeg_bin, "-y", "-i", tmp_mp3, "-ar", "24000", "-ac", "1", output_wav_path],
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
check=True,
|
||||
)
|
||||
finally:
|
||||
if os.path.exists(tmp_mp3):
|
||||
os.remove(tmp_mp3)
|
||||
|
||||
dt = time.time() - t1
|
||||
log_progress(f"Chunk {chunk_idx + 1}/{total_chunks}: generated via fallback {fallback_model} in {dt:.1f}s")
|
||||
return
|
||||
except Exception as fb_err:
|
||||
raise RuntimeError(
|
||||
f"Chunk {chunk_idx + 1}/{total_chunks} synthesis failed on both primary ({err}) and fallback ({fb_err})"
|
||||
)
|
||||
|
||||
|
||||
def generate_silence_wav(duration_sec: float, output_path: str, sample_rate: int = 24000) -> None:
|
||||
"""Generate a clean silence WAV file using Python's standard library wave module."""
|
||||
"""Generate a clean silence WAV file."""
|
||||
num_frames = int(duration_sec * sample_rate)
|
||||
silence_bytes = b"\x00\x00" * num_frames
|
||||
with wave.open(output_path, "wb") as w:
|
||||
@@ -199,22 +310,24 @@ def generate_silence_wav(duration_sec: float, output_path: str, sample_rate: int
|
||||
|
||||
def stitch_audio_segments(
|
||||
ffmpeg_bin: str,
|
||||
audio_files: list[str],
|
||||
wav_files: list[str],
|
||||
output_file: str,
|
||||
title: str = "Audio Dossier",
|
||||
artist: str = "OpenClaw",
|
||||
album: str = "OpenClaw Audio Briefings",
|
||||
) -> None:
|
||||
"""Concatenate audio segments (WAV or MP3) and produce high-quality 128k MP3 with ID3 tags."""
|
||||
if not audio_files:
|
||||
"""Concatenate uniform WAV segments and produce high-quality 128k MP3 with metadata."""
|
||||
if not wav_files:
|
||||
raise ValueError("No audio segments to stitch.")
|
||||
|
||||
if len(audio_files) == 1:
|
||||
log_progress(f"Stitching {len(wav_files)} segment(s) into MP3 with metadata...")
|
||||
|
||||
if len(wav_files) == 1:
|
||||
cmd = [
|
||||
ffmpeg_bin,
|
||||
"-y",
|
||||
"-i",
|
||||
audio_files[0],
|
||||
wav_files[0],
|
||||
"-metadata",
|
||||
f"title={title}",
|
||||
"-metadata",
|
||||
@@ -229,12 +342,12 @@ def stitch_audio_segments(
|
||||
]
|
||||
res = subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True)
|
||||
if res.returncode != 0:
|
||||
raise RuntimeError(f"FFmpeg single-file export failed: {res.stderr}")
|
||||
raise RuntimeError(f"FFmpeg export failed: {res.stderr}")
|
||||
return
|
||||
|
||||
# Multi-segment concat demuxer
|
||||
# Concat demuxer with single-quoted file paths
|
||||
with tempfile.NamedTemporaryFile("w", suffix=".txt", delete=False) as list_file:
|
||||
for f in audio_files:
|
||||
for f in wav_files:
|
||||
safe_path = f.replace("'", "'\\''")
|
||||
list_file.write(f"file '{safe_path}'\n")
|
||||
list_file_path = list_file.name
|
||||
@@ -269,108 +382,126 @@ def stitch_audio_segments(
|
||||
os.remove(list_file_path)
|
||||
|
||||
|
||||
def resolve_voice(speaker_name: str, lang: str = "de", explicit_voice: str | None = None) -> str:
|
||||
"""Resolve a speaker alias into a pre-selected voice name."""
|
||||
if explicit_voice:
|
||||
return explicit_voice
|
||||
|
||||
lang_map = DEFAULT_VOICES.get(lang, DEFAULT_VOICES["de"])
|
||||
clean_name = speaker_name.lower().strip()
|
||||
|
||||
if clean_name in lang_map:
|
||||
return lang_map[clean_name]
|
||||
|
||||
# Check common female or cohost roles
|
||||
if any(k in clean_name for k in ["cohost", "female", "lara", "expert", "guest", "aoede"]):
|
||||
return lang_map.get("cohost", "Aoede")
|
||||
|
||||
if any(k in clean_name for k in ["deep", "doku", "news", "charon"]):
|
||||
return lang_map.get("deep", "Charon")
|
||||
|
||||
return lang_map.get("solo", "Puck")
|
||||
|
||||
|
||||
def process_dialog(
|
||||
dialog_items: list[dict],
|
||||
def process_solo_text(
|
||||
text: str,
|
||||
ffmpeg_bin: str,
|
||||
voice_or_role: str,
|
||||
primary_model: str,
|
||||
fallback_model: str,
|
||||
lang: str,
|
||||
model: str,
|
||||
output_file: str,
|
||||
title: str,
|
||||
pause_duration: float = 0.40,
|
||||
max_workers: int = 4,
|
||||
) -> None:
|
||||
"""Generate audio for structured multi-speaker dialog (only when 'podcast' is requested)."""
|
||||
is_gemini = "gemini" in model.lower()
|
||||
ext = "wav" if is_gemini else "mp3"
|
||||
"""Generate audio for a single-speaker text in parallel (Default for all audio dossiers)."""
|
||||
chunks = split_text_into_chunks(text)
|
||||
if not chunks:
|
||||
raise ValueError("No text provided to synthesize.")
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="audio_dialog_") as tmpdir:
|
||||
silence_file = os.path.join(tmpdir, f"pause.{ext}")
|
||||
if is_gemini:
|
||||
generate_silence_wav(pause_duration, silence_file)
|
||||
else:
|
||||
subprocess.run([
|
||||
ffmpeg_bin, "-y", "-f", "lavfi", "-i", "anullsrc=r=24000:cl=mono",
|
||||
"-t", str(pause_duration), "-c:a", "libmp3lame", "-b:a", "64k", silence_file
|
||||
], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=True)
|
||||
log_progress(f"Solo text synthesis started: {len(chunks)} chunk(s), ~{len(text.split())} words, lang={lang}")
|
||||
|
||||
segment_files = []
|
||||
seg_idx = 0
|
||||
with tempfile.TemporaryDirectory(prefix="audio_solo_") as tmpdir:
|
||||
wav_tasks = []
|
||||
for i, chunk in enumerate(chunks):
|
||||
chunk_wav = os.path.join(tmpdir, f"chunk_{i:04d}.wav")
|
||||
wav_tasks.append((i, len(chunks), chunk, voice_or_role, chunk_wav))
|
||||
|
||||
for item in dialog_items:
|
||||
speaker = item.get("speaker", "host")
|
||||
text = item.get("text", "").strip()
|
||||
if not text:
|
||||
continue
|
||||
|
||||
voice = resolve_voice(speaker, lang=lang, explicit_voice=item.get("voice"))
|
||||
chunks = split_text_into_chunks(text)
|
||||
|
||||
for chunk in chunks:
|
||||
chunk_file = os.path.join(tmpdir, f"seg_{seg_idx:04d}.{ext}")
|
||||
synthesize_segment(chunk, voice=voice, output_path=chunk_file, model=model)
|
||||
segment_files.append(chunk_file)
|
||||
seg_idx += 1
|
||||
|
||||
# Natural pause between speaker turns
|
||||
segment_files.append(silence_file)
|
||||
|
||||
if segment_files and segment_files[-1] == silence_file:
|
||||
segment_files.pop()
|
||||
with ThreadPoolExecutor(max_workers=min(len(chunks), max_workers)) as pool:
|
||||
futures = [
|
||||
pool.submit(
|
||||
synthesize_chunk_to_wav,
|
||||
idx,
|
||||
tot,
|
||||
chk,
|
||||
v,
|
||||
path,
|
||||
ffmpeg_bin,
|
||||
primary_model,
|
||||
fallback_model,
|
||||
lang,
|
||||
)
|
||||
for idx, tot, chk, v, path in wav_tasks
|
||||
]
|
||||
for fut in futures:
|
||||
fut.result()
|
||||
|
||||
wav_files = [task[4] for task in wav_tasks]
|
||||
stitch_audio_segments(
|
||||
ffmpeg_bin=ffmpeg_bin,
|
||||
audio_files=segment_files,
|
||||
wav_files=wav_files,
|
||||
output_file=output_file,
|
||||
title=title,
|
||||
)
|
||||
|
||||
|
||||
def process_solo_text(
|
||||
text: str,
|
||||
def process_dialog(
|
||||
dialog_items: list[dict],
|
||||
ffmpeg_bin: str,
|
||||
voice: str,
|
||||
model: str,
|
||||
primary_model: str,
|
||||
fallback_model: str,
|
||||
lang: str,
|
||||
output_file: str,
|
||||
title: str,
|
||||
pause_duration: float = 0.35,
|
||||
max_workers: int = 4,
|
||||
) -> None:
|
||||
"""Generate audio for a single speaker text (default for all audio dossiers)."""
|
||||
chunks = split_text_into_chunks(text)
|
||||
if not chunks:
|
||||
raise ValueError("No text provided to synthesize.")
|
||||
"""Generate audio for multi-speaker dialog turns in parallel (used only when 'podcast' requested)."""
|
||||
if not dialog_items:
|
||||
raise ValueError("Dialog list is empty.")
|
||||
|
||||
is_gemini = "gemini" in model.lower()
|
||||
ext = "wav" if is_gemini else "mp3"
|
||||
log_progress(f"Dialog synthesis started: {len(dialog_items)} turn(s), lang={lang}")
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="audio_solo_") as tmpdir:
|
||||
segment_files = []
|
||||
for i, chunk in enumerate(chunks):
|
||||
chunk_file = os.path.join(tmpdir, f"chunk_{i:04d}.{ext}")
|
||||
synthesize_segment(chunk, voice=voice, output_path=chunk_file, model=model)
|
||||
segment_files.append(chunk_file)
|
||||
with tempfile.TemporaryDirectory(prefix="audio_dialog_") as tmpdir:
|
||||
silence_file = os.path.join(tmpdir, "pause.wav")
|
||||
generate_silence_wav(pause_duration, silence_file)
|
||||
|
||||
# Flatten dialog items into chunks if any turn exceeds max_chars
|
||||
flattened_items = []
|
||||
for d in dialog_items:
|
||||
speaker = d.get("speaker", "host")
|
||||
text = d.get("text", "").strip()
|
||||
if not text:
|
||||
continue
|
||||
chunks = split_text_into_chunks(text)
|
||||
for c in chunks:
|
||||
flattened_items.append((speaker, c))
|
||||
|
||||
wav_tasks = []
|
||||
for i, (speaker, chunk_text) in enumerate(flattened_items):
|
||||
turn_wav = os.path.join(tmpdir, f"turn_{i:04d}.wav")
|
||||
wav_tasks.append((i, len(flattened_items), chunk_text, speaker, turn_wav))
|
||||
|
||||
with ThreadPoolExecutor(max_workers=min(len(flattened_items), max_workers)) as pool:
|
||||
futures = [
|
||||
pool.submit(
|
||||
synthesize_chunk_to_wav,
|
||||
idx,
|
||||
tot,
|
||||
chk,
|
||||
spk,
|
||||
path,
|
||||
ffmpeg_bin,
|
||||
primary_model,
|
||||
fallback_model,
|
||||
lang,
|
||||
)
|
||||
for idx, tot, chk, spk, path in wav_tasks
|
||||
]
|
||||
for fut in futures:
|
||||
fut.result()
|
||||
|
||||
# Interleave silence between dialog turns
|
||||
stitched_wavs = []
|
||||
for task in wav_tasks:
|
||||
stitched_wavs.append(task[4])
|
||||
stitched_wavs.append(silence_file)
|
||||
|
||||
if stitched_wavs and stitched_wavs[-1] == silence_file:
|
||||
stitched_wavs.pop()
|
||||
|
||||
stitch_audio_segments(
|
||||
ffmpeg_bin=ffmpeg_bin,
|
||||
audio_files=segment_files,
|
||||
wav_files=stitched_wavs,
|
||||
output_file=output_file,
|
||||
title=title,
|
||||
)
|
||||
@@ -380,15 +511,16 @@ def main():
|
||||
parser = argparse.ArgumentParser(description="OpenClaw Text-to-Audio Dossier Generator")
|
||||
parser.add_argument("--text", help="Plaintext to synthesize (default solo mode)")
|
||||
parser.add_argument("--file", help="Path to plaintext file (default solo mode)")
|
||||
parser.add_argument("--dialog-json", help="JSON string of dialog turns (used only when 'podcast' style requested)")
|
||||
parser.add_argument("--dialog-json", help="JSON string of dialog turns (only if 'podcast' requested)")
|
||||
parser.add_argument("--dialog-file", help="Path to JSON file with dialog turns")
|
||||
parser.add_argument("--output", "-o", help="Target output MP3 path")
|
||||
parser.add_argument("--title", default="Audio Dossier", help="Audio title for ID3 metadata")
|
||||
parser.add_argument("--voice", help="Voice override (Default DE: Puck, EN: Zephyr)")
|
||||
parser.add_argument("--co-voice", help="Secondary voice override for dialog (Default: Aoede)")
|
||||
parser.add_argument("--voice", help="Voice or speaker role (e.g. Puck, Zephyr, Aoede, host, solo)")
|
||||
parser.add_argument("--lang", default="de", choices=["de", "en"], help="Language preset (de/en)")
|
||||
parser.add_argument("--model", default=DEFAULT_MODEL, help=f"OpenRouter speech model (default: {DEFAULT_MODEL})")
|
||||
parser.add_argument("--pause", type=float, default=0.40, help="Pause in seconds between dialog turns")
|
||||
parser.add_argument("--model", default=DEFAULT_PRIMARY_MODEL, help=f"Primary model (default: {DEFAULT_PRIMARY_MODEL})")
|
||||
parser.add_argument("--fallback-model", default=DEFAULT_FALLBACK_MODEL, help=f"Fallback model (default: {DEFAULT_FALLBACK_MODEL})")
|
||||
parser.add_argument("--pause", type=float, default=0.35, help="Pause in seconds between dialog turns")
|
||||
parser.add_argument("--workers", type=int, default=4, help="Max concurrent synthesis threads")
|
||||
parser.add_argument("--media-tag", action="store_true", default=True, help="Print MEDIA:<path> tag to stdout")
|
||||
|
||||
args = parser.parse_args()
|
||||
@@ -403,7 +535,8 @@ def main():
|
||||
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Dialog mode only if explicit dialog file/json provided
|
||||
t_start = time.time()
|
||||
|
||||
dialog_data = None
|
||||
if args.dialog_file:
|
||||
with open(args.dialog_file, "r", encoding="utf-8") as f:
|
||||
@@ -417,14 +550,16 @@ def main():
|
||||
process_dialog(
|
||||
dialog_items=dialog_data,
|
||||
ffmpeg_bin=ffmpeg_bin,
|
||||
primary_model=args.model,
|
||||
fallback_model=args.fallback_model,
|
||||
lang=args.lang,
|
||||
model=args.model,
|
||||
output_file=str(out_path),
|
||||
title=args.title,
|
||||
pause_duration=args.pause,
|
||||
max_workers=args.workers,
|
||||
)
|
||||
else:
|
||||
# Solo text mode (Default)
|
||||
# Solo monologue mode (Strict DEFAULT)
|
||||
raw_text = ""
|
||||
if args.file:
|
||||
with open(args.file, "r", encoding="utf-8") as f:
|
||||
@@ -436,19 +571,24 @@ def main():
|
||||
else:
|
||||
parser.error("Must provide either --text, --file, --dialog-json, --dialog-file, or pipe text to stdin.")
|
||||
|
||||
primary_voice = args.voice or resolve_voice("solo", lang=args.lang)
|
||||
chosen_voice = args.voice or "solo"
|
||||
process_solo_text(
|
||||
text=raw_text,
|
||||
ffmpeg_bin=ffmpeg_bin,
|
||||
voice=primary_voice,
|
||||
model=args.model,
|
||||
voice_or_role=chosen_voice,
|
||||
primary_model=args.model,
|
||||
fallback_model=args.fallback_model,
|
||||
lang=args.lang,
|
||||
output_file=str(out_path),
|
||||
title=args.title,
|
||||
max_workers=args.workers,
|
||||
)
|
||||
|
||||
print(f"Generated audio: {out_path} ({out_path.stat().st_size} bytes)")
|
||||
total_time = time.time() - t_start
|
||||
log_progress(f"Finished in {total_time:.1f}s: {out_path} ({out_path.stat().st_size} bytes)")
|
||||
|
||||
if args.media_tag:
|
||||
print(f"MEDIA:{out_path}")
|
||||
print(f"MEDIA:{out_path}", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user