修复直播测试与 TTS 预热

This commit is contained in:
2026-08-17 15:44:23 +08:00
parent f623959fd4
commit da8a82bfd5
8 changed files with 718 additions and 676 deletions
+303 -25
View File
@@ -61,6 +61,10 @@ from pathlib import Path
from typing import Any
MIHOYO_SDK_REGISTRY_SUBKEY = r"Software\miHoYoSDK"
MANUAL_LIVE_TEST_DURATION_SECONDS = 30 * 60
BILIBILI_LIVE_STATUS_POLL_TIMEOUT_SECONDS = 45.0
BILIBILI_LIVE_STATUS_POLL_INTERVAL_SECONDS = 2.0
BILIBILI_LIVE_STATUS_REFRESH_SECONDS = 60.0
def delete_mihoyo_sdk_registry(logger: logging.Logger, subkey: str = MIHOYO_SDK_REGISTRY_SUBKEY) -> dict[str, Any]:
@@ -511,6 +515,7 @@ class Config:
tts_all["faster-qwen3-tts"].setdefault("chunk_size", 8)
tts_all["faster-qwen3-tts"].setdefault("append_silence", True)
tts_all["faster-qwen3-tts"].setdefault("streaming", True)
tts_all["faster-qwen3-tts"].setdefault("startup_timeout_seconds", 900)
self.data.setdefault("frontend", {})
if legacy_frontend:
self.data["frontend"].update(legacy_frontend)
@@ -2450,6 +2455,10 @@ class FasterQwen3TTSEngine:
self.cpu_threads = max(1, min(8, int(self.cfg.get("cpu_threads", 4) or 4)))
self.cpu_affinity_count = max(0, int(self.cfg.get("cpu_affinity_count", 8) or 0))
self.process_priority = str(self.cfg.get("process_priority", "below_normal") or "below_normal")
self.startup_timeout_seconds = max(
60,
int(self.cfg.get("startup_timeout_seconds", 900) or 900),
)
self.logger = logger
self._parent = parent
self._worker_ready = False
@@ -2473,6 +2482,7 @@ class FasterQwen3TTSEngine:
},
logger,
synthesis_timeout_seconds=120,
startup_timeout_seconds=self.startup_timeout_seconds,
)
if bool(self.cfg.get("streaming", False)):
self.logger.warning("[FasterQwenTTS] 独立 worker 仅使用非流式模式,已忽略 streaming=true")
@@ -2720,6 +2730,7 @@ class Broadcaster:
self._tts_sequence = 0
self._tts_started = False
self._tts_warmup_done = asyncio.Event()
self._tts_warmup_task: asyncio.Task | None = None
self._broadcast_channels: dict[str, dict[str, str]] = {}
self._stop = False
# 弹幕会话已登录标志: 启动时ping一下 nav API, -101 表示 SESSDATA 失效, 关闭弹幕发送
@@ -2762,9 +2773,14 @@ class Broadcaster:
self.logger.warning("[TTS] 配置热更新时无法调度旧 worker 关闭")
self.logger.info(f"[配置] 播报配置已热更新: danmu={self.enable_danmu}, tts={self.config.tts_provider}")
async def _warmup_tts_in_background(self, tts: TTSEngine):
try:
await tts.warmup()
finally:
self._tts_warmup_done.set()
async def start(self):
if self._tts_started:
await self._tts_warmup_done.wait()
return
self._tts_started = True
if not self.tts.enabled:
@@ -2777,10 +2793,16 @@ class Broadcaster:
for task in (synth_task, play_task):
task.add_done_callback(self._tts_worker_tasks.discard)
try:
if bool(self._tts_queue_cfg.get("warmup_on_start", True)):
await self.tts.warmup()
finally:
if bool(self._tts_queue_cfg.get("warmup_on_start", True)):
warmup_task = asyncio.create_task(
self._warmup_tts_in_background(self.tts),
name="TTS后台预热",
)
self._tts_warmup_task = warmup_task
self._tts_worker_tasks.add(warmup_task)
warmup_task.add_done_callback(self._tts_worker_tasks.discard)
self.logger.info("[TTS队列] 后台预热已启动,不阻塞弹幕监听")
else:
self._tts_warmup_done.set()
self.logger.info(
f"[TTS队列] 流水线启动, pending={self._tts_pending.maxsize}, "
@@ -5285,6 +5307,11 @@ class CommandHandler:
)
def _is_live_time(self) -> bool:
if self.system and (
self.system.manual_live_override_active()
or self.system.bilibili_live_active()
):
return True
return is_within_live_time(self.config.system_cfg)
def _rule_matches(self, rule: dict, text: str) -> bool:
@@ -6867,6 +6894,23 @@ def get_real_room_id(room_id: int, cookie: str = "") -> int:
return real_room_id
def get_bilibili_live_status(room_id: int, cookie: str = "") -> int:
"""返回 B站房间直播状态:0=未开播,1=直播中,其他值按接口原样返回。"""
data = _request_bilibili_json(
f"https://api.live.bilibili.com/room/v1/Room/get_info?room_id={int(room_id)}",
cookie,
f"https://live.bilibili.com/{int(room_id)}",
)
if data.get("code") != 0:
raise RuntimeError(
f"查询直播状态失败: code={data.get('code')} message={data.get('message', '')}"
)
try:
return int((data.get("data") or {}).get("live_status", -1))
except (TypeError, ValueError) as exc:
raise RuntimeError("直播状态接口未返回有效 live_status") from exc
def get_danmu_server(room_id: int, cookie: str = "") -> dict:
"""使用登录 Cookie 获取弹幕服务器和专用 tokentoken 不得回退为 SESSDATA。"""
url = f"https://api.live.bilibili.com/xlive/web-room/v1/index/getDanmuInfo?id={int(room_id)}"
@@ -7241,6 +7285,9 @@ class SystemScheduler:
self._live_session_started_at: datetime | None = None
self._startup_compensation_task: asyncio.Task | None = None
self._startup_compensation_window_key: str | None = None
self._last_live_action_error = ""
self._last_bilibili_live_status: int | None = None
self._last_live_status_refresh_monotonic = 0.0
@property
def cfg(self) -> dict:
@@ -7452,6 +7499,89 @@ class SystemScheduler:
user32.EnumWindows(EnumWindowsProc(callback), 0)
return hwnd_found[0] if hwnd_found else None
@property
def last_live_action_error(self) -> str:
return self._last_live_action_error
@staticmethod
def _force_foreground_window(hwnd: int) -> bool:
"""在同一登录会话中临时合并输入线程,可靠地前置目标窗口。"""
user32 = ctypes.windll.user32
kernel32 = ctypes.windll.kernel32
current_thread = int(kernel32.GetCurrentThreadId())
foreground = int(user32.GetForegroundWindow() or 0)
target_pid = ctypes.wintypes.DWORD()
target_thread = int(user32.GetWindowThreadProcessId(hwnd, ctypes.byref(target_pid)) or 0)
foreground_thread = 0
if foreground:
foreground_pid = ctypes.wintypes.DWORD()
foreground_thread = int(
user32.GetWindowThreadProcessId(foreground, ctypes.byref(foreground_pid)) or 0
)
attached: list[int] = []
try:
for thread_id in {target_thread, foreground_thread}:
if thread_id and thread_id != current_thread:
if user32.AttachThreadInput(current_thread, thread_id, True):
attached.append(thread_id)
if user32.IsIconic(hwnd):
user32.ShowWindow(hwnd, 9) # SW_RESTORE
else:
user32.ShowWindow(hwnd, 5) # SW_SHOW
user32.BringWindowToTop(hwnd)
user32.SetForegroundWindow(hwnd)
user32.SetActiveWindow(hwnd)
user32.SetFocus(hwnd)
finally:
for thread_id in reversed(attached):
user32.AttachThreadInput(current_thread, thread_id, False)
return int(user32.GetForegroundWindow() or 0) == int(hwnd)
async def _get_bilibili_live_status(self, *, record_error: bool = True) -> int | None:
try:
status = await asyncio.to_thread(
get_bilibili_live_status,
int(self.config.room_id),
self.config.bilibili_cookie,
)
self._last_bilibili_live_status = status
if self.system:
self.system.set_bilibili_live_status(status)
return status
except Exception as exc:
message = f"查询 B站直播状态失败: {exc}"
if record_error:
self._last_live_action_error = message
self.logger.warning(f"[系统定时] {message}")
else:
self.logger.debug(f"[系统定时] {message}")
return None
async def _wait_for_bilibili_live_status(
self,
expected_status: int,
*,
timeout_seconds: float = BILIBILI_LIVE_STATUS_POLL_TIMEOUT_SECONDS,
) -> bool:
effective_timeout = max(1.0, float(timeout_seconds))
deadline = time.monotonic() + effective_timeout
while True:
status = await self._get_bilibili_live_status()
if status == expected_status:
self._last_live_action_error = ""
return True
remaining = deadline - time.monotonic()
if remaining <= 0:
state_name = "开播" if expected_status == 1 else "关播"
self._last_live_action_error = (
f"已点击直播姬,但 B站在 {int(effective_timeout)} 秒内未确认{state_name}"
f"live_status={status}"
)
self.logger.warning(f"[系统定时] {self._last_live_action_error}")
return False
await asyncio.sleep(min(BILIBILI_LIVE_STATUS_POLL_INTERVAL_SECONDS, remaining))
async def _click_bilibili_live(
self,
*,
@@ -7460,57 +7590,137 @@ class SystemScheduler:
action_name: str,
confirm_enter: bool = False,
) -> bool:
"""直播姬窗口前置,在指定比例坐标点击;可选按 Enter 确认弹窗"""
"""确认直播姬获得前台后,在恢复后的窗口坐标点击。"""
self._last_live_action_error = ""
if os.name != "nt":
self.logger.warning(f"[系统定时] 当前系统不支持自动{action_name}")
self._last_live_action_error = f"当前系统不支持自动{action_name}"
self.logger.warning(f"[系统定时] {self._last_live_action_error}")
return False
keyword = self.cfg.get("bilibili_push_window_keyword", "直播姬")
hwnd = self._find_window_by_keyword(keyword)
if not hwnd:
self.logger.warning(f"[系统定时] 未找到直播姬窗口,关键字: {keyword}")
self._last_live_action_error = f"未找到直播姬窗口,关键字: {keyword}"
self.logger.warning(f"[系统定时] {self._last_live_action_error}")
return False
user32 = ctypes.windll.user32
rect = ctypes.wintypes.RECT()
if not user32.GetWindowRect(hwnd, ctypes.byref(rect)):
self.logger.warning("[系统定时] 获取直播姬窗口位置失败")
return False
x_ratio = float(self.cfg.get(x_ratio_key, 0.741))
y_ratio = float(self.cfg.get(y_ratio_key, 0.907))
x = int(rect.left + max(0.0, min(1.0, x_ratio)) * (rect.right - rect.left))
y = int(rect.top + max(0.0, min(1.0, y_ratio)) * (rect.bottom - rect.top))
previous_dpi_context = None
set_thread_dpi_context = getattr(user32, "SetThreadDpiAwarenessContext", None)
try:
user32.ShowWindow(hwnd, 9) # SW_RESTORE
user32.SetForegroundWindow(hwnd)
if set_thread_dpi_context:
set_thread_dpi_context.restype = ctypes.c_void_p
previous_dpi_context = set_thread_dpi_context(ctypes.c_void_p(-4))
if user32.IsIconic(hwnd):
user32.ShowWindow(hwnd, 9) # SW_RESTORE
else:
user32.ShowWindow(hwnd, 5) # SW_SHOW
await asyncio.sleep(0.5)
user32.SetCursorPos(x, y)
if not self._force_foreground_window(hwnd):
await asyncio.sleep(0.3)
if int(user32.GetForegroundWindow() or 0) != int(hwnd):
self._last_live_action_error = "直播姬窗口无法获得前台,已取消点击以避免误操作其他窗口"
self.logger.warning(f"[系统定时] {self._last_live_action_error}")
return False
rect = ctypes.wintypes.RECT()
if not user32.GetWindowRect(hwnd, ctypes.byref(rect)):
self._last_live_action_error = "获取直播姬窗口位置失败"
self.logger.warning(f"[系统定时] {self._last_live_action_error}")
return False
width = int(rect.right - rect.left)
height = int(rect.bottom - rect.top)
if width < 200 or height < 150:
self._last_live_action_error = (
f"直播姬窗口尺寸异常: rect=({rect.left},{rect.top},{rect.right},{rect.bottom})"
)
self.logger.warning(f"[系统定时] {self._last_live_action_error}")
return False
x_ratio = float(self.cfg.get(x_ratio_key, 0.741))
y_ratio = float(self.cfg.get(y_ratio_key, 0.907))
x = int(rect.left + max(0.0, min(1.0, x_ratio)) * width)
y = int(rect.top + max(0.0, min(1.0, y_ratio)) * height)
virtual_left = int(user32.GetSystemMetrics(76))
virtual_top = int(user32.GetSystemMetrics(77))
virtual_right = virtual_left + int(user32.GetSystemMetrics(78))
virtual_bottom = virtual_top + int(user32.GetSystemMetrics(79))
if not (virtual_left <= x < virtual_right and virtual_top <= y < virtual_bottom):
self._last_live_action_error = (
f"直播姬点击坐标超出屏幕: x={x}, y={y}, "
f"screen=({virtual_left},{virtual_top},{virtual_right},{virtual_bottom})"
)
self.logger.warning(f"[系统定时] {self._last_live_action_error}")
return False
previous_cursor = ctypes.wintypes.POINT()
has_previous_cursor = bool(user32.GetCursorPos(ctypes.byref(previous_cursor)))
if not user32.SetCursorPos(x, y):
self._last_live_action_error = (
f"无法移动鼠标到直播姬按钮位置: x={x}, y={y}"
"请通过桌面上可见的 run.bat 控制台启动服务后重试"
)
self.logger.warning(f"[系统定时] {self._last_live_action_error}")
return False
await asyncio.sleep(0.1)
if int(user32.GetForegroundWindow() or 0) != int(hwnd):
self._last_live_action_error = "点击前直播姬失去前台,已取消操作"
self.logger.warning(f"[系统定时] {self._last_live_action_error}")
if has_previous_cursor:
user32.SetCursorPos(previous_cursor.x, previous_cursor.y)
return False
user32.mouse_event(0x0002, 0, 0, 0, 0) # LEFTDOWN
await asyncio.sleep(0.05)
user32.mouse_event(0x0004, 0, 0, 0, 0) # LEFTUP
if has_previous_cursor:
user32.SetCursorPos(previous_cursor.x, previous_cursor.y)
if confirm_enter:
await asyncio.sleep(0.8)
user32.keybd_event(0x0D, 0, 0, 0) # VK_RETURN key down
user32.keybd_event(0x0D, 0, 0x0002, 0) # key up
self.logger.info(f"[系统定时] 已点击直播姬{action_name}: hwnd={hwnd}, x={x}, y={y}")
self.logger.info(
f"[系统定时] 已点击直播姬{action_name}: hwnd={hwnd}, x={x}, y={y}, "
f"rect=({rect.left},{rect.top},{rect.right},{rect.bottom})"
)
return True
except Exception as e:
self.logger.warning(f"[系统定时] 点击直播姬{action_name}失败: {e}")
self._last_live_action_error = f"点击直播姬{action_name}失败: {e}"
self.logger.warning(f"[系统定时] {self._last_live_action_error}")
return False
finally:
if set_thread_dpi_context and previous_dpi_context:
try:
set_thread_dpi_context(ctypes.c_void_p(previous_dpi_context))
except Exception:
pass
async def push_bilibili_live(self) -> bool:
return await self._click_bilibili_live(
self._last_live_action_error = ""
if await self._get_bilibili_live_status() == 1:
self.logger.info("[系统定时] B站已处于开播状态,跳过重复点击")
return True
clicked = await self._click_bilibili_live(
x_ratio_key="bilibili_push_click_x_ratio",
y_ratio_key="bilibili_push_click_y_ratio",
action_name="开启推流按钮",
)
if not clicked:
return False
return await self._wait_for_bilibili_live_status(1)
async def stop_bilibili_live(self) -> bool:
return await self._click_bilibili_live(
self._last_live_action_error = ""
if await self._get_bilibili_live_status() == 0:
self.logger.info("[系统定时] B站已处于未开播状态,跳过重复关播点击")
return True
clicked = await self._click_bilibili_live(
x_ratio_key="bilibili_stop_push_click_x_ratio",
y_ratio_key="bilibili_stop_push_click_y_ratio",
action_name="关闭推流按钮",
confirm_enter=bool(self.cfg.get("bilibili_stop_push_confirm_enter", True)),
)
if not clicked:
return False
return await self._wait_for_bilibili_live_status(0)
def _schedule_reboot_after_stop(self, stopped_at: datetime) -> bool:
if not self._is_enabled("reboot_after_stop_enabled"):
@@ -7677,6 +7887,13 @@ class SystemScheduler:
now = datetime.now().replace(second=0, microsecond=0)
today = date.today().isoformat()
try:
monotonic_now = time.monotonic()
if (
monotonic_now - self._last_live_status_refresh_monotonic
>= BILIBILI_LIVE_STATUS_REFRESH_SECONDS
):
self._last_live_status_refresh_monotonic = monotonic_now
await self._get_bilibili_live_status(record_error=False)
for event, when in self._live_occurrences(now):
key = self._event_key(event, when)
if key in self._triggered_events or not self._event_due(now, when):
@@ -7744,6 +7961,11 @@ class QueueSystem:
self.stats_store = stats_store
self.health = ServiceRegistry(stats_store)
self.live_client = None
self._manual_live_override_until_monotonic = 0.0
self._manual_live_override_until_wall = 0.0
self._bilibili_live_status: int | None = None
self._bilibili_live_status_updated_monotonic = 0.0
self._bilibili_live_status_updated_wall = 0.0
self.health.set("主程序", ServiceRegistry.STARTING, "初始化")
self.health.set("配置", ServiceRegistry.RUNNING, f"revision={config.revision}")
self.user_mgr = UserManager(
@@ -7792,6 +8014,54 @@ class QueueSystem:
self.handler.login_monitor = self.login_monitor
self.health.set("主程序", ServiceRegistry.RUNNING, "初始化完成")
def enable_manual_live_override(
self,
duration_seconds: int = MANUAL_LIVE_TEST_DURATION_SECONDS,
) -> float:
duration = max(60, min(int(duration_seconds), 6 * 60 * 60))
self._manual_live_override_until_monotonic = time.monotonic() + duration
self._manual_live_override_until_wall = time.time() + duration
self.logger.info(f"[直播测试] 已临时开放弹幕指令 {duration // 60} 分钟")
return self._manual_live_override_until_wall
def clear_manual_live_override(self) -> None:
was_active = self.manual_live_override_active()
self._manual_live_override_until_monotonic = 0.0
self._manual_live_override_until_wall = 0.0
if was_active:
self.logger.info("[直播测试] 已关闭手动直播指令时段")
def manual_live_override_active(self) -> bool:
if self._manual_live_override_until_monotonic <= time.monotonic():
self._manual_live_override_until_monotonic = 0.0
self._manual_live_override_until_wall = 0.0
return False
return True
def manual_live_override_snapshot(self) -> dict[str, Any]:
active = self.manual_live_override_active()
return {
"active": active,
"until": self._manual_live_override_until_wall if active else 0.0,
}
def set_bilibili_live_status(self, status: int) -> None:
self._bilibili_live_status = int(status)
self._bilibili_live_status_updated_monotonic = time.monotonic()
self._bilibili_live_status_updated_wall = time.time()
def bilibili_live_active(self, max_age_seconds: float = 120.0) -> bool:
status = getattr(self, "_bilibili_live_status", None)
updated = float(getattr(self, "_bilibili_live_status_updated_monotonic", 0.0) or 0.0)
return status == 1 and time.monotonic() - updated <= max(1.0, max_age_seconds)
def bilibili_live_status_snapshot(self) -> dict[str, Any]:
return {
"status": getattr(self, "_bilibili_live_status", None),
"updated_at": float(getattr(self, "_bilibili_live_status_updated_wall", 0.0) or 0.0),
"active": self.bilibili_live_active(),
}
def apply_config(self):
self.config.apply_runtime_settings()
self.user_mgr.config = self.config
@@ -9223,6 +9493,8 @@ class WebServer:
"active_song_request": song_state.get("active"),
"recent_danmu": list(getattr(self.system.handler, "recent_danmu", [])[-25:]),
"recent_gifts": list(getattr(self.system.handler, "recent_gifts", [])[-25:]),
"manual_live_test": self.system.manual_live_override_snapshot(),
"bilibili_live": self.system.bilibili_live_status_snapshot(),
})
return state
@@ -10284,13 +10556,19 @@ class WebServer:
elif action == "test_bilibili_push":
ok = await self.scheduler.push_bilibili_live()
if not ok:
return web.json_response({"success": False, "error": "未找到直播姬窗口或点击失败"}, status=400)
error = self.scheduler.last_live_action_error or "直播姬开播测试失败"
return web.json_response({"success": False, "error": error}, status=400)
manual_until = self.system.enable_manual_live_override()
audit_target = "bilibili_push"
audit_detail = f"live confirmed manual_until={manual_until:.0f}"
elif action == "test_bilibili_stop_push":
ok = await self.scheduler.stop_bilibili_live()
if not ok:
return web.json_response({"success": False, "error": "未找到直播姬窗口或关闭推流点击失败"}, status=400)
error = self.scheduler.last_live_action_error or "直播姬关播测试失败"
return web.json_response({"success": False, "error": error}, status=400)
self.system.clear_manual_live_override()
audit_target = "bilibili_stop_push"
audit_detail = "offline confirmed"
elif action == "save_system_schedule_config":
data = await request.json()
self.config.reload()
+112 -18
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
import io
import multiprocessing
import os
import sys
import threading
import time
import traceback
@@ -13,13 +14,14 @@ from typing import Any, Callable
MAX_NEW_TOKENS = 384
DEFAULT_SYNTHESIS_TIMEOUT_SECONDS = 120.0
DEFAULT_STARTUP_TIMEOUT_SECONDS = 480.0
DEFAULT_STARTUP_TIMEOUT_SECONDS = 900.0
# 启动失败后再次拉起 worker 的最小间隔:避免"启动超时→立即重启→再超时"的死循环
# 在系统高负载时持续加载 torch/CUDA,进一步加剧卡顿。
STARTUP_FAILURE_BACKOFF_SECONDS = 60.0
DEFAULT_CPU_THREADS = 4
DEFAULT_CPU_AFFINITY_COUNT = 8
DEFAULT_PROCESS_PRIORITY = "below_normal"
STARTUP_PROGRESS_POLL_SECONDS = 1.0
class FasterQwenWorkerError(RuntimeError):
@@ -122,15 +124,26 @@ def _apply_worker_process_limits(settings: dict[str, Any]) -> dict[str, Any]:
}
def _load_runtime(settings: dict[str, Any]) -> dict[str, Any]:
def _load_runtime(
settings: dict[str, Any],
progress: Callable[[str], None] | None = None,
) -> dict[str, Any]:
def report(stage: str) -> None:
if progress:
progress(stage)
device = str(settings.get("device") or "cuda")
if device == "cpu" and "CUDA_VISIBLE_DEVICES" not in os.environ:
os.environ["CUDA_VISIBLE_DEVICES"] = ""
cpu_threads = _configure_worker_environment(settings)
report("torch_import_started")
import torch
report("torch_import_complete")
_configure_torch_threads(torch, cpu_threads)
report("tts_library_import_started")
from faster_qwen3_tts import FasterQwen3TTS
report("tts_library_import_complete")
if device == "cpu":
torch.cuda.is_available = lambda: False
@@ -138,10 +151,12 @@ def _load_runtime(settings: dict[str, Any]) -> dict[str, Any]:
load_kwargs: dict[str, Any] = {}
if device == "cpu":
load_kwargs["device"] = "cpu"
report("from_pretrained_started")
model = FasterQwen3TTS.from_pretrained(
str(settings.get("model_name_or_path") or "Qwen/Qwen3-TTS-12Hz-0.6B-Base"),
**load_kwargs,
)
report("from_pretrained_complete")
# voice_clone_prompt 不再预计算:当前 faster_qwen3_tts 的 FasterQwen3TTS 没有
# create_voice_clone_prompt 方法,预计算只会失败。改为每次合成时在
@@ -193,21 +208,63 @@ def _synthesize_wav(runtime: dict[str, Any], text: str) -> bytes:
def faster_qwen_worker_main(connection: Connection, settings: dict[str, Any]) -> None:
def report(stage: str, **details: Any) -> None:
try:
connection.send({
"type": "progress",
"stage": stage,
"pid": os.getpid(),
**details,
})
except Exception:
pass
try:
report("process_started", executable=sys.executable, prefix=sys.prefix)
cpu_threads = _configure_worker_environment(settings)
process_limits = _apply_worker_process_limits(settings)
report("environment_ready", cpu_threads=cpu_threads)
load_started = time.monotonic()
runtime = _load_runtime(settings)
report("model_loading", model_name=str(settings.get("model_name_or_path") or ""))
runtime = _load_runtime(settings, progress=report)
load_ms = int((time.monotonic() - load_started) * 1000)
report("model_loaded", load_ms=load_ms)
warmup_started = time.monotonic()
graph_warmup_ms = 0
report("warmup_started")
warmup = getattr(runtime["model"], "warmup", None)
if callable(warmup):
graph_warmup_started = time.monotonic()
warmup()
graph_warmup_ms = int((time.monotonic() - graph_warmup_started) * 1000)
report("graph_warmup_complete", graph_warmup_ms=graph_warmup_ms)
# model.warmup() 只捕获模型内部 CUDA Graph,不会初始化参考音频和
# voice-clone 生成路径。若把这一步留到首条播报,后台会在近一分钟内
# 看起来毫无响应,因此在降低进程优先级之前完成一次真实短句合成。
voice_clone_warmup_started = time.monotonic()
report("voice_clone_warmup_started")
_synthesize_wav(runtime, "系统启动")
voice_clone_warmup_ms = int((time.monotonic() - voice_clone_warmup_started) * 1000)
report("voice_clone_warmup_complete", voice_clone_warmup_ms=voice_clone_warmup_ms)
warmup_ms = int((time.monotonic() - warmup_started) * 1000)
report(
"warmup_complete",
warmup_ms=warmup_ms,
graph_warmup_ms=graph_warmup_ms,
voice_clone_warmup_ms=voice_clone_warmup_ms,
)
# 加载和 CUDA Graph 捕获需要完整 CPU 调度能力。仅在预热完成后降低
# worker 优先级和亲和性,避免繁忙直播环境下启动时间被放大到超时。
process_limits = _apply_worker_process_limits(settings)
connection.send({
"type": "ready",
"pid": os.getpid(),
"load_ms": load_ms,
"warmup_ms": warmup_ms,
"graph_warmup_ms": graph_warmup_ms,
"voice_clone_warmup_ms": voice_clone_warmup_ms,
"max_new_tokens": MAX_NEW_TOKENS,
"cpu_threads": cpu_threads,
**process_limits,
@@ -278,7 +335,7 @@ class FasterQwenWorkerClient:
self.settings = dict(settings)
self.logger = logger
self.synthesis_timeout_seconds = max(1.0, float(synthesis_timeout_seconds))
self.startup_timeout_seconds = max(10.0, float(startup_timeout_seconds))
self.startup_timeout_seconds = max(0.1, float(startup_timeout_seconds))
self._context = context or multiprocessing.get_context("spawn")
self._process_target = process_target or faster_qwen_worker_main
self._lock = threading.RLock()
@@ -330,21 +387,55 @@ class FasterQwenWorkerClient:
self._worker_pid = int(getattr(process, "pid", 0) or 0)
self._log("info", "[FasterQwenTTS] worker 已启动, pid=%s,正在加载和预热", self.worker_pid)
if not parent_connection.poll(self.startup_timeout_seconds):
self._next_start_after = time.monotonic() + STARTUP_FAILURE_BACKOFF_SECONDS
self._terminate_worker_locked("startup_timeout", graceful=False)
raise FasterQwenWorkerError(
f"Faster-Qwen3-TTS worker startup exceeded {self.startup_timeout_seconds:.0f}s"
)
try:
message = parent_connection.recv()
except (EOFError, OSError) as exc:
self._next_start_after = time.monotonic() + STARTUP_FAILURE_BACKOFF_SECONDS
self._terminate_worker_locked("startup_connection_closed", graceful=False)
raise FasterQwenWorkerError("Faster-Qwen3-TTS worker exited during startup") from exc
started = time.monotonic()
deadline = started + self.startup_timeout_seconds
last_stage = "process_spawned"
message: Any = None
while True:
remaining = deadline - time.monotonic()
if remaining <= 0:
self._next_start_after = time.monotonic() + STARTUP_FAILURE_BACKOFF_SECONDS
self._terminate_worker_locked("startup_timeout", graceful=False)
raise FasterQwenWorkerError(
f"Faster-Qwen3-TTS worker startup exceeded {self.startup_timeout_seconds:.0f}s "
f"(last_stage={last_stage})"
)
if not parent_connection.poll(min(STARTUP_PROGRESS_POLL_SECONDS, remaining)):
if not process.is_alive():
exit_code = getattr(process, "exitcode", None)
self._next_start_after = time.monotonic() + STARTUP_FAILURE_BACKOFF_SECONDS
self._terminate_worker_locked("startup_process_exited", graceful=False)
raise FasterQwenWorkerError(
f"Faster-Qwen3-TTS worker exited during startup "
f"(exit_code={exit_code}, last_stage={last_stage})"
)
continue
try:
message = parent_connection.recv()
except (EOFError, OSError) as exc:
self._next_start_after = time.monotonic() + STARTUP_FAILURE_BACKOFF_SECONDS
self._terminate_worker_locked("startup_connection_closed", graceful=False)
raise FasterQwenWorkerError(
f"Faster-Qwen3-TTS worker connection closed during startup "
f"(last_stage={last_stage})"
) from exc
if isinstance(message, dict) and message.get("type") == "progress":
last_stage = str(message.get("stage") or "unknown")
elapsed_ms = int((time.monotonic() - started) * 1000)
self._log(
"info",
"[FasterQwenTTS] worker 启动进度: %s, elapsed=%sms",
last_stage,
elapsed_ms,
)
continue
break
if not isinstance(message, dict) or message.get("type") != "ready":
if isinstance(message, dict):
error = str(message.get("error") or message.get("error_type") or "unknown startup error")
worker_traceback = str(message.get("traceback") or "").strip()
if worker_traceback:
self._log("error", "[FasterQwenTTS] worker 启动异常:\n%s", worker_traceback)
else:
error = "invalid startup response"
self._next_start_after = time.monotonic() + STARTUP_FAILURE_BACKOFF_SECONDS
@@ -353,10 +444,13 @@ class FasterQwenWorkerClient:
self._next_start_after = 0.0
self._log(
"info",
"[FasterQwenTTS] worker 预热完成, pid=%s load=%sms warmup=%sms max_new_tokens=%s",
"[FasterQwenTTS] worker 预热完成, pid=%s load=%sms warmup=%sms "
"graph=%sms voice_clone=%sms max_new_tokens=%s",
self.worker_pid,
message.get("load_ms"),
message.get("warmup_ms"),
message.get("graph_warmup_ms"),
message.get("voice_clone_warmup_ms"),
message.get("max_new_tokens"),
)
self._log(