合并外部维护版本并同步自动每日功能

app/faster_qwen_worker.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import gc
 import io
 import multiprocessing
 import os
@@ -18,6 +19,8 @@ DEFAULT_STARTUP_TIMEOUT_SECONDS = 900.0
 # 启动失败后再次拉起 worker 的最小间隔：避免"启动超时→立即重启→再超时"的死循环
 # 在系统高负载时持续加载 torch/CUDA，进一步加剧卡顿。
 STARTUP_FAILURE_BACKOFF_SECONDS = 60.0
+# 每 N 次合成才做一次显存回收（empty_cache 会 stall 共用 GPU 的游戏/编码）
+FREE_GPU_EVERY_N_SYNTH = 50
 DEFAULT_CPU_THREADS = 4
 DEFAULT_CPU_AFFINITY_COUNT = 8
 DEFAULT_PROCESS_PRIORITY = "below_normal"
@@ -207,6 +210,68 @@ def _synthesize_wav(runtime: dict[str, Any], text: str) -> bytes:
     return output.getvalue()
 
 
+def _collect_memory_stats(runtime: dict[str, Any]) -> dict[str, float]:
+    """采集 worker 进程的内存/显存占用，用于定位长跑性能退化（泄漏 vs 碎片化）。"""
+    stats: dict[str, float] = {}
+    torch = runtime.get("torch")
+    try:
+        if torch is not None and torch.cuda.is_available():
+            stats["vram_alloc_mb"] = round(torch.cuda.memory_allocated() / 1048576.0, 1)
+            stats["vram_reserved_mb"] = round(torch.cuda.memory_reserved() / 1048576.0, 1)
+    except Exception:
+        pass
+    try:
+        import psutil
+
+        stats["rss_mb"] = round(psutil.Process().memory_info().rss / 1048576.0, 1)
+    except Exception:
+        pass
+    return stats
+
+
+def _free_gpu_memory(runtime: dict[str, Any]) -> None:
+    """回收碎片化显存与 Python 内存。
+
+    注意：empty_cache 会把缓存显存还给驱动，下次合成需重新申请，
+    在原神/直播姬共用 GPU 时会引发驱动级页表重整，导致 GPU 瞬时停摆
+    （占用掉到 0、直播画面卡顿）。因此不能在每次合成后调用，
+    只能在低频维护窗口（每 FREE_GPU_EVERY_N_SYNTH 次）执行。
+    """
+    gc.collect()
+    torch = runtime.get("torch")
+    try:
+        if torch is not None and torch.cuda.is_available():
+            torch.cuda.empty_cache()
+    except Exception:
+        pass
+
+
+def _append_memory_sample(
+    log_path: str,
+    duration_ms: int,
+    mem: dict[str, float],
+    status: str,
+) -> None:
+    """把每次合成的耗时与内存/显存写入独立日志文件，用于定位长跑退化根因。"""
+    if not log_path:
+        return
+    try:
+        from datetime import datetime
+
+        line = (
+            f"{datetime.now().strftime('%Y-%m-%dT%H:%M:%S')} "
+            f"duration_ms={int(duration_ms)} "
+            f"vram_alloc_mb={mem.get('vram_alloc_mb', '')} "
+            f"vram_reserved_mb={mem.get('vram_reserved_mb', '')} "
+            f"rss_mb={mem.get('rss_mb', '')} "
+            f"status={status}"
+        )
+        with open(log_path, "a", encoding="utf-8") as handle:
+            handle.write(line + "\n")
+    except Exception:
+        pass
+
+
 def faster_qwen_worker_main(connection: Connection, settings: dict[str, Any]) -> None:
     def report(stage: str, **details: Any) -> None:
         try:
@@ -219,6 +284,12 @@ def faster_qwen_worker_main(connection: Connection, settings: dict[str, Any]) ->
         except Exception:
             pass
 
+    # pythonw 无窗口模式下 sys.stdout/sys.stderr 为 None，
+    # 第三方库（transformers/tqdm 等）写入时会抛 NoneType.write，这里兜底到 devnull。
+    if sys.stdout is None:
+        sys.stdout = open(os.devnull, "w", encoding="utf-8")
+    if sys.stderr is None:
+        sys.stderr = open(os.devnull, "w", encoding="utf-8")
     try:
         report("process_started", executable=sys.executable, prefix=sys.prefix)
         cpu_threads = _configure_worker_environment(settings)
@@ -282,6 +353,7 @@ def faster_qwen_worker_main(connection: Connection, settings: dict[str, Any]) ->
         connection.close()
         return
 
+    synth_count = 0
     while True:
         try:
             message = connection.recv()
@@ -297,17 +369,36 @@ def faster_qwen_worker_main(connection: Connection, settings: dict[str, Any]) ->
 
         request_id = str(message.get("request_id") or "")
         started = time.monotonic()
+        memory_log_path = str(settings.get("memory_log_path") or "")
         try:
             audio = _synthesize_wav(runtime, str(message.get("text") or ""))
+            duration_ms = int((time.monotonic() - started) * 1000)
+            mem = _collect_memory_stats(runtime)
+            synth_count += 1
+            if synth_count % FREE_GPU_EVERY_N_SYNTH == 0:
+                _free_gpu_memory(runtime)
+            _append_memory_sample(memory_log_path, duration_ms, mem, "ok")
             connection.send({
                 "type": "result",
                 "request_id": request_id,
                 "audio": audio,
-                "duration_ms": int((time.monotonic() - started) * 1000),
+                "duration_ms": duration_ms,
                 "bytes": len(audio),
                 "max_new_tokens": MAX_NEW_TOKENS,
+                "mem": mem,
             })
         except BaseException as exc:
+            duration_ms = int((time.monotonic() - started) * 1000)
+            mem = _collect_memory_stats(runtime)
+            synth_count += 1
+            if synth_count % FREE_GPU_EVERY_N_SYNTH == 0:
+                _free_gpu_memory(runtime)
+            _append_memory_sample(
+                memory_log_path,
+                duration_ms,
+                mem,
+                f"error:{type(exc).__name__}",
+            )
             try:
                 connection.send({
                     "type": "error",