myron/kino-app
1
0
Fork 0
mirror of https://github.com/myronblair/kino-app synced 2026-07-27 21:18:43 -05:00
Code Issues Packages Projects Releases Wiki Activity

Fix Library Cleanup scan freezing the entire server

Browse Source 
scan() was doing hundreds of blocking Path.is_file()/stat() calls directly
in an async function -- movie/episode/track files often live on the
NFS/CIFS-mounted NAS, so each stat call has real network latency. Since
the backend runs a single-threaded asyncio event loop, this froze every
other request (confirmed live: a plain GET /api/movies hung for 60s+ while
a cleanup scan was in flight) for as long as the scan took.

Moved every filesystem-touching check (missing files, orphaned HLS dirs)
into a plain sync function run via asyncio.to_thread, so the scan can no
longer block anything else.
This commit is contained in:
Myron Blair
2026-07-26 22:35:37 -05:00
parent 5fc37e46e4
commit ef5c13d587
1 changed files with 44 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/library_cleanup.py
+44 -23
View File
@@ -7,6 +7,8 @@ from datetime import datetime, timezone
from pathlib import Path
from typing import Dict, List
import asyncio
def _file_path(storage_type: str, storage_path: str, videos_dir: Path) -> Path:
if storage_type in ("radarr", "sonarr", "scan"):
@@ -14,44 +16,63 @@ def _file_path(storage_type: str, storage_path: str, videos_dir: Path) -> Path:
return videos_dir / storage_path
def _scan_filesystem(movies: List[Dict], episodes: List[Dict], tracks: List[Dict],
videos_dir: Path, hls_dir: Path, live_ids: set) -> Dict[str, List[Dict]]:
"""Every filesystem stat call in the whole scan lives here, run off the event loop via
asyncio.to_thread — movie/episode/track paths are often on an NFS/CIFS-mounted NAS, and
doing hundreds of blocking stats directly in an async function would freeze the entire
server (every other request) for as long as the scan takes."""
out: Dict[str, List[Dict]] = {"missing_files": [], "orphaned_hls": []}
for m in movies:
if m.get("storage_type") not in ("local", "radarr", "scan") or not m.get("storage_path"):
continue
if not _file_path(m["storage_type"], m["storage_path"], videos_dir).is_file():
out["missing_files"].append({"id": m["id"], "content_type": "movie", "label": m.get("title", "Untitled"), "detail": m["storage_path"]})
for e in episodes:
if e.get("_orphaned"):
continue # already reported as orphaned_episodes; don't also report a missing file
if e.get("storage_type") not in ("local", "sonarr", "scan") or not e.get("storage_path"):
continue
label = f"S{e.get('season_number')}E{e.get('episode_number')} {e.get('title') or ''}".strip()
if not _file_path(e["storage_type"], e["storage_path"], videos_dir).is_file():
out["missing_files"].append({"id": e["id"], "content_type": "episode", "label": label, "detail": e["storage_path"]})
for t in tracks:
if not Path(t["storage_path"]).is_file():
out["missing_files"].append({"id": t["id"], "content_type": "track", "label": f"{t.get('artist','')} — {t.get('title','Untitled')}", "detail": t["storage_path"]})
if hls_dir.is_dir():
for d in hls_dir.iterdir():
if d.is_dir() and d.name not in live_ids:
size = sum(f.stat().st_size for f in d.rglob("*") if f.is_file())
out["orphaned_hls"].append({"id": d.name, "content_type": "hls_dir", "label": d.name, "detail": f"{round(size / 1_000_000)} MB reclaimable"})
return out
async def scan(db, videos_dir: Path, hls_dir: Path) -> Dict[str, List[Dict]]:
findings: Dict[str, List[Dict]] = {
"missing_files": [], "orphaned_episodes": [], "orphaned_hls": [],
"stuck_jobs": [], "duplicate_paths": [], "orphaned_subtitles": [], "dangling_refs": [],
}
# --- Missing files ---
movies = await db.movies.find({}, {"_id": 0, "id": 1, "title": 1, "storage_type": 1, "storage_path": 1}).to_list(10000)
for m in movies:
if m.get("storage_type") not in ("local", "radarr", "scan") or not m.get("storage_path"):
continue
if not _file_path(m["storage_type"], m["storage_path"], videos_dir).is_file():
findings["missing_files"].append({"id": m["id"], "content_type": "movie", "label": m.get("title", "Untitled"), "detail": m["storage_path"]})
episodes = await db.episodes.find({}, {"_id": 0, "id": 1, "title": 1, "show_id": 1, "season_number": 1, "episode_number": 1, "storage_type": 1, "storage_path": 1}).to_list(20000)
tracks = await db.tracks.find({}, {"_id": 0, "id": 1, "title": 1, "artist": 1, "storage_path": 1}).to_list(20000)
show_ids = set(await db.shows.distinct("id"))
for e in episodes:
label = f"S{e.get('season_number')}E{e.get('episode_number')} {e.get('title') or ''}".strip()
if e.get("show_id") not in show_ids:
findings["orphaned_episodes"].append({"id": e["id"], "content_type": "episode", "label": label, "detail": f"show_id {e.get('show_id')} not found"})
continue
if e.get("storage_type") not in ("local", "sonarr", "scan") or not e.get("storage_path"):
continue
if not _file_path(e["storage_type"], e["storage_path"], videos_dir).is_file():
findings["missing_files"].append({"id": e["id"], "content_type": "episode", "label": label, "detail": e["storage_path"]})
e["_orphaned"] = True
tracks = await db.tracks.find({}, {"_id": 0, "id": 1, "title": 1, "artist": 1, "storage_path": 1}).to_list(20000)
for t in tracks:
if not Path(t["storage_path"]).is_file():
findings["missing_files"].append({"id": t["id"], "content_type": "track", "label": f"{t.get('artist','')} — {t.get('title','Untitled')}", "detail": t["storage_path"]})
# --- Orphaned HLS output dirs (no matching movie or episode id) ---
live_ids = {m["id"] for m in movies} | {e["id"] for e in episodes}
if hls_dir.is_dir():
for d in hls_dir.iterdir():
if d.is_dir() and d.name not in live_ids:
size = sum(f.stat().st_size for f in d.rglob("*") if f.is_file())
findings["orphaned_hls"].append({"id": d.name, "content_type": "hls_dir", "label": d.name, "detail": f"{round(size / 1_000_000)} MB reclaimable"})
fs_findings = await asyncio.to_thread(_scan_filesystem, movies, episodes, tracks, videos_dir, hls_dir, live_ids)
findings["missing_files"] = fs_findings["missing_files"]
findings["orphaned_hls"] = fs_findings["orphaned_hls"]
# --- Stuck transcode jobs (running with no progress for 2+ hours — likely dead from a restart) ---
now = datetime.now(timezone.utc)