Remove redundant nav entries; parallelize cleanup file checks

- Navbar: drop the Users link (already reachable from Admin's quick-links)
- Admin quick-links: drop Settings (already reachable from the navbar
  for every user)
- library_cleanup.py: parallelize the missing-file stat checks across a
  small thread pool instead of one file at a time — sequential NAS round
  trips were taking minutes for a library of any real size, even after
  moving them off the event loop.
This commit is contained in:
Myron Blair
2026-07-26 23:05:53 -05:00
parent ef5c13d587
commit 43340c0f49
3 changed files with 28 additions and 15 deletions
+28 -11
View File
@@ -3,12 +3,18 @@ Read-only by design: scan() only reports findings; server.py's /library/cleanup/
performs the actual deletion/reset the admin picked, one category at a time. performs the actual deletion/reset the admin picked, one category at a time.
""" """
import shutil import shutil
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone from datetime import datetime, timezone
from pathlib import Path from pathlib import Path
from typing import Dict, List from typing import Dict, List
import asyncio import asyncio
# Movie/episode/track files often live on an NFS/CIFS-mounted NAS, where every is_file() stat
# is a real network round trip. Checking hundreds of them one at a time (even off the event
# loop) can take minutes; a bounded worker pool checks them concurrently instead.
_STAT_WORKERS = 24
def _file_path(storage_type: str, storage_path: str, videos_dir: Path) -> Path: def _file_path(storage_type: str, storage_path: str, videos_dir: Path) -> Path:
if storage_type in ("radarr", "sonarr", "scan"): if storage_type in ("radarr", "sonarr", "scan"):
@@ -19,29 +25,40 @@ def _file_path(storage_type: str, storage_path: str, videos_dir: Path) -> Path:
def _scan_filesystem(movies: List[Dict], episodes: List[Dict], tracks: List[Dict], def _scan_filesystem(movies: List[Dict], episodes: List[Dict], tracks: List[Dict],
videos_dir: Path, hls_dir: Path, live_ids: set) -> Dict[str, List[Dict]]: videos_dir: Path, hls_dir: Path, live_ids: set) -> Dict[str, List[Dict]]:
"""Every filesystem stat call in the whole scan lives here, run off the event loop via """Every filesystem stat call in the whole scan lives here, run off the event loop via
asyncio.to_thread — movie/episode/track paths are often on an NFS/CIFS-mounted NAS, and asyncio.to_thread, with the individual stat calls themselves parallelized across a small
doing hundreds of blocking stats directly in an async function would freeze the entire worker pool — sequential one-at-a-time NAS round trips were taking minutes for a library
server (every other request) for as long as the scan takes.""" of any real size."""
out: Dict[str, List[Dict]] = {"missing_files": [], "orphaned_hls": []} out: Dict[str, List[Dict]] = {"missing_files": [], "orphaned_hls": []}
candidates = [] # (finding_dict, path_to_check)
for m in movies: for m in movies:
if m.get("storage_type") not in ("local", "radarr", "scan") or not m.get("storage_path"): if m.get("storage_type") not in ("local", "radarr", "scan") or not m.get("storage_path"):
continue continue
if not _file_path(m["storage_type"], m["storage_path"], videos_dir).is_file(): candidates.append((
out["missing_files"].append({"id": m["id"], "content_type": "movie", "label": m.get("title", "Untitled"), "detail": m["storage_path"]}) {"id": m["id"], "content_type": "movie", "label": m.get("title", "Untitled"), "detail": m["storage_path"]},
_file_path(m["storage_type"], m["storage_path"], videos_dir),
))
for e in episodes: for e in episodes:
if e.get("_orphaned"): if e.get("_orphaned"):
continue # already reported as orphaned_episodes; don't also report a missing file continue # already reported as orphaned_episodes; don't also report a missing file
if e.get("storage_type") not in ("local", "sonarr", "scan") or not e.get("storage_path"): if e.get("storage_type") not in ("local", "sonarr", "scan") or not e.get("storage_path"):
continue continue
label = f"S{e.get('season_number')}E{e.get('episode_number')} {e.get('title') or ''}".strip() label = f"S{e.get('season_number')}E{e.get('episode_number')} {e.get('title') or ''}".strip()
if not _file_path(e["storage_type"], e["storage_path"], videos_dir).is_file(): candidates.append((
out["missing_files"].append({"id": e["id"], "content_type": "episode", "label": label, "detail": e["storage_path"]}) {"id": e["id"], "content_type": "episode", "label": label, "detail": e["storage_path"]},
_file_path(e["storage_type"], e["storage_path"], videos_dir),
))
for t in tracks: for t in tracks:
if not Path(t["storage_path"]).is_file(): candidates.append((
out["missing_files"].append({"id": t["id"], "content_type": "track", "label": f"{t.get('artist','')}{t.get('title','Untitled')}", "detail": t["storage_path"]}) {"id": t["id"], "content_type": "track", "label": f"{t.get('artist','')}{t.get('title','Untitled')}", "detail": t["storage_path"]},
Path(t["storage_path"]),
))
with ThreadPoolExecutor(max_workers=_STAT_WORKERS) as pool:
exists = pool.map(lambda pair: pair[1].is_file(), candidates)
for (finding, _path), is_present in zip(candidates, exists):
if not is_present:
out["missing_files"].append(finding)
if hls_dir.is_dir(): if hls_dir.is_dir():
for d in hls_dir.iterdir(): for d in hls_dir.iterdir():
-1
View File
@@ -35,7 +35,6 @@ export const Navbar = () => {
<NavLink to="/music" className={linkClass} data-testid="nav-music">Music</NavLink> <NavLink to="/music" className={linkClass} data-testid="nav-music">Music</NavLink>
<NavLink to="/requests" className={linkClass} data-testid="nav-requests">Wishlist</NavLink> <NavLink to="/requests" className={linkClass} data-testid="nav-requests">Wishlist</NavLink>
{user.is_admin && <NavLink to="/admin" className={linkClass} data-testid="nav-admin">Admin</NavLink>} {user.is_admin && <NavLink to="/admin" className={linkClass} data-testid="nav-admin">Admin</NavLink>}
{user.is_admin && <NavLink to="/admin/users" className={linkClass} data-testid="nav-users">Users</NavLink>}
</nav> </nav>
)} )}
</div> </div>
-3
View File
@@ -94,9 +94,6 @@ export default function Admin() {
<Link to="/admin/queue" className="flex items-center gap-2 bg-white/10 hover:bg-white/20 text-white px-5 py-2 text-xs uppercase tracking-[0.2em] border border-white/10" data-testid="admin-queue-link"> <Link to="/admin/queue" className="flex items-center gap-2 bg-white/10 hover:bg-white/20 text-white px-5 py-2 text-xs uppercase tracking-[0.2em] border border-white/10" data-testid="admin-queue-link">
<SettingsIcon size={14} strokeWidth={1.5} /> Queue <SettingsIcon size={14} strokeWidth={1.5} /> Queue
</Link> </Link>
<Link to="/settings" className="flex items-center gap-2 bg-white/10 hover:bg-white/20 text-white px-5 py-2 text-xs uppercase tracking-[0.2em] border border-white/10" data-testid="admin-settings-link">
<SettingsIcon size={14} strokeWidth={1.5} /> Settings
</Link>
<Link to="/admin/users" className="flex items-center gap-2 bg-white/10 hover:bg-white/20 text-white px-5 py-2 text-xs uppercase tracking-[0.2em] border border-white/10" data-testid="admin-users-link"> <Link to="/admin/users" className="flex items-center gap-2 bg-white/10 hover:bg-white/20 text-white px-5 py-2 text-xs uppercase tracking-[0.2em] border border-white/10" data-testid="admin-users-link">
<SettingsIcon size={14} strokeWidth={1.5} /> Users <SettingsIcon size={14} strokeWidth={1.5} /> Users
</Link> </Link>