mirror of
https://github.com/myronblair/kino-app
synced 2026-07-27 19:24:45 -05:00
Add Library Cleanup admin page — missing files, orphaned records, stuck jobs, duplicates
New read-only scan (GET /api/library/cleanup/scan) across movies, episodes, tracks, subtitles, HLS output, transcode queue, and watchlist/progress rows: - Missing files: DB record exists but the file is gone from disk - Orphaned episodes: show_id no longer exists - Orphaned HLS output: transcoded folders left behind by deleted content - Stuck transcode jobs: "running" 2+ hours with no completion (backend-restart artifact) - Duplicate imports: same file imported into the library more than once - Orphaned subtitles / dangling watchlist+progress rows Nothing is deleted until an admin clicks Fix (per-item or Fix All per category) via POST /api/library/cleanup/fix.
This commit is contained in:
Myron Blair
@@ -0,0 +1,152 @@
|
||||
"""Library integrity scan — missing files, orphaned records/disk usage, stuck jobs, duplicates.
|
||||
Read-only by design: scan() only reports findings; server.py's /library/cleanup/fix endpoint
|
||||
performs the actual deletion/reset the admin picked, one category at a time.
|
||||
"""
|
||||
import shutil
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Dict, List
|
||||
|
||||
|
||||
def _file_path(storage_type: str, storage_path: str, videos_dir: Path) -> Path:
|
||||
if storage_type in ("radarr", "sonarr", "scan"):
|
||||
return Path(storage_path)
|
||||
return videos_dir / storage_path
|
||||
|
||||
|
||||
async def scan(db, videos_dir: Path, hls_dir: Path) -> Dict[str, List[Dict]]:
|
||||
findings: Dict[str, List[Dict]] = {
|
||||
"missing_files": [], "orphaned_episodes": [], "orphaned_hls": [],
|
||||
"stuck_jobs": [], "duplicate_paths": [], "orphaned_subtitles": [], "dangling_refs": [],
|
||||
}
|
||||
|
||||
# --- Missing files ---
|
||||
movies = await db.movies.find({}, {"_id": 0, "id": 1, "title": 1, "storage_type": 1, "storage_path": 1}).to_list(10000)
|
||||
for m in movies:
|
||||
if m.get("storage_type") not in ("local", "radarr", "scan") or not m.get("storage_path"):
|
||||
continue
|
||||
if not _file_path(m["storage_type"], m["storage_path"], videos_dir).is_file():
|
||||
findings["missing_files"].append({"id": m["id"], "content_type": "movie", "label": m.get("title", "Untitled"), "detail": m["storage_path"]})
|
||||
|
||||
episodes = await db.episodes.find({}, {"_id": 0, "id": 1, "title": 1, "show_id": 1, "season_number": 1, "episode_number": 1, "storage_type": 1, "storage_path": 1}).to_list(20000)
|
||||
show_ids = set(await db.shows.distinct("id"))
|
||||
for e in episodes:
|
||||
label = f"S{e.get('season_number')}E{e.get('episode_number')} {e.get('title') or ''}".strip()
|
||||
if e.get("show_id") not in show_ids:
|
||||
findings["orphaned_episodes"].append({"id": e["id"], "content_type": "episode", "label": label, "detail": f"show_id {e.get('show_id')} not found"})
|
||||
continue
|
||||
if e.get("storage_type") not in ("local", "sonarr", "scan") or not e.get("storage_path"):
|
||||
continue
|
||||
if not _file_path(e["storage_type"], e["storage_path"], videos_dir).is_file():
|
||||
findings["missing_files"].append({"id": e["id"], "content_type": "episode", "label": label, "detail": e["storage_path"]})
|
||||
|
||||
tracks = await db.tracks.find({}, {"_id": 0, "id": 1, "title": 1, "artist": 1, "storage_path": 1}).to_list(20000)
|
||||
for t in tracks:
|
||||
if not Path(t["storage_path"]).is_file():
|
||||
findings["missing_files"].append({"id": t["id"], "content_type": "track", "label": f"{t.get('artist','')} — {t.get('title','Untitled')}", "detail": t["storage_path"]})
|
||||
|
||||
# --- Orphaned HLS output dirs (no matching movie or episode id) ---
|
||||
live_ids = {m["id"] for m in movies} | {e["id"] for e in episodes}
|
||||
if hls_dir.is_dir():
|
||||
for d in hls_dir.iterdir():
|
||||
if d.is_dir() and d.name not in live_ids:
|
||||
size = sum(f.stat().st_size for f in d.rglob("*") if f.is_file())
|
||||
findings["orphaned_hls"].append({"id": d.name, "content_type": "hls_dir", "label": d.name, "detail": f"{round(size / 1_000_000)} MB reclaimable"})
|
||||
|
||||
# --- Stuck transcode jobs (running with no progress for 2+ hours — likely dead from a restart) ---
|
||||
now = datetime.now(timezone.utc)
|
||||
running = await db.transcode_queue.find({"status": "running"}, {"_id": 0}).to_list(500)
|
||||
for j in running:
|
||||
started = j.get("started_at")
|
||||
if not started:
|
||||
continue
|
||||
try:
|
||||
age_hours = (now - datetime.fromisoformat(started)).total_seconds() / 3600
|
||||
except ValueError:
|
||||
continue
|
||||
if age_hours > 2:
|
||||
findings["stuck_jobs"].append({"id": j["id"], "content_type": "job", "label": j.get("movie_title") or j["movie_id"], "detail": f"running {round(age_hours, 1)}h with no completion"})
|
||||
|
||||
# --- Duplicate storage_path within the same collection (same file imported twice) ---
|
||||
for coll_name, coll, label_fn in (
|
||||
("movies", db.movies, lambda d: d.get("title", "Untitled")),
|
||||
("episodes", db.episodes, lambda d: d.get("title") or d.get("id")),
|
||||
("tracks", db.tracks, lambda d: d.get("title", "Untitled")),
|
||||
):
|
||||
docs = await coll.find({"storage_path": {"$ne": None}}, {"_id": 0}).sort("created_at", 1).to_list(20000)
|
||||
seen = {}
|
||||
for d in docs:
|
||||
key = d.get("storage_path")
|
||||
if not key:
|
||||
continue
|
||||
if key in seen:
|
||||
findings["duplicate_paths"].append({"id": d["id"], "content_type": coll_name[:-1], "label": label_fn(d), "detail": f"duplicate of {seen[key]}"})
|
||||
else:
|
||||
seen[key] = d["id"]
|
||||
|
||||
# --- Orphaned subtitles (reference a movie/episode that no longer exists) ---
|
||||
movie_ids = {m["id"] for m in movies}
|
||||
episode_ids = {e["id"] for e in episodes}
|
||||
subs = await db.subtitles.find({}, {"_id": 0}).to_list(5000)
|
||||
for s in subs:
|
||||
mid, eid = s.get("movie_id") or "", s.get("episode_id") or ""
|
||||
if mid and mid not in movie_ids:
|
||||
findings["orphaned_subtitles"].append({"id": s["id"], "content_type": "subtitle", "label": s.get("label", "Subtitle"), "detail": f"movie_id {mid} not found"})
|
||||
elif eid and eid not in episode_ids:
|
||||
findings["orphaned_subtitles"].append({"id": s["id"], "content_type": "subtitle", "label": s.get("label", "Subtitle"), "detail": f"episode_id {eid} not found"})
|
||||
|
||||
# --- Dangling watchlist/progress rows (reference a deleted movie) ---
|
||||
for coll_name, coll in (("watchlist", db.watchlist), ("progress", db.progress)):
|
||||
rows = await coll.find({}, {"_id": 0}).to_list(20000)
|
||||
for r in rows:
|
||||
if r.get("movie_id") and r["movie_id"] not in movie_ids:
|
||||
findings["dangling_refs"].append({"id": r["id"] if "id" in r else f"{r['profile_id']}:{r['movie_id']}", "content_type": coll_name, "label": r["movie_id"], "detail": f"references deleted movie {r['movie_id']}", "profile_id": r.get("profile_id"), "movie_id": r.get("movie_id")})
|
||||
|
||||
return findings
|
||||
|
||||
|
||||
async def fix_one(db, videos_dir: Path, hls_dir: Path, category: str, item: Dict) -> None:
|
||||
content_type = item.get("content_type")
|
||||
item_id = item.get("id")
|
||||
|
||||
if category == "missing_files":
|
||||
coll = {"movie": db.movies, "episode": db.episodes, "track": db.tracks}[content_type]
|
||||
await coll.delete_one({"id": item_id})
|
||||
if content_type == "movie":
|
||||
await db.watchlist.delete_many({"movie_id": item_id})
|
||||
await db.progress.delete_many({"movie_id": item_id})
|
||||
await db.subtitles.delete_many({"movie_id": item_id})
|
||||
|
||||
elif category == "orphaned_episodes":
|
||||
await db.episodes.delete_one({"id": item_id})
|
||||
await db.episode_progress.delete_many({"episode_id": item_id})
|
||||
await db.subtitles.delete_many({"episode_id": item_id})
|
||||
|
||||
elif category == "orphaned_hls":
|
||||
shutil.rmtree(hls_dir / item_id, ignore_errors=True)
|
||||
|
||||
elif category == "stuck_jobs":
|
||||
await db.transcode_queue.update_one({"id": item_id}, {"$set": {
|
||||
"status": "failed", "error": "Reset by library cleanup (stuck with no progress)",
|
||||
"finished_at": datetime.now(timezone.utc).isoformat(),
|
||||
}})
|
||||
|
||||
elif category == "duplicate_paths":
|
||||
coll = {"movie": db.movies, "episode": db.episodes, "track": db.tracks}[content_type]
|
||||
await coll.delete_one({"id": item_id})
|
||||
|
||||
elif category == "orphaned_subtitles":
|
||||
sub = await db.subtitles.find_one({"id": item_id}, {"_id": 0})
|
||||
if sub and sub.get("storage_path"):
|
||||
try:
|
||||
(Path(str(videos_dir.parent / "subtitles")) / sub["storage_path"]).unlink(missing_ok=True)
|
||||
except OSError:
|
||||
pass
|
||||
await db.subtitles.delete_one({"id": item_id})
|
||||
|
||||
elif category == "dangling_refs":
|
||||
coll = {"watchlist": db.watchlist, "progress": db.progress}[content_type]
|
||||
await coll.delete_one({"profile_id": item.get("profile_id"), "movie_id": item.get("movie_id")})
|
||||
|
||||
else:
|
||||
raise ValueError(f"Unknown cleanup category: {category}")
|
||||
@@ -36,6 +36,7 @@ import radarr as radarr_client
|
||||
import sonarr as sonarr_client
|
||||
import services as services_client
|
||||
import library_scan
|
||||
import library_cleanup
|
||||
import musicbrainz as musicbrainz_client
|
||||
import trakt as trakt_client
|
||||
import opensubtitles as opensubs_client
|
||||
@@ -1510,6 +1511,29 @@ async def scan_import(payload: dict, user: dict = Depends(require_admin)):
|
||||
return {"ok": True, "imported": created}
|
||||
|
||||
|
||||
# ============ LIBRARY CLEANUP (admin) ============
|
||||
@api.get("/library/cleanup/scan")
|
||||
async def library_cleanup_scan(user: dict = Depends(require_admin)):
|
||||
return await library_cleanup.scan(db, VIDEOS_DIR, HLS_DIR)
|
||||
|
||||
|
||||
@api.post("/library/cleanup/fix")
|
||||
async def library_cleanup_fix(payload: dict, user: dict = Depends(require_admin)):
|
||||
"""Body: {category: str, items: [<finding dicts as returned by GET /library/cleanup/scan>]}"""
|
||||
category = payload.get("category")
|
||||
items = payload.get("items") or []
|
||||
if not items:
|
||||
raise HTTPException(status_code=400, detail="No items provided")
|
||||
fixed = 0
|
||||
for item in items:
|
||||
try:
|
||||
await library_cleanup.fix_one(db, VIDEOS_DIR, HLS_DIR, category, item)
|
||||
fixed += 1
|
||||
except Exception as e:
|
||||
logger.warning(f"Library cleanup fix failed for {category}/{item.get('id')}: {e}")
|
||||
return {"ok": True, "fixed": fixed}
|
||||
|
||||
|
||||
# ============ MUSIC ============
|
||||
@api.get("/tracks", response_model=List[Track])
|
||||
async def list_tracks(user: dict = Depends(get_current_user)):
|
||||
|
||||
@@ -28,6 +28,7 @@ import Users from "./pages/Users";
|
||||
import Music from "./pages/Music";
|
||||
import AlbumDetail from "./pages/AlbumDetail";
|
||||
import LibraryScan from "./pages/LibraryScan";
|
||||
import LibraryCleanup from "./pages/LibraryCleanup";
|
||||
|
||||
const ProfileGate = ({ children }) => {
|
||||
const { user, loading: authLoading } = useAuth();
|
||||
@@ -101,6 +102,7 @@ function App() {
|
||||
<Route path="/admin/radarr" element={<AdminGate><RadarrImport /></AdminGate>} />
|
||||
<Route path="/admin/sonarr" element={<AdminGate><SonarrImport /></AdminGate>} />
|
||||
<Route path="/admin/scan" element={<AdminGate><LibraryScan /></AdminGate>} />
|
||||
<Route path="/admin/cleanup" element={<AdminGate><LibraryCleanup /></AdminGate>} />
|
||||
<Route path="/admin/shows/:id/episodes" element={<AdminGate><AdminShowEpisodes /></AdminGate>} />
|
||||
<Route path="/admin/queue" element={<AdminGate><TranscodeQueue /></AdminGate>} />
|
||||
<Route path="*" element={<Navigate to="/" replace />} />
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { useEffect, useState } from "react";
|
||||
import api from "../lib/api";
|
||||
import { toast } from "sonner";
|
||||
import { Trash2, Star, Film, Settings as SettingsIcon, Download, RefreshCcw, FolderSearch } from "lucide-react";
|
||||
import { Trash2, Star, Film, Settings as SettingsIcon, Download, RefreshCcw, FolderSearch, Wrench } from "lucide-react";
|
||||
import { useNavigate, Link } from "react-router-dom";
|
||||
|
||||
export default function Admin() {
|
||||
@@ -88,6 +88,9 @@ export default function Admin() {
|
||||
<Link to="/admin/scan" className="flex items-center gap-2 bg-white/10 hover:bg-white/20 text-white px-5 py-2 text-xs uppercase tracking-[0.2em] border border-white/10" data-testid="admin-scan-link">
|
||||
<FolderSearch size={14} strokeWidth={1.5} /> Library Scan
|
||||
</Link>
|
||||
<Link to="/admin/cleanup" className="flex items-center gap-2 bg-white/10 hover:bg-white/20 text-white px-5 py-2 text-xs uppercase tracking-[0.2em] border border-white/10" data-testid="admin-cleanup-link">
|
||||
<Wrench size={14} strokeWidth={1.5} /> Library Cleanup
|
||||
</Link>
|
||||
<Link to="/admin/queue" className="flex items-center gap-2 bg-white/10 hover:bg-white/20 text-white px-5 py-2 text-xs uppercase tracking-[0.2em] border border-white/10" data-testid="admin-queue-link">
|
||||
<SettingsIcon size={14} strokeWidth={1.5} /> Queue
|
||||
</Link>
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
import { useEffect, useState } from "react";
|
||||
import api from "../lib/api";
|
||||
import { toast } from "sonner";
|
||||
import { RefreshCcw, Trash2, Wrench, ShieldCheck } from "lucide-react";
|
||||
|
||||
const CATEGORIES = [
|
||||
{ key: "missing_files", label: "Missing Files", desc: "Database record exists but the file is gone from disk.", action: "Remove record" },
|
||||
{ key: "orphaned_episodes", label: "Orphaned Episodes", desc: "Episode references a show that no longer exists.", action: "Delete episode" },
|
||||
{ key: "orphaned_hls", label: "Orphaned HLS Output", desc: "Transcoded HLS folders left behind by deleted movies/episodes.", action: "Delete folder" },
|
||||
{ key: "stuck_jobs", label: "Stuck Transcode Jobs", desc: "Jobs stuck \"running\" for 2+ hours, usually from a backend restart mid-job.", action: "Mark failed" },
|
||||
{ key: "duplicate_paths", label: "Duplicate Imports", desc: "The same file imported into the library more than once.", action: "Delete duplicate" },
|
||||
{ key: "orphaned_subtitles", label: "Orphaned Subtitles", desc: "Subtitle references a movie/episode that no longer exists.", action: "Delete subtitle" },
|
||||
{ key: "dangling_refs", label: "Dangling Watchlist/Progress", desc: "Watchlist or progress rows pointing at a deleted movie.", action: "Delete row" },
|
||||
];
|
||||
|
||||
export default function LibraryCleanup() {
|
||||
const [findings, setFindings] = useState(null);
|
||||
const [loading, setLoading] = useState(false);
|
||||
const [fixing, setFixing] = useState({});
|
||||
|
||||
const scan = async () => {
|
||||
setLoading(true);
|
||||
try {
|
||||
const { data } = await api.get("/library/cleanup/scan");
|
||||
setFindings(data);
|
||||
} catch (err) {
|
||||
toast.error(err.response?.data?.detail || "Cleanup scan failed");
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
};
|
||||
useEffect(() => { scan(); }, []);
|
||||
|
||||
const fix = async (category, items) => {
|
||||
if (items.length === 0) return;
|
||||
setFixing((p) => ({ ...p, [category]: true }));
|
||||
try {
|
||||
const { data } = await api.post("/library/cleanup/fix", { category, items });
|
||||
toast.success(`Fixed ${data.fixed} item(s)`);
|
||||
scan();
|
||||
} catch (err) {
|
||||
toast.error(err.response?.data?.detail || "Fix failed");
|
||||
} finally {
|
||||
setFixing((p) => ({ ...p, [category]: false }));
|
||||
}
|
||||
};
|
||||
|
||||
const totalIssues = findings ? Object.values(findings).reduce((n, arr) => n + arr.length, 0) : 0;
|
||||
|
||||
return (
|
||||
<div className="min-h-screen bg-[#050505] pt-32 pb-24" data-testid="library-cleanup-page">
|
||||
<div className="px-6 md:px-12 max-w-[1500px] mx-auto">
|
||||
<span className="text-xs uppercase tracking-[0.3em] text-[#D9381E]">Maintenance</span>
|
||||
<h1 className="font-display text-5xl md:text-6xl font-black tracking-tighter text-white mt-3">Library Cleanup</h1>
|
||||
<p className="text-[#8A8A8A] mt-4 max-w-2xl">
|
||||
Checks for missing files, orphaned records, stuck jobs, and duplicate imports across movies, shows, and music. Nothing is deleted until you click Fix.
|
||||
</p>
|
||||
|
||||
<div className="mt-8 flex items-center gap-4">
|
||||
<button onClick={scan} disabled={loading} className="flex items-center gap-2 border border-[#222] hover:border-white text-[#8A8A8A] hover:text-white px-5 py-2 text-xs uppercase tracking-[0.2em] transition-colors disabled:opacity-50" data-testid="cleanup-scan">
|
||||
<RefreshCcw size={14} strokeWidth={1.5} className={loading ? "animate-spin" : ""} /> {loading ? "Scanning…" : "Rescan"}
|
||||
</button>
|
||||
{findings && (
|
||||
<span className="flex items-center gap-2 text-sm" data-testid="cleanup-summary">
|
||||
{totalIssues === 0 ? (
|
||||
<><ShieldCheck size={16} className="text-[#86efac]" /> <span className="text-[#86efac]">No issues found</span></>
|
||||
) : (
|
||||
<span className="text-[#fcd34d]">{totalIssues} issue(s) found</span>
|
||||
)}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="mt-10 space-y-8">
|
||||
{CATEGORIES.map((cat) => {
|
||||
const items = findings?.[cat.key] || [];
|
||||
if (findings && items.length === 0) return null;
|
||||
return (
|
||||
<div key={cat.key} className="border border-[#222]" data-testid={`cleanup-section-${cat.key}`}>
|
||||
<div className="flex items-center justify-between px-5 py-4 border-b border-[#222] bg-[#0F0F0F]">
|
||||
<div>
|
||||
<h2 className="text-white text-lg font-display">{cat.label} <span className="text-[#8A8A8A] text-sm">({items.length})</span></h2>
|
||||
<p className="text-xs text-[#8A8A8A] mt-1">{cat.desc}</p>
|
||||
</div>
|
||||
<button onClick={() => fix(cat.key, items)} disabled={fixing[cat.key] || items.length === 0}
|
||||
className="flex items-center gap-2 bg-[#D9381E] hover:bg-[#ED4B32] disabled:opacity-50 text-white px-4 py-2 text-xs uppercase tracking-[0.2em] shrink-0"
|
||||
data-testid={`cleanup-fix-all-${cat.key}`}>
|
||||
<Wrench size={13} strokeWidth={1.5} /> {cat.action} All
|
||||
</button>
|
||||
</div>
|
||||
<div>
|
||||
{items.map((it) => (
|
||||
<div key={`${it.content_type}-${it.id}`} className="flex items-center justify-between gap-4 px-5 py-3 border-b border-[#222] last:border-b-0" data-testid={`cleanup-item-${it.id}`}>
|
||||
<div className="min-w-0">
|
||||
<p className="text-white text-sm truncate">{it.label}</p>
|
||||
<p className="text-xs text-[#666] truncate">{it.detail}</p>
|
||||
</div>
|
||||
<button onClick={() => fix(cat.key, [it])} disabled={fixing[cat.key]}
|
||||
className="flex items-center gap-1 text-[10px] uppercase tracking-[0.2em] border border-[#222] hover:border-[#fca5a5] hover:text-[#fca5a5] text-[#8A8A8A] px-3 py-2 shrink-0 disabled:opacity-50"
|
||||
data-testid={`cleanup-fix-${it.id}`}>
|
||||
<Trash2 size={12} strokeWidth={1.5} /> {cat.action}
|
||||
</button>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
{findings && totalIssues === 0 && (
|
||||
<div className="text-center py-16 text-[#8A8A8A]">
|
||||
<ShieldCheck size={40} strokeWidth={1.5} className="mx-auto mb-4 text-[#86efac]" />
|
||||
Library is clean — no missing files, orphaned records, or stuck jobs.
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user