feat: automate exact dedupe and improve web feedback
This commit is contained in:
@@ -43,7 +43,7 @@ am-deduper dedupe /music/AllMusic --db /data/dedupe.db --dry-run
|
||||
# 确认后执行
|
||||
am-deduper dedupe /music/AllMusic --db /data/dedupe.db
|
||||
|
||||
# 长驻监听:默认只自动执行安全的 Exact hard-link
|
||||
# 长驻监听:启动时先做一次全量索引 + Exact 去重,之后自动执行安全的 Exact hard-link
|
||||
am-deduper watch /music/AllMusic --db /data/dedupe.db
|
||||
|
||||
# 开启 L2 acoustic 匹配;建议先 dry-run 验证
|
||||
@@ -123,7 +123,11 @@ http://<NAS-IP>:18080/
|
||||
- 待审核:查看相似度、候选文件,可人工确认或保留。
|
||||
- 任务:查看 scan / dry-run / dedupe 状态。
|
||||
- 历史:查看 Hard Link 操作,并对仍保持同 inode 的操作执行安全回滚。
|
||||
- 监听服务启动时会先做一次全量扫描与 Exact 去重;新文件稳定后也会自动索引并执行 Exact 去重。
|
||||
- 跨 Web 与 watcher 进程使用数据库锁,避免两个容器同时执行 Hard Link。
|
||||
- 文件被替换时会自动清空旧 SHA-256 / Chromaprint,防止旧内容哈希污染新文件。
|
||||
- 扫描会把数据库中已经不存在的文件标记为 `missing`,避免脏索引继续计入统计。
|
||||
- Web UI 的每个操作都有明确的启动、运行、完成/失败反馈,并保留任务结果。
|
||||
|
||||
> Web UI 支持可选管理 Token。生产环境建议在 NAS 的 Compose 同目录 `.env` 中设置 `AM_DEDUPER_TOKEN=<随机长字符串>`;不要把 Token 提交到 Git。设置后 Web UI 会在首次访问 API 时要求输入 Token。若未设置 Token,则仅适合可信内网环境,不要直接暴露到公网。
|
||||
|
||||
|
||||
+40
-41
@@ -5,6 +5,7 @@ from pathlib import Path
|
||||
|
||||
from .db import connect
|
||||
from .hashing import sha256
|
||||
from .locking import exclusive
|
||||
|
||||
|
||||
def confirm_sha256(conn, row) -> str | None:
|
||||
@@ -59,50 +60,48 @@ def choose_winner(rows):
|
||||
def exact_dedupe(db_path: Path, root: Path, dry_run: bool = False) -> list[tuple[str, str]]:
|
||||
actions: list[tuple[str, str]] = []
|
||||
root = root.resolve()
|
||||
with connect(db_path) as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM audio_tracks WHERE status IN ('active','hardlinked')"
|
||||
).fetchall()
|
||||
groups: dict[tuple[int, str], list] = {}
|
||||
for row in rows:
|
||||
groups.setdefault((row["file_size"], row["fast_hash"]), []).append(row)
|
||||
|
||||
for candidates in groups.values():
|
||||
if len(candidates) < 2:
|
||||
continue
|
||||
for row in candidates:
|
||||
if not row["sha256"]:
|
||||
confirm_sha256(conn, row)
|
||||
ids = [row["id"] for row in candidates]
|
||||
placeholders = ",".join("?" * len(ids))
|
||||
refreshed = conn.execute(
|
||||
f"SELECT * FROM audio_tracks WHERE id IN ({placeholders})", ids
|
||||
with exclusive(db_path):
|
||||
with connect(db_path) as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM audio_tracks WHERE status IN ('active','hardlinked')"
|
||||
).fetchall()
|
||||
by_sha: dict[str, list] = {}
|
||||
for row in refreshed:
|
||||
if row["sha256"]:
|
||||
by_sha.setdefault(row["sha256"], []).append(row)
|
||||
groups: dict[tuple[int, str], list] = {}
|
||||
for row in rows:
|
||||
groups.setdefault((row["file_size"], row["fast_hash"]), []).append(row)
|
||||
|
||||
for same in by_sha.values():
|
||||
if len(same) < 2:
|
||||
for candidates in groups.values():
|
||||
if len(candidates) < 2:
|
||||
continue
|
||||
winner = choose_winner(same)
|
||||
wp = Path(winner["file_path"])
|
||||
if not wp.exists() or not wp.is_relative_to(root):
|
||||
continue
|
||||
for row in same:
|
||||
if row["id"] == winner["id"]:
|
||||
for row in candidates:
|
||||
if not row["sha256"]:
|
||||
confirm_sha256(conn, row)
|
||||
ids = [row["id"] for row in candidates]
|
||||
placeholders = ",".join("?" * len(ids))
|
||||
refreshed = conn.execute(
|
||||
f"SELECT * FROM audio_tracks WHERE id IN ({placeholders})", ids
|
||||
).fetchall()
|
||||
by_sha: dict[str, list] = {}
|
||||
for row in refreshed:
|
||||
if row["sha256"]:
|
||||
by_sha.setdefault(row["sha256"], []).append(row)
|
||||
|
||||
for same in by_sha.values():
|
||||
if len(same) < 2:
|
||||
continue
|
||||
lp = Path(row["file_path"])
|
||||
if not lp.exists() or not lp.is_relative_to(root):
|
||||
winner = choose_winner(same)
|
||||
wp = Path(winner["file_path"])
|
||||
if not wp.exists() or not wp.is_relative_to(root):
|
||||
continue
|
||||
if lp.stat().st_ino != wp.stat().st_ino:
|
||||
if not dry_run:
|
||||
hardlink_duplicate(lp, wp)
|
||||
conn.execute("INSERT INTO operations(action,file_path,winner_path,details) VALUES(?,?,?,?)", ('exact-hardlink',str(lp),str(wp),'sha256 exact duplicate'))
|
||||
conn.execute(
|
||||
"UPDATE audio_tracks SET status='hardlinked',updated_at=CURRENT_TIMESTAMP WHERE id=?",
|
||||
(row["id"],),
|
||||
)
|
||||
actions.append((str(lp), str(wp)))
|
||||
for row in same:
|
||||
if row["id"] == winner["id"]:
|
||||
continue
|
||||
lp = Path(row["file_path"])
|
||||
if not lp.exists() or not lp.is_relative_to(root):
|
||||
continue
|
||||
if lp.stat().st_ino != wp.stat().st_ino:
|
||||
if not dry_run:
|
||||
hardlink_duplicate(lp, wp)
|
||||
conn.execute("INSERT INTO operations(action,file_path,winner_path,details) VALUES(?,?,?,?)", ('exact-hardlink',str(lp),str(wp),'sha256 exact duplicate'))
|
||||
conn.execute("UPDATE audio_tracks SET status='hardlinked',updated_at=CURRENT_TIMESTAMP WHERE id=?", (row["id"],))
|
||||
actions.append((str(lp), str(wp)))
|
||||
return actions
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
import fcntl
|
||||
|
||||
|
||||
@contextmanager
|
||||
def exclusive(db_path: Path):
|
||||
"""Cross-process lock for destructive dedupe operations."""
|
||||
lock_path = Path(str(db_path) + ".lock")
|
||||
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with lock_path.open("w") as handle:
|
||||
fcntl.flock(handle.fileno(), fcntl.LOCK_EX)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
|
||||
+14
-1
@@ -22,14 +22,27 @@ def index_file(conn, path: Path) -> None:
|
||||
fh = fast_hash(path)
|
||||
meta = read_metadata(path)
|
||||
flags = ",".join(meta.get("version_flags", []))
|
||||
old = conn.execute("SELECT file_size,fast_hash,inode,device FROM audio_tracks WHERE file_path=?", (str(path),)).fetchone()
|
||||
identity_changed = old is not None and (old["file_size"], old["fast_hash"], old["inode"], old["device"]) != (st.st_size, fh, st.st_ino, st.st_dev)
|
||||
conn.execute("""
|
||||
INSERT INTO audio_tracks(file_path,file_size,fast_hash,duration,artist,title,album,format,bitrate,sample_rate,bit_depth,version_flags,inode,device,status)
|
||||
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,'active')
|
||||
ON CONFLICT(file_path) DO UPDATE SET file_size=excluded.file_size, fast_hash=excluded.fast_hash,
|
||||
duration=excluded.duration,artist=excluded.artist,title=excluded.title,album=excluded.album,format=excluded.format,
|
||||
bitrate=excluded.bitrate,sample_rate=excluded.sample_rate,bit_depth=excluded.bit_depth,version_flags=excluded.version_flags,
|
||||
inode=excluded.inode,device=excluded.device,updated_at=CURRENT_TIMESTAMP,status='active'
|
||||
inode=excluded.inode,device=excluded.device,updated_at=CURRENT_TIMESTAMP,status='active',
|
||||
sha256=CASE WHEN excluded.file_size!=audio_tracks.file_size OR excluded.fast_hash!=audio_tracks.fast_hash
|
||||
OR excluded.inode!=audio_tracks.inode OR excluded.device!=audio_tracks.device
|
||||
THEN NULL ELSE audio_tracks.sha256 END,
|
||||
chromaprint=CASE WHEN excluded.file_size!=audio_tracks.file_size OR excluded.fast_hash!=audio_tracks.fast_hash
|
||||
OR excluded.inode!=audio_tracks.inode OR excluded.device!=audio_tracks.device
|
||||
THEN NULL ELSE audio_tracks.chromaprint END,
|
||||
review_status=CASE WHEN excluded.file_size!=audio_tracks.file_size OR excluded.fast_hash!=audio_tracks.fast_hash
|
||||
OR excluded.inode!=audio_tracks.inode OR excluded.device!=audio_tracks.device
|
||||
THEN NULL ELSE audio_tracks.review_status END
|
||||
""", (str(path),st.st_size,fh,meta.get("duration"),meta.get("artist"),meta.get("title"),meta.get("album"),meta.get("format"),meta.get("bitrate"),meta.get("sample_rate"),meta.get("bit_depth"),flags,st.st_ino,st.st_dev))
|
||||
if identity_changed:
|
||||
conn.execute("UPDATE audio_tracks SET updated_at=CURRENT_TIMESTAMP WHERE file_path=?", (str(path),))
|
||||
|
||||
|
||||
def scan(root: Path, db_path: Path) -> int:
|
||||
|
||||
+64
-15
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
@@ -11,7 +12,7 @@ from .constants import MUSIC_EXTENSIONS, STABILITY_CHECKS, STABILITY_INTERVAL_SE
|
||||
from .db import connect
|
||||
from .exact import exact_dedupe
|
||||
from .l2 import acoustic_dedupe
|
||||
from .scanner import index_file
|
||||
from .scanner import index_file, scan
|
||||
|
||||
|
||||
class Handler(FileSystemEventHandler):
|
||||
@@ -20,6 +21,17 @@ class Handler(FileSystemEventHandler):
|
||||
self.db_path = db_path
|
||||
self.l2 = l2
|
||||
self._lock = threading.Lock()
|
||||
self._timers: dict[str, threading.Timer] = {}
|
||||
self._timers_lock = threading.Lock()
|
||||
|
||||
def _record_job(self, kind: str, status: str, job_id: int, result=None, error=None):
|
||||
with connect(self.db_path) as conn:
|
||||
conn.execute("UPDATE jobs SET status=?,finished_at=CURRENT_TIMESTAMP,result=?,error=? WHERE id=?",
|
||||
(status, json.dumps(result, ensure_ascii=False) if result is not None else None, error, job_id))
|
||||
|
||||
def _new_job(self, kind: str) -> int:
|
||||
with connect(self.db_path) as conn:
|
||||
return conn.execute("INSERT INTO jobs(kind,status) VALUES(?,?)", (kind, "running")).lastrowid
|
||||
|
||||
def _process(self, filename: str) -> None:
|
||||
path = Path(filename)
|
||||
@@ -32,40 +44,75 @@ class Handler(FileSystemEventHandler):
|
||||
if ".dedupe" in path.parts or path.name.startswith("."):
|
||||
return
|
||||
|
||||
# watchdog may emit several events while a downloader is still writing.
|
||||
# Serialize the worker so a file is never indexed while another event
|
||||
# is changing the same database or replacing a duplicate.
|
||||
with self._lock:
|
||||
job_id = self._new_job("auto-exact")
|
||||
try:
|
||||
previous = path.stat()
|
||||
stable = 0
|
||||
while stable < STABILITY_CHECKS - 1:
|
||||
time.sleep(STABILITY_INTERVAL_SECONDS)
|
||||
current = path.stat()
|
||||
if (current.st_size, current.st_mtime_ns) == (
|
||||
previous.st_size,
|
||||
previous.st_mtime_ns,
|
||||
):
|
||||
if (current.st_size, current.st_mtime_ns) == (previous.st_size, previous.st_mtime_ns):
|
||||
stable += 1
|
||||
else:
|
||||
previous = current
|
||||
stable = 0
|
||||
with connect(self.db_path) as conn:
|
||||
index_file(conn, path)
|
||||
# Exact byte-for-byte dedupe is always safe. L2 remains opt-in.
|
||||
exact_dedupe(self.db_path, self.root)
|
||||
actions = exact_dedupe(self.db_path, self.root)
|
||||
result = {"indexed": 1, "exact_hardlinked": len(actions), "path": str(path)}
|
||||
if self.l2:
|
||||
acoustic_dedupe(self.db_path, self.root)
|
||||
except (FileNotFoundError, PermissionError, OSError):
|
||||
return
|
||||
result["acoustic"] = acoustic_dedupe(self.db_path, self.root)
|
||||
self._record_job("auto-exact", "success", job_id, result=result)
|
||||
except (FileNotFoundError, PermissionError, OSError) as exc:
|
||||
self._record_job("auto-exact", "skipped", job_id, error=str(exc))
|
||||
except Exception as exc:
|
||||
self._record_job("auto-exact", "failed", job_id, error=str(exc))
|
||||
|
||||
def _schedule(self, filename: str) -> None:
|
||||
with self._timers_lock:
|
||||
old = self._timers.pop(filename, None)
|
||||
if old:
|
||||
old.cancel()
|
||||
timer = threading.Timer(1.0, self._run_scheduled, args=(filename,))
|
||||
timer.daemon = True
|
||||
self._timers[filename] = timer
|
||||
timer.start()
|
||||
|
||||
def _run_scheduled(self, filename: str) -> None:
|
||||
with self._timers_lock:
|
||||
self._timers.pop(filename, None)
|
||||
self._process(filename)
|
||||
|
||||
def on_created(self, event):
|
||||
if not event.is_directory:
|
||||
self._process(event.src_path)
|
||||
self._schedule(event.src_path)
|
||||
|
||||
def on_modified(self, event):
|
||||
if not event.is_directory:
|
||||
self._schedule(event.src_path)
|
||||
|
||||
def on_moved(self, event):
|
||||
if not event.is_directory:
|
||||
self._process(event.dest_path)
|
||||
self._schedule(event.dest_path)
|
||||
|
||||
|
||||
def _initial_reconcile(root: Path, db_path: Path, l2: bool) -> None:
|
||||
with connect(db_path) as conn:
|
||||
job_id = conn.execute("INSERT INTO jobs(kind,status) VALUES(?,?)", ("startup-reconcile", "running")).lastrowid
|
||||
try:
|
||||
indexed = scan(root, db_path)
|
||||
actions = exact_dedupe(db_path, root)
|
||||
result = {"indexed": indexed, "exact_hardlinked": len(actions)}
|
||||
if l2:
|
||||
result["acoustic"] = acoustic_dedupe(db_path, root)
|
||||
with connect(db_path) as conn:
|
||||
conn.execute("UPDATE jobs SET status='success',finished_at=CURRENT_TIMESTAMP,result=? WHERE id=?",
|
||||
(json.dumps(result, ensure_ascii=False), job_id))
|
||||
except Exception as exc:
|
||||
with connect(db_path) as conn:
|
||||
conn.execute("UPDATE jobs SET status='failed',finished_at=CURRENT_TIMESTAMP,error=? WHERE id=?", (str(exc), job_id))
|
||||
raise
|
||||
|
||||
|
||||
def watch(root: Path, db_path: Path, l2: bool = False) -> None:
|
||||
@@ -73,9 +120,11 @@ def watch(root: Path, db_path: Path, l2: bool = False) -> None:
|
||||
if not root.is_dir():
|
||||
raise ValueError(f"music root does not exist or is not a directory: {root}")
|
||||
db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
_initial_reconcile(root, db_path, l2)
|
||||
observer = Observer()
|
||||
observer.schedule(Handler(root, db_path, l2=l2), str(root), recursive=True)
|
||||
observer.start()
|
||||
print(f"AM Deduper watcher active: {root}", flush=True)
|
||||
try:
|
||||
while True:
|
||||
time.sleep(1)
|
||||
|
||||
+120
-104
@@ -1,191 +1,207 @@
|
||||
from __future__ import annotations
|
||||
import json, os, shutil, threading, uuid
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import threading
|
||||
import uuid
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from .db import connect
|
||||
from .scanner import scan
|
||||
from .exact import exact_dedupe, hardlink_duplicate, choose_winner
|
||||
from .l2 import acoustic_dedupe
|
||||
|
||||
ROOT=Path(os.environ.get("AM_DEDUPER_ROOT","/music/AllMusic")).resolve()
|
||||
DB=Path(os.environ.get("AM_DEDUPER_DB","/data/dedupe.db")).resolve()
|
||||
TOKEN=os.environ.get("AM_DEDUPER_TOKEN","").strip()
|
||||
LOCK=threading.Lock()
|
||||
ROOT = Path(os.environ.get("AM_DEDUPER_ROOT", "/music/AllMusic")).resolve()
|
||||
DB = Path(os.environ.get("AM_DEDUPER_DB", "/data/dedupe.db")).resolve()
|
||||
TOKEN = os.environ.get("AM_DEDUPER_TOKEN", "").strip()
|
||||
LOCK = threading.Lock()
|
||||
|
||||
HTML=r'''<!doctype html><html lang="zh-CN"><meta charset="utf-8">
|
||||
HTML = r'''<!doctype html><html lang="zh-CN"><meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1"><title>AM Deduper</title>
|
||||
<style>
|
||||
body{font:14px system-ui;margin:0;background:#f5f6f8;color:#202124}header{background:#111827;color:#fff;padding:18px 24px}
|
||||
main{max-width:1280px;margin:18px auto;padding:0 14px}.grid{display:grid;grid-template-columns:repeat(5,1fr);gap:10px}
|
||||
.card,section{background:#fff;border:1px solid #ddd;border-radius:10px;padding:14px;margin-bottom:14px}.num{font-size:24px;font-weight:700}
|
||||
nav{display:flex;gap:8px;flex-wrap:wrap;margin-bottom:14px}button{padding:8px 12px;border:1px solid #bbb;border-radius:7px;background:#fff;cursor:pointer}
|
||||
button.primary{background:#111827;color:#fff}button.danger{background:#fee2e2}table{width:100%;border-collapse:collapse}th,td{padding:9px;border-bottom:1px solid #eee;text-align:left;vertical-align:top}
|
||||
.path{font:12px monospace;word-break:break-all}.muted{color:#666}.badge{background:#eee;border-radius:10px;padding:3px 7px}
|
||||
@media(max-width:800px){.grid{grid-template-columns:repeat(2,1fr)}table{font-size:12px}}
|
||||
:root{font:14px system-ui,-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif;color:#172033;background:#f4f6f8}*{box-sizing:border-box}
|
||||
body{margin:0}header{background:#111827;color:white;padding:18px 24px;position:sticky;top:0;z-index:5;box-shadow:0 2px 10px #0002}
|
||||
header .sub{opacity:.7;font-size:12px;margin-top:4px}.wrap{max-width:1400px;margin:18px auto;padding:0 14px}.actions{display:flex;gap:8px;flex-wrap:wrap;margin-bottom:12px}
|
||||
button{border:1px solid #cbd0d8;border-radius:8px;padding:9px 13px;background:white;color:#172033;cursor:pointer}button:hover{background:#f0f2f5}button:disabled{opacity:.5;cursor:wait}
|
||||
button.primary{background:#111827;color:#fff;border-color:#111827}button.warn{background:#fff7ed;border-color:#fed7aa}button.danger{background:#fef2f2;border-color:#fecaca}
|
||||
#toast{position:fixed;right:18px;bottom:18px;z-index:20;max-width:420px;padding:12px 16px;border-radius:10px;background:#111827;color:#fff;box-shadow:0 6px 24px #0003;display:none}
|
||||
#toast.error{background:#991b1b}.statusbar{background:#fff;border:1px solid #dfe3e8;border-radius:10px;padding:12px 14px;margin-bottom:14px;display:flex;justify-content:space-between;gap:12px;align-items:center}.status-main{font-weight:600}.status-detail{font-size:12px;color:#667085;margin-top:3px}
|
||||
.grid{display:grid;grid-template-columns:repeat(5,1fr);gap:10px}.card,section{background:#fff;border:1px solid #dfe3e8;border-radius:12px;padding:15px;margin-bottom:14px}.card .label{color:#667085}.num{font-size:26px;font-weight:750;margin-top:4px}.hint{font-size:12px;color:#667085;margin-top:3px}
|
||||
h2{font-size:17px;margin:0 0 12px}table{width:100%;border-collapse:collapse}th,td{padding:9px;border-bottom:1px solid #edf0f2;text-align:left;vertical-align:top}th{color:#667085;font-weight:600;font-size:12px}.path{font:12px ui-monospace,SFMono-Regular,Menlo,monospace;word-break:break-all}.muted{color:#667085}.badge{display:inline-block;padding:3px 7px;border-radius:999px;background:#eef2f6;font-size:12px}.success{color:#166534}.error{color:#991b1b}.running{color:#1d4ed8}.empty{padding:22px;text-align:center;color:#667085;background:#fafbfc;border-radius:8px}
|
||||
.jobline{display:flex;align-items:center;gap:8px}.spinner{width:12px;height:12px;border:2px solid #cbd5e1;border-top-color:#2563eb;border-radius:50%;animation:spin .8s linear infinite}@keyframes spin{to{transform:rotate(360deg)}}
|
||||
details{margin-top:8px}summary{cursor:pointer;color:#475467}.toolbar{display:flex;justify-content:space-between;gap:8px;align-items:center;margin-bottom:8px}.small{font-size:12px}
|
||||
@media(max-width:900px){.grid{grid-template-columns:repeat(2,1fr)}.statusbar{align-items:flex-start;flex-direction:column}table{display:block;overflow:auto;white-space:nowrap}}
|
||||
</style>
|
||||
<header><b>AM Deduper</b> · NAS Music Library</header><main>
|
||||
<nav><button class="primary" onclick="run('scan')">扫描索引</button><button onclick="run('dry')">分析 / Dry-run</button>
|
||||
<button onclick="run('dedupe')">执行去重</button><button onclick="load()">刷新</button></nav>
|
||||
<header><div><b>AM Deduper</b> · NAS Music Library</div><div class="sub">自动监听 · SHA-256 精确去重 · Hard Link · 人工审核</div></header>
|
||||
<div class="wrap">
|
||||
<div class="actions"><button class="primary" data-action="scan" onclick="run('scan')">扫描并更新索引</button><button data-action="dry" onclick="run('dry')">分析重复(Dry-run)</button><button class="warn" data-action="dedupe" onclick="run('dedupe')">执行安全去重</button><button onclick="load(true)">刷新</button></div>
|
||||
<div id="status" class="statusbar"><div><div class="status-main">正在加载…</div><div class="status-detail">读取任务和索引状态</div></div><span class="badge">自动刷新 5 秒</span></div>
|
||||
<div class="grid" id="stats"></div>
|
||||
<section><h2>重复组</h2><div id="dupes"></div></section>
|
||||
<section><h2>待人工审核</h2><div id="review"></div></section>
|
||||
<section><h2>任务</h2><div id="jobsBox"></div></section>
|
||||
<section><h2>最近操作</h2><div id="historyBox"></div></section>
|
||||
<section><h2>设置</h2><div id="settingsBox" class="muted">加载中...</div></section>
|
||||
</main>
|
||||
|
||||
<section><div class="toolbar"><h2>重复组</h2><span class="small muted" id="dupeCount"></span></div><div id="dupes"></div></section>
|
||||
<section><div class="toolbar"><h2>待人工审核</h2><span class="small muted" id="reviewCount"></span></div><div id="review"></div></section>
|
||||
<section><div class="toolbar"><h2>任务执行记录</h2><span class="small muted">包含自动去重任务</span></div><div id="jobsBox"></div></section>
|
||||
<section><div class="toolbar"><h2>最近操作</h2><span class="small muted">可回滚仍保持同 inode 的 Hard Link</span></div><div id="historyBox"></div></section>
|
||||
<section><h2>运行设置</h2><div id="settingsBox" class="muted">加载中...</div></section>
|
||||
</div><div id="toast"></div>
|
||||
<script>
|
||||
const esc=s=>String(s??'').replace(/[&<>"]/g,c=>({'&':'&','<':'<','>':'>','"':'"'}[c]));
|
||||
async function api(u,o={}){o.headers=Object.assign({},o.headers||{});let t=localStorage.getItem('am_token');if(t)o.headers.Authorization='Bearer '+t;let r=await fetch(u,o);let j=await r.json();if(r.status===401){t=prompt('请输入 AM Deduper 管理 Token');if(t){localStorage.setItem('am_token',t);return api(u,o)} }if(!r.ok)throw Error(j.error||r.status);return j}
|
||||
async function load(){try{
|
||||
let s=await api('/api/stats');stats.innerHTML=s.cards.map(x=>'<div class="card"><div class="muted">'+esc(x.name)+'</div><div class="num">'+esc(x.value)+'</div></div>').join('');
|
||||
let d=await api('/api/duplicates');dupes.innerHTML=d.groups.length?d.groups.map(g=>'<div class="card"><b>'+esc(g.type)+' · '+esc(g.savings)+' MB 可节省</b><table><tr><th>文件</th><th>状态</th></tr>'+g.items.map(x=>'<tr><td class="path">'+esc(x.file_path)+'</td><td>'+esc(x.status)+'</td></tr>').join('')+'</table></div>').join(''):'<p class="muted">暂无重复组。</p>';
|
||||
|
||||
let q=await api('/api/review');review.innerHTML=q.items.length?'<table><tr><th>相似度</th><th>文件</th><th>候选目标</th><th>操作</th></tr>'+q.items.map(x=>'<tr><td>'+esc(x.similarity||'-')+'</td><td class="path">'+esc(x.file_path)+'<br>'+esc(x.title||'')+' '+esc(x.artist||'')+'</td><td class="path">'+esc(x.candidate_path||'-')+'</td><td><button onclick="reviewAction('+x.id+',\'reject\')">保留</button> '+(x.candidate_id?'<button class="primary" onclick="reviewAction('+x.id+',\'approve\','+x.candidate_id+')">确认重复</button>':'')+'</td></tr>').join('')+'</table>':'<p class="muted">暂无待审核项目。</p>';
|
||||
let j=await api('/api/jobs');jobsBox.innerHTML=j.items.length?'<table><tr><th>ID</th><th>任务</th><th>状态</th><th>开始</th><th>结束</th></tr>'+j.items.map(x=>'<tr><td>'+x.id+'</td><td>'+esc(x.kind)+'</td><td>'+esc(x.status)+'</td><td>'+esc(x.started_at)+'</td><td>'+esc(x.finished_at||'-')+'</td></tr>').join('')+'</table>':'<p class="muted">暂无任务。</p>';
|
||||
|
||||
let h=await api('/api/history');historyBox.innerHTML=h.items.length?'<table><tr><th>ID</th><th>时间</th><th>操作</th><th>文件</th><th>目标</th><th></th></tr>'+h.items.map(x=>'<tr><td>'+x.id+'</td><td>'+esc(x.created_at)+'</td><td>'+esc(x.action)+'</td><td class="path">'+esc(x.file_path)+'</td><td class="path">'+esc(x.winner_path||'')+'</td><td>'+(x.reversible?'<button onclick="rollback('+x.id+')">回滚</button>':'')+'</td></tr>').join('')+'</table>':'<p class="muted">暂无记录。</p>';
|
||||
}catch(e){alert(e.message)}}
|
||||
async function run(kind){try{await api('/api/jobs',{method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify({kind})});load()}catch(e){alert(e.message)}}
|
||||
async function reviewAction(id,action,candidate_id){try{await api('/api/review',{method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify({id,action,candidate_id})});load()}catch(e){alert(e.message)}}
|
||||
async function rollback(id){if(!confirm('确认回滚此 Hard Link?'))return;try{await api('/api/rollback',{method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify({id})});load()}catch(e){alert(e.message)}}
|
||||
let cfg=await api('/api/settings');settingsBox.innerHTML='<div>音乐目录:<span class="path">'+esc(cfg.root)+'</span></div><div>数据库:<span class="path">'+esc(cfg.db)+'</span></div><div>认证:'+esc(cfg.auth)+'</div><div>L2:相似度 ≥ '+esc(cfg.l2_min)+' 自动判定,'+esc(cfg.l2_review)+' 起进入审核;时长窗口 ±'+esc(cfg.duration_delta)+' 秒</div>';
|
||||
}catch(e){alert(e.message)}}
|
||||
load();setInterval(load,5000);
|
||||
const esc=s=>String(s??'').replace(/[&<>\"]/g,c=>({'&':'&','<':'<','>':'>','"':'"'}[c]));
|
||||
const labels={scan:'扫描索引',dry:'Dry-run 分析',dedupe:'执行去重'};
|
||||
let busy=false;
|
||||
function toast(msg,error=false){const e=document.getElementById('toast');e.textContent=msg;e.className=error?'error':'';e.style.display='block';clearTimeout(window._toast);window._toast=setTimeout(()=>e.style.display='none',4500)}
|
||||
async function api(u,o={}){o.headers=Object.assign({},o.headers||{});const t=localStorage.getItem('am_token');if(t)o.headers.Authorization='Bearer '+t;let r=await fetch(u,o);let j={};try{j=await r.json()}catch{}if(r.status===401){const nt=prompt('请输入 AM Deduper 管理 Token');if(nt){localStorage.setItem('am_token',nt);return api(u,o)}throw Error('需要管理 Token')}if(!r.ok)throw Error(j.error||('HTTP '+r.status));return j}
|
||||
function setButtons(v){document.querySelectorAll('[data-action]').forEach(b=>b.disabled=v)}
|
||||
function renderStatus(j){const e=document.getElementById('status');if(!j||!j.items?.length){e.innerHTML='<div><div class="status-main">系统空闲</div><div class="status-detail">监听服务会在新音乐稳定写入后自动进行 SHA-256 精确去重。</div></div><span class="badge">运行正常</span>';return}const x=j.items[0],running=x.status==='running';let detail=x.result||x.error||'等待结果';e.innerHTML='<div><div class="status-main '+(running?'running':x.status==='failed'?'error':'')+'">'+(running?'<span class="jobline"><span class="spinner"></span>':'')+esc(labels[x.kind]||x.kind)+(running?'…</span>':'')+'</div><div class="status-detail">'+esc(detail)+'</div></div><span class="badge">'+esc(x.status)+'</span>';}
|
||||
function formatResult(r){if(!r)return '';try{const x=JSON.parse(r);const parts=[];if(x.indexed!=null)parts.push('索引 '+x.indexed);if(x.exact_hardlinked!=null)parts.push('精确去重 '+x.exact_hardlinked+' 个');if(x.exact!=null)parts.push('Exact '+x.exact+' 个');if(x.acoustic)parts.push('L2 '+(x.acoustic.equivalent??0)+' 个');return parts.join(' · ')||r}catch{return r}}
|
||||
async function load(manual=false){try{
|
||||
const [s,d,q,j,h,cfg]=await Promise.all([api('/api/stats'),api('/api/duplicates'),api('/api/review'),api('/api/jobs'),api('/api/history'),api('/api/settings')]);
|
||||
document.getElementById('stats').innerHTML=s.cards.map(x=>'<div class="card"><div class="label">'+esc(x.name)+'</div><div class="num">'+esc(x.value)+'</div><div class="hint">'+esc(x.hint||'')+'</div></div>').join('');
|
||||
document.getElementById('dupeCount').textContent=d.groups.length+' 组';document.getElementById('dupes').innerHTML=d.groups.length?d.groups.map(g=>'<div class="card"><b>'+esc(g.type)+'</b> · 可节省约 '+esc(g.savings)+' MB · '+esc(g.items.length)+' 个逻辑路径<table><tr><th>路径</th><th>状态</th><th>inode</th></tr>'+g.items.map(x=>'<tr><td class="path">'+esc(x.file_path)+'</td><td><span class="badge">'+esc(x.status)+'</span>'+(x.id===g.winner_id?' <span class="badge">保留路径</span>':'')+'</td><td>'+esc(x.inode)+'</td></tr>').join('')+'</table></div>').join(''):'<div class="empty">没有发现 SHA-256 完全重复。新文件会由监听服务自动检查。</div>';
|
||||
document.getElementById('reviewCount').textContent=q.items.length+' 项';document.getElementById('review').innerHTML=q.items.length?'<table><tr><th>相似度</th><th>文件</th><th>候选目标</th><th>操作</th></tr>'+q.items.map(x=>'<tr><td>'+esc(x.similarity||'-')+'</td><td class="path">'+esc(x.file_path)+'<br>'+esc(x.title||'')+' '+esc(x.artist||'')+'</td><td class="path">'+esc(x.candidate_path||'-')+'</td><td><button onclick="reviewAction('+x.id+',\'reject\')">保留原文件</button> '+(x.candidate_id?'<button class="primary" onclick="reviewAction('+x.id+',\'approve\','+x.candidate_id+')">确认硬链接</button>':'')+'</td></tr>').join('')+'</table>':'<div class="empty">暂无人工审核项目。</div>';
|
||||
document.getElementById('jobsBox').innerHTML=j.items.length?'<table><tr><th>ID</th><th>任务</th><th>状态</th><th>时间</th><th>结果</th></tr>'+j.items.map(x=>'<tr><td>'+x.id+'</td><td>'+esc(labels[x.kind]||x.kind)+'</td><td class="'+(x.status==='failed'?'error':x.status==='running'?'running':'success')+'">'+esc(x.status)+'</td><td>'+esc(x.started_at)+'<br>'+esc(x.finished_at||'进行中')+'</td><td>'+esc(formatResult(x.result)||x.error||'-')+(x.error?'<details><summary>错误详情</summary>'+esc(x.error)+'</details>':'')+'</td></tr>').join('')+'</table>':'<div class="empty">暂无任务。启动服务后会自动记录 startup-reconcile。</div>';
|
||||
document.getElementById('historyBox').innerHTML=h.items.length?'<table><tr><th>ID</th><th>时间</th><th>操作</th><th>文件</th><th>目标</th><th></th></tr>'+h.items.map(x=>'<tr><td>'+x.id+'</td><td>'+esc(x.created_at)+'</td><td>'+esc(x.action)+'</td><td class="path">'+esc(x.file_path)+'</td><td class="path">'+esc(x.winner_path||'')+'</td><td>'+(x.reversible?'<button onclick="rollback('+x.id+')">回滚</button>':'')+'</td></tr>').join('')+'</table>':'<div class="empty">暂无去重操作记录。</div>';
|
||||
document.getElementById('settingsBox').innerHTML='<div>音乐目录:<span class="path">'+esc(cfg.root)+'</span></div><div>数据库:<span class="path">'+esc(cfg.db)+'</span></div><div>认证:'+esc(cfg.auth)+'</div><div>自动精确去重:<b>已启用</b>(watch 启动时会先扫描一次,之后监听新文件)</div><div>L2:相似度 ≥ '+esc(cfg.l2_min)+' 自动判定,'+esc(cfg.l2_review)+' 起进入审核;时长窗口 ±'+esc(cfg.duration_delta)+' 秒</div>';
|
||||
renderStatus(j);if(manual)toast('已刷新');
|
||||
}catch(e){renderStatus(null);toast(e.message,true)}}
|
||||
async function run(kind){if(busy)return;if(kind==='dedupe'&&!confirm('确认执行安全去重?\n\nSHA-256 完全相同的文件会使用 Hard Link 合并物理存储;逻辑路径会保留。'))return;busy=true;setButtons(true);try{const r=await api('/api/jobs',{method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify({kind})});toast((labels[kind]||kind)+' 已启动,结果会显示在“任务执行记录”中。');await load();const timer=setInterval(async()=>{try{const j=await api('/api/jobs');renderStatus(j);if(!j.items.length||j.items[0].id!==r.job_id||j.items[0].status==='running')return;clearInterval(timer);busy=false;setButtons(false);toast((labels[kind]||kind)+'完成:'+(formatResult(j.items[0].result)||j.items[0].error||j.items[0].status),j.items[0].status==='failed');await load()}catch(e){clearInterval(timer);busy=false;setButtons(false);toast(e.message,true)}},1000)}catch(e){busy=false;setButtons(false);toast(e.message,true)}}
|
||||
async function reviewAction(id,action,candidate_id){try{await api('/api/review',{method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify({id,action,candidate_id})});toast(action==='approve'?'已确认并建立 Hard Link':'已保留原文件');await load()}catch(e){toast(e.message,true)}}
|
||||
async function rollback(id){if(!confirm('确认回滚此 Hard Link?回滚会恢复 loser 的独立文件内容,并保留两个逻辑路径。'))return;try{await api('/api/rollback',{method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify({id})});toast('回滚完成');await load()}catch(e){toast(e.message,true)}}
|
||||
load();setInterval(()=>load(),5000);
|
||||
</script></html>'''
|
||||
|
||||
|
||||
def stats():
|
||||
with connect(DB) as c:
|
||||
total=c.execute("SELECT COUNT(*) n FROM audio_tracks WHERE status IN ('active','hardlinked','review')").fetchone()["n"]
|
||||
hard=c.execute("SELECT COUNT(*) n FROM audio_tracks WHERE status='hardlinked'").fetchone()["n"]
|
||||
review=c.execute("SELECT COUNT(*) n FROM audio_tracks WHERE status='review'").fetchone()["n"]
|
||||
size=c.execute("SELECT COALESCE(SUM(file_size),0) n FROM audio_tracks WHERE status IN ('active','hardlinked','review')").fetchone()["n"]
|
||||
ops=c.execute("SELECT COUNT(*) n FROM operations").fetchone()["n"]
|
||||
return {"cards":[{"name":"音乐文件","value":total},{"name":"Hard Link","value":hard},{"name":"待人工审核","value":review},{"name":"索引大小","value":f"{size/1024/1024/1024:.2f} GB"},{"name":"操作记录","value":ops}]}
|
||||
total = c.execute("SELECT COUNT(*) n FROM audio_tracks WHERE status IN ('active','hardlinked','review')").fetchone()["n"]
|
||||
hard = c.execute("SELECT COUNT(*) n FROM audio_tracks WHERE status='hardlinked'").fetchone()["n"]
|
||||
review = c.execute("SELECT COUNT(*) n FROM audio_tracks WHERE status='review'").fetchone()["n"]
|
||||
size = c.execute("SELECT COALESCE(SUM(file_size),0) n FROM audio_tracks WHERE status IN ('active','hardlinked','review')").fetchone()["n"]
|
||||
ops = c.execute("SELECT COUNT(*) n FROM operations").fetchone()["n"]
|
||||
return {"cards":[
|
||||
{"name":"逻辑音乐文件","value":total,"hint":"每个用户路径各算 1 个"},
|
||||
{"name":"Hard Link 文件","value":hard,"hint":"物理数据已共享"},
|
||||
{"name":"待人工审核","value":review,"hint":"L2 相似或版本冲突"},
|
||||
{"name":"索引逻辑大小","value":f"{size/1024/1024/1024:.2f} GB","hint":"按路径统计,非实际占用"},
|
||||
{"name":"操作记录","value":ops,"hint":"可追溯/可回滚"}]}
|
||||
|
||||
|
||||
def duplicates():
|
||||
with connect(DB) as c:
|
||||
rows=c.execute("SELECT * FROM audio_tracks WHERE status IN ('active','hardlinked') AND sha256 IS NOT NULL ORDER BY sha256,id").fetchall()
|
||||
groups={}
|
||||
for r in rows: groups.setdefault(r["sha256"],[]).append(dict(r))
|
||||
out=[]
|
||||
for _,items in groups.items():
|
||||
if len(items)>1:
|
||||
winner=choose_winner(items)
|
||||
ordered=[winner]+[x for x in items if x["id"]!=winner["id"]]
|
||||
savings=round(sum(x["file_size"] for x in ordered[1:])/1024/1024,2)
|
||||
out.append({"type":"SHA256 完全重复","savings":savings,"winner_id":winner["id"],"items":ordered})
|
||||
rows = c.execute("SELECT * FROM audio_tracks WHERE status IN ('active','hardlinked') AND sha256 IS NOT NULL ORDER BY sha256,id").fetchall()
|
||||
groups = {}
|
||||
for r in rows: groups.setdefault(r["sha256"], []).append(dict(r))
|
||||
out = []
|
||||
for _, items in groups.items():
|
||||
if len(items) > 1:
|
||||
winner = choose_winner(items)
|
||||
ordered = [winner] + [x for x in items if x["id"] != winner["id"]]
|
||||
savings = round(sum(x["file_size"] for x in ordered[1:]) / 1024 / 1024, 2)
|
||||
out.append({"type":"SHA256 完全重复", "savings":savings, "winner_id":winner["id"], "items":ordered})
|
||||
return {"groups":out[:100]}
|
||||
|
||||
|
||||
def review_items():
|
||||
with connect(DB) as c: rows=c.execute("SELECT * FROM audio_tracks WHERE status='review' ORDER BY updated_at DESC").fetchall()
|
||||
out=[]
|
||||
with connect(DB) as c: rows = c.execute("SELECT * FROM audio_tracks WHERE status='review' ORDER BY updated_at DESC").fetchall()
|
||||
out = []
|
||||
for r in rows:
|
||||
d=dict(r); d["similarity"]=(r["review_status"] or "").replace("similarity=","")
|
||||
d = dict(r); d["similarity"] = (r["review_status"] or "").replace("similarity=", "")
|
||||
with connect(DB) as c2:
|
||||
cand=c2.execute("SELECT id,file_path FROM audio_tracks WHERE id!=? AND status IN ('active','hardlinked') AND duration BETWEEN ? AND ? ORDER BY ABS(duration-?) LIMIT 1",(r["id"],(r["duration"] or 0)-3,(r["duration"] or 0)+3,r["duration"] or 0)).fetchone()
|
||||
d["candidate_id"]=cand["id"] if cand else None; d["candidate_path"]=cand["file_path"] if cand else None; out.append(d)
|
||||
cand = c2.execute("SELECT id,file_path FROM audio_tracks WHERE id!=? AND status IN ('active','hardlinked') AND duration BETWEEN ? AND ? ORDER BY ABS(duration-?) LIMIT 1", (r["id"], (r["duration"] or 0)-3, (r["duration"] or 0)+3, r["duration"] or 0)).fetchone()
|
||||
d["candidate_id"] = cand["id"] if cand else None; d["candidate_path"] = cand["file_path"] if cand else None; out.append(d)
|
||||
return {"items":out}
|
||||
|
||||
|
||||
def history():
|
||||
with connect(DB) as c: rows=c.execute("SELECT id,created_at,action,file_path,winner_path,details FROM operations ORDER BY id DESC LIMIT 100").fetchall()
|
||||
with connect(DB) as c: rows = c.execute("SELECT id,created_at,action,file_path,winner_path,details FROM operations ORDER BY id DESC LIMIT 100").fetchall()
|
||||
return {"items":[dict(r, reversible=r["action"] in ("exact-hardlink","acoustic-hardlink","manual-hardlink")) for r in rows]}
|
||||
|
||||
|
||||
def jobs():
|
||||
with connect(DB) as c: rows=c.execute("SELECT * FROM jobs ORDER BY id DESC LIMIT 30").fetchall()
|
||||
with connect(DB) as c: rows = c.execute("SELECT * FROM jobs ORDER BY id DESC LIMIT 30").fetchall()
|
||||
return {"items":[dict(r) for r in rows]}
|
||||
|
||||
|
||||
def settings():
|
||||
from .l2 import MIN_SIMILARITY, REVIEW_SIMILARITY, DURATION_DELTA
|
||||
return {"root":str(ROOT),"db":str(DB),"auth":"Token 已启用" if TOKEN else "未启用(仅建议内网)",
|
||||
"l2_min":MIN_SIMILARITY,"l2_review":REVIEW_SIMILARITY,"duration_delta":DURATION_DELTA}
|
||||
return {"root":str(ROOT),"db":str(DB),"auth":"Token 已启用" if TOKEN else "未启用(仅建议内网)","l2_min":MIN_SIMILARITY,"l2_review":REVIEW_SIMILARITY,"duration_delta":DURATION_DELTA}
|
||||
|
||||
|
||||
def run_job(kind):
|
||||
with connect(DB) as c:
|
||||
cur=c.execute("INSERT INTO jobs(kind,status) VALUES(?,?)",(kind,"running")); jid=cur.lastrowid
|
||||
jid = c.execute("INSERT INTO jobs(kind,status) VALUES(?,?)", (kind,"running")).lastrowid
|
||||
def work():
|
||||
try:
|
||||
with LOCK:
|
||||
if kind=="scan": result={"count":scan(ROOT,DB)}
|
||||
elif kind=="dry":
|
||||
scan(ROOT,DB); result={"exact":len(exact_dedupe(DB,ROOT,True)),"acoustic":acoustic_dedupe(DB,ROOT,True)}
|
||||
indexed = scan(ROOT, DB)
|
||||
if kind == "scan": result = {"indexed": indexed}
|
||||
elif kind == "dry":
|
||||
result = {"indexed": indexed, "exact": len(exact_dedupe(DB, ROOT, True)), "acoustic": acoustic_dedupe(DB, ROOT, True)}
|
||||
else:
|
||||
scan(ROOT,DB); result={"exact":len(exact_dedupe(DB,ROOT)),"acoustic":acoustic_dedupe(DB,ROOT)}
|
||||
with connect(DB) as c: c.execute("UPDATE jobs SET status='success',finished_at=CURRENT_TIMESTAMP,result=? WHERE id=?",(json.dumps(result,ensure_ascii=False),jid))
|
||||
result = {"indexed": indexed, "exact": len(exact_dedupe(DB, ROOT)), "acoustic": acoustic_dedupe(DB, ROOT)}
|
||||
with connect(DB) as c: c.execute("UPDATE jobs SET status='success',finished_at=CURRENT_TIMESTAMP,result=? WHERE id=?", (json.dumps(result,ensure_ascii=False),jid))
|
||||
except Exception as e:
|
||||
with connect(DB) as c: c.execute("UPDATE jobs SET status='failed',finished_at=CURRENT_TIMESTAMP,error=? WHERE id=?",(str(e),jid))
|
||||
with connect(DB) as c: c.execute("UPDATE jobs SET status='failed',finished_at=CURRENT_TIMESTAMP,error=? WHERE id=?", (str(e),jid))
|
||||
threading.Thread(target=work,daemon=True).start()
|
||||
return jid
|
||||
|
||||
|
||||
def rollback(opid):
|
||||
with connect(DB) as c: row=c.execute("SELECT * FROM operations WHERE id=?",(opid,)).fetchone()
|
||||
with connect(DB) as c: row = c.execute("SELECT * FROM operations WHERE id=?", (opid,)).fetchone()
|
||||
if not row or row["action"] not in ("exact-hardlink","acoustic-hardlink","manual-hardlink"): raise ValueError("operation not reversible")
|
||||
loser,winner=Path(row["file_path"]),Path(row["winner_path"])
|
||||
if not loser.exists() or not winner.exists() or loser.stat().st_ino!=winner.stat().st_ino: raise ValueError("current file state changed; refusing rollback")
|
||||
tmp=loser.with_name(loser.name+".dedupe-rollback-"+uuid.uuid4().hex)
|
||||
loser,winner = Path(row["file_path"]),Path(row["winner_path"])
|
||||
if not loser.exists() or not winner.exists() or loser.stat().st_ino != winner.stat().st_ino: raise ValueError("current file state changed; refusing rollback")
|
||||
tmp = loser.with_name(loser.name+".dedupe-rollback-"+uuid.uuid4().hex)
|
||||
shutil.copy2(winner,tmp); os.replace(tmp,loser)
|
||||
with connect(DB) as c:
|
||||
c.execute("UPDATE audio_tracks SET status='active',review_status='rolled-back',updated_at=CURRENT_TIMESTAMP WHERE file_path=?",(str(loser),))
|
||||
c.execute("INSERT INTO operations(action,file_path,winner_path,details) VALUES(?,?,?,?)",("rollback",str(loser),str(winner),f"rollback of operation {opid}"))
|
||||
c.execute("UPDATE audio_tracks SET status='active',review_status='rolled-back',updated_at=CURRENT_TIMESTAMP WHERE file_path=?", (str(loser),))
|
||||
c.execute("INSERT INTO operations(action,file_path,winner_path,details) VALUES(?,?,?,?)", ("rollback",str(loser),str(winner),f"rollback of operation {opid}"))
|
||||
|
||||
|
||||
def approve(rid,cid):
|
||||
with connect(DB) as c:
|
||||
r=c.execute("SELECT * FROM audio_tracks WHERE id=? AND status='review'",(rid,)).fetchone()
|
||||
w=c.execute("SELECT * FROM audio_tracks WHERE id=? AND status IN ('active','hardlinked')",(cid,)).fetchone()
|
||||
r=c.execute("SELECT * FROM audio_tracks WHERE id=? AND status='review'",(rid,)).fetchone(); w=c.execute("SELECT * FROM audio_tracks WHERE id=? AND status IN ('active','hardlinked')",(cid,)).fetchone()
|
||||
if not r or not w: raise ValueError("review item or candidate not found")
|
||||
lp,wp=Path(r["file_path"]),Path(w["file_path"])
|
||||
if not lp.exists() or not wp.exists() or not lp.is_relative_to(ROOT) or not wp.is_relative_to(ROOT): raise ValueError("invalid library path")
|
||||
if lp.stat().st_dev!=wp.stat().st_dev: raise ValueError("different filesystem")
|
||||
if lp.stat().st_ino!=wp.stat().st_ino:
|
||||
hardlink_duplicate(lp,wp)
|
||||
if lp.stat().st_dev != wp.stat().st_dev: raise ValueError("different filesystem")
|
||||
if lp.stat().st_ino != wp.stat().st_ino: hardlink_duplicate(lp,wp)
|
||||
with connect(DB) as c:
|
||||
c.execute("UPDATE audio_tracks SET status='hardlinked',review_status='manual-approved',updated_at=CURRENT_TIMESTAMP WHERE id=?",(rid,))
|
||||
c.execute("INSERT INTO operations(action,file_path,winner_path,details) VALUES(?,?,?,?)",("manual-hardlink",str(lp),str(wp),"manual review approval"))
|
||||
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def send(self,status,data,ctype="application/json"):
|
||||
raw=data.encode() if isinstance(data,str) else json.dumps(data,ensure_ascii=False).encode()
|
||||
self.send_response(status); self.send_header("Content-Type",ctype+"; charset=utf-8"); self.send_header("Content-Length",str(len(raw))); self.end_headers(); self.wfile.write(raw)
|
||||
def authorized(self):
|
||||
if not TOKEN: return True
|
||||
return self.headers.get("Authorization","") == "Bearer "+TOKEN
|
||||
|
||||
raw=data.encode() if isinstance(data,str) else json.dumps(data,ensure_ascii=False).encode(); self.send_response(status); self.send_header("Content-Type",ctype+"; charset=utf-8"); self.send_header("Content-Length",str(len(raw))); self.end_headers(); self.wfile.write(raw)
|
||||
def authorized(self): return not TOKEN or self.headers.get("Authorization","") == "Bearer "+TOKEN
|
||||
def do_GET(self):
|
||||
p=urlparse(self.path).path
|
||||
try:
|
||||
if p=="/": return self.send(200,HTML,"text/html")
|
||||
if not self.authorized(): return self.send(401,{"error":"unauthorized"})
|
||||
if p=="/api/stats": return self.send(200,stats())
|
||||
if p=="/api/duplicates": return self.send(200,duplicates())
|
||||
if p=="/api/review": return self.send(200,review_items())
|
||||
if p=="/api/history": return self.send(200,history())
|
||||
if p=="/api/jobs": return self.send(200,jobs())
|
||||
if p=="/api/settings": return self.send(200,settings())
|
||||
routes={"/api/stats":stats,"/api/duplicates":duplicates,"/api/review":review_items,"/api/history":history,"/api/jobs":jobs,"/api/settings":settings}
|
||||
if p in routes: return self.send(200,routes[p]())
|
||||
return self.send(404,{"error":"not found"})
|
||||
except Exception as e: return self.send(500,{"error":str(e)})
|
||||
|
||||
def do_POST(self):
|
||||
p=urlparse(self.path).path
|
||||
if not self.authorized(): return self.send(401,{"error":"unauthorized"})
|
||||
try: body=json.loads(self.rfile.read(int(self.headers.get("content-length","0"))) or b"{}")
|
||||
except Exception: return self.send(400,{"error":"invalid json"})
|
||||
try:
|
||||
if p=="/api/jobs":
|
||||
if body.get("kind") not in {"scan","dry","dedupe"}: return self.send(400,{"error":"invalid job"})
|
||||
return self.send(202,{"ok":True,"job_id":run_job(body["kind"])})
|
||||
if p=="/api/jobs" and body.get("kind") in {"scan","dry","dedupe"}: return self.send(202,{"ok":True,"job_id":run_job(body["kind"])})
|
||||
if p=="/api/review":
|
||||
if body.get("action")=="reject":
|
||||
with connect(DB) as c: c.execute("UPDATE audio_tracks SET status='active',review_status='rejected',updated_at=CURRENT_TIMESTAMP WHERE id=? AND status='review'",(int(body["id"]),))
|
||||
return self.send(200,{"ok":True})
|
||||
if body.get("action")=="approve": approve(int(body["id"]),int(body["candidate_id"])); return self.send(200,{"ok":True})
|
||||
return self.send(400,{"error":"invalid action"})
|
||||
if p=="/api/rollback": rollback(int(body["id"])); return self.send(200,{"ok":True})
|
||||
return self.send(404,{"error":"not found"})
|
||||
return self.send(400,{"error":"invalid request"})
|
||||
except Exception as e: return self.send(409,{"error":str(e)})
|
||||
def log_message(self,*args): pass
|
||||
|
||||
def serve(host="0.0.0.0",port=8080):
|
||||
print(f"AM Deduper UI listening on {host}:{port}",flush=True)
|
||||
ThreadingHTTPServer((host,port),Handler).serve_forever()
|
||||
|
||||
def serve(host="0.0.0.0",port=8080):
|
||||
print(f"AM Deduper UI listening on {host}:{port}",flush=True); ThreadingHTTPServer((host,port),Handler).serve_forever()
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "am-music-deduper"
|
||||
version = "0.1.0"
|
||||
version = "0.2.0"
|
||||
description = "Lightweight NAS music deduplication service for AllMusic"
|
||||
requires-python = ">=3.10"
|
||||
dependencies = [
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
from pathlib import Path
|
||||
|
||||
from am_deduper.db import connect
|
||||
from am_deduper.scanner import scan
|
||||
|
||||
|
||||
def test_replacing_file_invalidates_content_hashes(tmp_path: Path):
|
||||
root = tmp_path / "AllMusic"
|
||||
(root / "admin").mkdir(parents=True)
|
||||
path = root / "admin" / "song.flac"
|
||||
db = root / ".dedupe" / "dedupe.db"
|
||||
path.write_bytes(b"first" * 1000)
|
||||
scan(root, db)
|
||||
with connect(db) as c:
|
||||
c.execute("UPDATE audio_tracks SET sha256='old-sha', chromaprint='old-fp' WHERE file_path=?", (str(path),))
|
||||
path.write_bytes(b"second" * 1000)
|
||||
scan(root, db)
|
||||
with connect(db) as c:
|
||||
row = c.execute("SELECT sha256,chromaprint FROM audio_tracks WHERE file_path=?", (str(path),)).fetchone()
|
||||
assert row["sha256"] is None
|
||||
assert row["chromaprint"] is None
|
||||
Reference in New Issue
Block a user