192 lines
14 KiB
Python
192 lines
14 KiB
Python
from __future__ import annotations
|
||
import json, os, shutil, threading, uuid
|
||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||
from pathlib import Path
|
||
from urllib.parse import urlparse
|
||
from .db import connect
|
||
from .scanner import scan
|
||
from .exact import exact_dedupe, hardlink_duplicate, choose_winner
|
||
from .l2 import acoustic_dedupe
|
||
|
||
ROOT=Path(os.environ.get("AM_DEDUPER_ROOT","/music/AllMusic")).resolve()
|
||
DB=Path(os.environ.get("AM_DEDUPER_DB","/data/dedupe.db")).resolve()
|
||
TOKEN=os.environ.get("AM_DEDUPER_TOKEN","").strip()
|
||
LOCK=threading.Lock()
|
||
|
||
HTML=r'''<!doctype html><html lang="zh-CN"><meta charset="utf-8">
|
||
<meta name="viewport" content="width=device-width,initial-scale=1"><title>AM Deduper</title>
|
||
<style>
|
||
body{font:14px system-ui;margin:0;background:#f5f6f8;color:#202124}header{background:#111827;color:#fff;padding:18px 24px}
|
||
main{max-width:1280px;margin:18px auto;padding:0 14px}.grid{display:grid;grid-template-columns:repeat(5,1fr);gap:10px}
|
||
.card,section{background:#fff;border:1px solid #ddd;border-radius:10px;padding:14px;margin-bottom:14px}.num{font-size:24px;font-weight:700}
|
||
nav{display:flex;gap:8px;flex-wrap:wrap;margin-bottom:14px}button{padding:8px 12px;border:1px solid #bbb;border-radius:7px;background:#fff;cursor:pointer}
|
||
button.primary{background:#111827;color:#fff}button.danger{background:#fee2e2}table{width:100%;border-collapse:collapse}th,td{padding:9px;border-bottom:1px solid #eee;text-align:left;vertical-align:top}
|
||
.path{font:12px monospace;word-break:break-all}.muted{color:#666}.badge{background:#eee;border-radius:10px;padding:3px 7px}
|
||
@media(max-width:800px){.grid{grid-template-columns:repeat(2,1fr)}table{font-size:12px}}
|
||
</style>
|
||
<header><b>AM Deduper</b> · NAS Music Library</header><main>
|
||
<nav><button class="primary" onclick="run('scan')">扫描索引</button><button onclick="run('dry')">分析 / Dry-run</button>
|
||
<button onclick="run('dedupe')">执行去重</button><button onclick="load()">刷新</button></nav>
|
||
<div class="grid" id="stats"></div>
|
||
<section><h2>重复组</h2><div id="dupes"></div></section>
|
||
<section><h2>待人工审核</h2><div id="review"></div></section>
|
||
<section><h2>任务</h2><div id="jobsBox"></div></section>
|
||
<section><h2>最近操作</h2><div id="historyBox"></div></section>
|
||
<section><h2>设置</h2><div id="settingsBox" class="muted">加载中...</div></section>
|
||
</main>
|
||
|
||
<script>
|
||
const esc=s=>String(s??'').replace(/[&<>"]/g,c=>({'&':'&','<':'<','>':'>','"':'"'}[c]));
|
||
async function api(u,o={}){o.headers=Object.assign({},o.headers||{});let t=localStorage.getItem('am_token');if(t)o.headers.Authorization='Bearer '+t;let r=await fetch(u,o);let j=await r.json();if(r.status===401){t=prompt('请输入 AM Deduper 管理 Token');if(t){localStorage.setItem('am_token',t);return api(u,o)} }if(!r.ok)throw Error(j.error||r.status);return j}
|
||
async function load(){try{
|
||
let s=await api('/api/stats');stats.innerHTML=s.cards.map(x=>'<div class="card"><div class="muted">'+esc(x.name)+'</div><div class="num">'+esc(x.value)+'</div></div>').join('');
|
||
let d=await api('/api/duplicates');dupes.innerHTML=d.groups.length?d.groups.map(g=>'<div class="card"><b>'+esc(g.type)+' · '+esc(g.savings)+' MB 可节省</b><table><tr><th>文件</th><th>状态</th></tr>'+g.items.map(x=>'<tr><td class="path">'+esc(x.file_path)+'</td><td>'+esc(x.status)+'</td></tr>').join('')+'</table></div>').join(''):'<p class="muted">暂无重复组。</p>';
|
||
|
||
let q=await api('/api/review');review.innerHTML=q.items.length?'<table><tr><th>相似度</th><th>文件</th><th>候选目标</th><th>操作</th></tr>'+q.items.map(x=>'<tr><td>'+esc(x.similarity||'-')+'</td><td class="path">'+esc(x.file_path)+'<br>'+esc(x.title||'')+' '+esc(x.artist||'')+'</td><td class="path">'+esc(x.candidate_path||'-')+'</td><td><button onclick="reviewAction('+x.id+',\'reject\')">保留</button> '+(x.candidate_id?'<button class="primary" onclick="reviewAction('+x.id+',\'approve\','+x.candidate_id+')">确认重复</button>':'')+'</td></tr>').join('')+'</table>':'<p class="muted">暂无待审核项目。</p>';
|
||
let j=await api('/api/jobs');jobsBox.innerHTML=j.items.length?'<table><tr><th>ID</th><th>任务</th><th>状态</th><th>开始</th><th>结束</th></tr>'+j.items.map(x=>'<tr><td>'+x.id+'</td><td>'+esc(x.kind)+'</td><td>'+esc(x.status)+'</td><td>'+esc(x.started_at)+'</td><td>'+esc(x.finished_at||'-')+'</td></tr>').join('')+'</table>':'<p class="muted">暂无任务。</p>';
|
||
|
||
let h=await api('/api/history');historyBox.innerHTML=h.items.length?'<table><tr><th>ID</th><th>时间</th><th>操作</th><th>文件</th><th>目标</th><th></th></tr>'+h.items.map(x=>'<tr><td>'+x.id+'</td><td>'+esc(x.created_at)+'</td><td>'+esc(x.action)+'</td><td class="path">'+esc(x.file_path)+'</td><td class="path">'+esc(x.winner_path||'')+'</td><td>'+(x.reversible?'<button onclick="rollback('+x.id+')">回滚</button>':'')+'</td></tr>').join('')+'</table>':'<p class="muted">暂无记录。</p>';
|
||
}catch(e){alert(e.message)}}
|
||
async function run(kind){try{await api('/api/jobs',{method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify({kind})});load()}catch(e){alert(e.message)}}
|
||
async function reviewAction(id,action,candidate_id){try{await api('/api/review',{method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify({id,action,candidate_id})});load()}catch(e){alert(e.message)}}
|
||
async function rollback(id){if(!confirm('确认回滚此 Hard Link？'))return;try{await api('/api/rollback',{method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify({id})});load()}catch(e){alert(e.message)}}
|
||
let cfg=await api('/api/settings');settingsBox.innerHTML='<div>音乐目录：<span class="path">'+esc(cfg.root)+'</span></div><div>数据库：<span class="path">'+esc(cfg.db)+'</span></div><div>认证：'+esc(cfg.auth)+'</div><div>L2：相似度 ≥ '+esc(cfg.l2_min)+' 自动判定，'+esc(cfg.l2_review)+' 起进入审核；时长窗口 ±'+esc(cfg.duration_delta)+' 秒</div>';
|
||
}catch(e){alert(e.message)}}
|
||
load();setInterval(load,5000);
|
||
</script></html>'''
|
||
|
||
def stats():
|
||
with connect(DB) as c:
|
||
total=c.execute("SELECT COUNT(*) n FROM audio_tracks WHERE status IN ('active','hardlinked','review')").fetchone()["n"]
|
||
hard=c.execute("SELECT COUNT(*) n FROM audio_tracks WHERE status='hardlinked'").fetchone()["n"]
|
||
review=c.execute("SELECT COUNT(*) n FROM audio_tracks WHERE status='review'").fetchone()["n"]
|
||
size=c.execute("SELECT COALESCE(SUM(file_size),0) n FROM audio_tracks WHERE status IN ('active','hardlinked','review')").fetchone()["n"]
|
||
ops=c.execute("SELECT COUNT(*) n FROM operations").fetchone()["n"]
|
||
return {"cards":[{"name":"音乐文件","value":total},{"name":"Hard Link","value":hard},{"name":"待人工审核","value":review},{"name":"索引大小","value":f"{size/1024/1024/1024:.2f} GB"},{"name":"操作记录","value":ops}]}
|
||
|
||
def duplicates():
|
||
with connect(DB) as c:
|
||
rows=c.execute("SELECT * FROM audio_tracks WHERE status IN ('active','hardlinked') AND sha256 IS NOT NULL ORDER BY sha256,id").fetchall()
|
||
groups={}
|
||
for r in rows: groups.setdefault(r["sha256"],[]).append(dict(r))
|
||
out=[]
|
||
for _,items in groups.items():
|
||
if len(items)>1:
|
||
winner=choose_winner(items)
|
||
ordered=[winner]+[x for x in items if x["id"]!=winner["id"]]
|
||
savings=round(sum(x["file_size"] for x in ordered[1:])/1024/1024,2)
|
||
out.append({"type":"SHA256 完全重复","savings":savings,"winner_id":winner["id"],"items":ordered})
|
||
return {"groups":out[:100]}
|
||
|
||
def review_items():
|
||
with connect(DB) as c: rows=c.execute("SELECT * FROM audio_tracks WHERE status='review' ORDER BY updated_at DESC").fetchall()
|
||
out=[]
|
||
for r in rows:
|
||
d=dict(r); d["similarity"]=(r["review_status"] or "").replace("similarity=","")
|
||
with connect(DB) as c2:
|
||
cand=c2.execute("SELECT id,file_path FROM audio_tracks WHERE id!=? AND status IN ('active','hardlinked') AND duration BETWEEN ? AND ? ORDER BY ABS(duration-?) LIMIT 1",(r["id"],(r["duration"] or 0)-3,(r["duration"] or 0)+3,r["duration"] or 0)).fetchone()
|
||
d["candidate_id"]=cand["id"] if cand else None; d["candidate_path"]=cand["file_path"] if cand else None; out.append(d)
|
||
return {"items":out}
|
||
|
||
def history():
|
||
with connect(DB) as c: rows=c.execute("SELECT id,created_at,action,file_path,winner_path,details FROM operations ORDER BY id DESC LIMIT 100").fetchall()
|
||
return {"items":[dict(r, reversible=r["action"] in ("exact-hardlink","acoustic-hardlink","manual-hardlink")) for r in rows]}
|
||
|
||
def jobs():
|
||
with connect(DB) as c: rows=c.execute("SELECT * FROM jobs ORDER BY id DESC LIMIT 30").fetchall()
|
||
return {"items":[dict(r) for r in rows]}
|
||
|
||
def settings():
|
||
from .l2 import MIN_SIMILARITY, REVIEW_SIMILARITY, DURATION_DELTA
|
||
return {"root":str(ROOT),"db":str(DB),"auth":"Token 已启用" if TOKEN else "未启用（仅建议内网）",
|
||
"l2_min":MIN_SIMILARITY,"l2_review":REVIEW_SIMILARITY,"duration_delta":DURATION_DELTA}
|
||
|
||
def run_job(kind):
|
||
with connect(DB) as c:
|
||
cur=c.execute("INSERT INTO jobs(kind,status) VALUES(?,?)",(kind,"running")); jid=cur.lastrowid
|
||
def work():
|
||
try:
|
||
with LOCK:
|
||
if kind=="scan": result={"count":scan(ROOT,DB)}
|
||
elif kind=="dry":
|
||
scan(ROOT,DB); result={"exact":len(exact_dedupe(DB,ROOT,True)),"acoustic":acoustic_dedupe(DB,ROOT,True)}
|
||
else:
|
||
scan(ROOT,DB); result={"exact":len(exact_dedupe(DB,ROOT)),"acoustic":acoustic_dedupe(DB,ROOT)}
|
||
with connect(DB) as c: c.execute("UPDATE jobs SET status='success',finished_at=CURRENT_TIMESTAMP,result=? WHERE id=?",(json.dumps(result,ensure_ascii=False),jid))
|
||
except Exception as e:
|
||
with connect(DB) as c: c.execute("UPDATE jobs SET status='failed',finished_at=CURRENT_TIMESTAMP,error=? WHERE id=?",(str(e),jid))
|
||
threading.Thread(target=work,daemon=True).start()
|
||
return jid
|
||
|
||
def rollback(opid):
|
||
with connect(DB) as c: row=c.execute("SELECT * FROM operations WHERE id=?",(opid,)).fetchone()
|
||
if not row or row["action"] not in ("exact-hardlink","acoustic-hardlink","manual-hardlink"): raise ValueError("operation not reversible")
|
||
loser,winner=Path(row["file_path"]),Path(row["winner_path"])
|
||
if not loser.exists() or not winner.exists() or loser.stat().st_ino!=winner.stat().st_ino: raise ValueError("current file state changed; refusing rollback")
|
||
tmp=loser.with_name(loser.name+".dedupe-rollback-"+uuid.uuid4().hex)
|
||
shutil.copy2(winner,tmp); os.replace(tmp,loser)
|
||
with connect(DB) as c:
|
||
c.execute("UPDATE audio_tracks SET status='active',review_status='rolled-back',updated_at=CURRENT_TIMESTAMP WHERE file_path=?",(str(loser),))
|
||
c.execute("INSERT INTO operations(action,file_path,winner_path,details) VALUES(?,?,?,?)",("rollback",str(loser),str(winner),f"rollback of operation {opid}"))
|
||
|
||
def approve(rid,cid):
|
||
with connect(DB) as c:
|
||
r=c.execute("SELECT * FROM audio_tracks WHERE id=? AND status='review'",(rid,)).fetchone()
|
||
w=c.execute("SELECT * FROM audio_tracks WHERE id=? AND status IN ('active','hardlinked')",(cid,)).fetchone()
|
||
if not r or not w: raise ValueError("review item or candidate not found")
|
||
lp,wp=Path(r["file_path"]),Path(w["file_path"])
|
||
if not lp.exists() or not wp.exists() or not lp.is_relative_to(ROOT) or not wp.is_relative_to(ROOT): raise ValueError("invalid library path")
|
||
if lp.stat().st_dev!=wp.stat().st_dev: raise ValueError("different filesystem")
|
||
if lp.stat().st_ino!=wp.stat().st_ino:
|
||
hardlink_duplicate(lp,wp)
|
||
with connect(DB) as c:
|
||
c.execute("UPDATE audio_tracks SET status='hardlinked',review_status='manual-approved',updated_at=CURRENT_TIMESTAMP WHERE id=?",(rid,))
|
||
c.execute("INSERT INTO operations(action,file_path,winner_path,details) VALUES(?,?,?,?)",("manual-hardlink",str(lp),str(wp),"manual review approval"))
|
||
|
||
class Handler(BaseHTTPRequestHandler):
|
||
def send(self,status,data,ctype="application/json"):
|
||
raw=data.encode() if isinstance(data,str) else json.dumps(data,ensure_ascii=False).encode()
|
||
self.send_response(status); self.send_header("Content-Type",ctype+"; charset=utf-8"); self.send_header("Content-Length",str(len(raw))); self.end_headers(); self.wfile.write(raw)
|
||
def authorized(self):
|
||
if not TOKEN: return True
|
||
return self.headers.get("Authorization","") == "Bearer "+TOKEN
|
||
|
||
def do_GET(self):
|
||
p=urlparse(self.path).path
|
||
try:
|
||
if p=="/": return self.send(200,HTML,"text/html")
|
||
if not self.authorized(): return self.send(401,{"error":"unauthorized"})
|
||
if p=="/api/stats": return self.send(200,stats())
|
||
if p=="/api/duplicates": return self.send(200,duplicates())
|
||
if p=="/api/review": return self.send(200,review_items())
|
||
if p=="/api/history": return self.send(200,history())
|
||
if p=="/api/jobs": return self.send(200,jobs())
|
||
if p=="/api/settings": return self.send(200,settings())
|
||
return self.send(404,{"error":"not found"})
|
||
except Exception as e: return self.send(500,{"error":str(e)})
|
||
|
||
def do_POST(self):
|
||
p=urlparse(self.path).path
|
||
if not self.authorized(): return self.send(401,{"error":"unauthorized"})
|
||
try: body=json.loads(self.rfile.read(int(self.headers.get("content-length","0"))) or b"{}")
|
||
except Exception: return self.send(400,{"error":"invalid json"})
|
||
try:
|
||
if p=="/api/jobs":
|
||
if body.get("kind") not in {"scan","dry","dedupe"}: return self.send(400,{"error":"invalid job"})
|
||
return self.send(202,{"ok":True,"job_id":run_job(body["kind"])})
|
||
if p=="/api/review":
|
||
if body.get("action")=="reject":
|
||
with connect(DB) as c: c.execute("UPDATE audio_tracks SET status='active',review_status='rejected',updated_at=CURRENT_TIMESTAMP WHERE id=? AND status='review'",(int(body["id"]),))
|
||
return self.send(200,{"ok":True})
|
||
if body.get("action")=="approve": approve(int(body["id"]),int(body["candidate_id"])); return self.send(200,{"ok":True})
|
||
return self.send(400,{"error":"invalid action"})
|
||
if p=="/api/rollback": rollback(int(body["id"])); return self.send(200,{"ok":True})
|
||
return self.send(404,{"error":"not found"})
|
||
except Exception as e: return self.send(409,{"error":str(e)})
|
||
def log_message(self,*args): pass
|
||
|
||
def serve(host="0.0.0.0",port=8080):
|
||
print(f"AM Deduper UI listening on {host}:{port}",flush=True)
|
||
ThreadingHTTPServer((host,port),Handler).serve_forever()
|
||
|