diff --git a/.gitignore b/.gitignore index 69e39f0..8dc243b 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ __pycache__/ .pytest_cache/ .venv/ +am_music_deduper.egg-info/ diff --git a/Dockerfile b/Dockerfile index 5635fa7..c5c60a8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ FROM python:3.12-slim -RUN apt-get update && apt-get install -y --no-install-recommends libchromaprint-tools && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install -y --no-install-recommends libchromaprint-tools libchromaprint1 && rm -rf /var/lib/apt/lists/* WORKDIR /app COPY pyproject.toml README.md ./ COPY am_deduper ./am_deduper diff --git a/README.md b/README.md index f721707..dcd3116 100644 --- a/README.md +++ b/README.md @@ -2,38 +2,94 @@ 轻量级家庭 NAS 音乐库去重服务，面向 `AllMusic/{username}/` 场景。 -## 设计目标 +## 处理模型 -- 本地 SQLite 元数据库 -- L1 极速候选：文件大小 + 头/尾 64 KiB hash -- Exact Duplicate 最终使用完整 SHA-256 确认 -- L2 使用 Chromaprint 做音频相似度候选 -- duration 预过滤，降低 CPU/I/O -- 同一版本才允许自动择优 -- loser 进入 `.dedupe/trash/`，默认保留 30 天 -- 不使用软链接；物理去重使用 hard link -- inotify 增量处理 + 低优先级全量 reconcile -- 任何不确定或异常默认保留原文件 +```text +AllMusic + ↓ +文件稳定检测 + ↓ +L1：size + head/tail 64 KiB + ↓ +完整 SHA-256 + ├─ 相同 → Exact Duplicate → hard link + └─ 不同 → duration ±3s → Chromaprint + ├─ 高相似且版本一致 → hard link + ├─ 中等相似/版本冲突 → review + └─ 低相似 → 保留 +``` -## 当前阶段 +### 关键安全规则 -第一阶段先完成安全的扫描、SQLite 索引、L1/Exact 去重、trash/quarantine 和测试。 -L2 Chromaprint 比对与生产 NAS 监听在后续阶段接入。 +- SHA-256 是 Exact Duplicate 的最终依据；L1 只负责缩小候选范围。 +- 不根据文件名、歌手、歌名、专辑或来源单独判重。 +- 确认相同版本后，优先使用 hard link 保留每个用户原来的逻辑路径。 +- 用户删除自己的路径不会影响其他 hard link；最后一个 link 删除后数据才释放。 +- 不跨文件系统创建 hard link。 +- 正在写入的文件必须先通过稳定性检查。 +- L2 对版本标记冲突的文件进入 review，不自动合并。 +- `--dry-run` 只报告/更新索引，不修改音乐文件。 +- SQLite 是加速索引，文件系统是最终事实；数据库丢失后可以重新扫描。 -## 安全原则 - -1. 不修改正在写入的文件。 -2. L1 命中必须再做完整 SHA-256 才能判定 exact duplicate。 -3. 不根据歌名/歌手/专辑单独判重。 -4. 不同格式或音频规格不会因为 metadata 相似而直接合并。 -5. 自动处理前保留可恢复路径；失败不删除原文件。 -6. 数据库只是索引，文件系统才是最终事实。 - -## CLI 预期 +## CLI ```bash -am-deduper scan /music/AllMusic -am-deduper dedupe /music/AllMusic -am-deduper reconcile /music/AllMusic -am-deduper trash-prune /music/AllMusic --days 30 +# 建立/更新索引，不修改音乐文件 +am-deduper scan /music/AllMusic --db /data/dedupe.db + +# Exact + L2；首次执行建议 dry-run +am-deduper dedupe /music/AllMusic --db /data/dedupe.db --dry-run + +# 确认后执行 +am-deduper dedupe /music/AllMusic --db /data/dedupe.db + +# 长驻监听：默认只自动执行安全的 Exact hard-link +am-deduper watch /music/AllMusic --db /data/dedupe.db + +# 开启 L2 自动 acoustic 匹配（建议在 dry-run 验证阈值后再启用） +am-deduper watch /music/AllMusic --db /data/dedupe.db --l2 ``` + +## Docker / Feiniu NAS 部署 + +wjyhk 负责构建镜像，推送到 Gitea Container Registry；NAS 只负责 pull + run，不在 NAS 上重新 build。 + +NAS 的 `AllMusic` 与 dedupe 数据建议分开： + +```text +NAS +├── AllMusic/ +│ ├── admin/ +│ ├── alice/ +│ └── bob/ +└── AppData/ + └── am-deduper/ + └── dedupe.db +``` + +注意：如果以后启用 trash，trash 目录必须位于 `AllMusic` 所在同一文件系统内，不能把需要原子 rename/hard-link 语义的文件直接移动到 `/data`。 + +### NAS `.env` + +```dotenv +AM_DEDUPER_IMAGE=//am-music-deduper: +ALLMUSIC_PATH=/AllMusic +DEDUPE_DATA_PATH=/AppData/am-deduper +``` + +### NAS 部署 + +```bash +mkdir -p "$DEDUPE_DATA_PATH" +docker login +docker compose pull +docker compose up -d +``` + +查看日志： + +```bash +docker logs -f am-deduper +``` + +首次上线建议先暂停 `watch`，执行一次完整 `scan` + `dedupe --dry-run`，确认候选数量与 review 项，再启用正式 dedupe；L2 自动处理单独验证后再打开 `--l2`。 diff --git a/am_deduper/acoustic.py b/am_deduper/acoustic.py index 18003d6..942d70e 100644 --- a/am_deduper/acoustic.py +++ b/am_deduper/acoustic.py @@ -5,6 +5,8 @@ import subprocess from pathlib import Path MAX_SECONDS = 120 +MAX_BIT_ERROR = 2 +MAX_ALIGN_OFFSET = 120 def chromaprint(path: Path) -> str | None: @@ -13,26 +15,36 @@ def chromaprint(path: Path) -> str | None: ["fpcalc", "-length", str(MAX_SECONDS), "-json", str(path)], capture_output=True, text=True, timeout=180, check=True, ) - return json.loads(p.stdout).get("fingerprint") + value = json.loads(p.stdout).get("fingerprint") + return value if isinstance(value, str) and value else None except (OSError, subprocess.SubprocessError, ValueError, json.JSONDecodeError): return None +def _popcount(value: int) -> int: + return value.bit_count() + + def similarity(a: str, b: str) -> float: - """Conservative normalized positional similarity for equal-length prefixes.""" + """Compare Chromaprint fingerprints using the AcoustID alignment method. + + This is deliberately bounded to the local fingerprint comparison and does + not contact the AcoustID service. + """ if not a or not b: return 0.0 - n = min(len(a), len(b)) - if not n: - return 0.0 - # Chromaprint JSON fingerprints are comma-separated integer features. try: - aa, bb = [int(x) for x in a.split(",")], [int(x) for x in b.split(",")] - except ValueError: + import chromaprint as cp + aa = [int(x) for x in cp.decode_fingerprint(a.encode())[0]] + bb = [int(x) for x in cp.decode_fingerprint(b.encode())[0]] + except (ImportError, AttributeError, TypeError, ValueError): return 0.0 - n = min(len(aa), len(bb)) - if not n: + if not aa or not bb: return 0.0 - # Feature-wise equality is intentionally conservative. A future backend - # can replace this with a library-specific fingerprint distance. - return sum(x == y for x, y in zip(aa[:n], bb[:n])) / n + + counts = [0] * (len(aa) + len(bb) + 1) + for i, av in enumerate(aa): + for j in range(max(0, i - MAX_ALIGN_OFFSET), min(len(bb), i + MAX_ALIGN_OFFSET)): + if _popcount(av ^ bb[j]) <= MAX_BIT_ERROR: + counts[i - j + len(bb)] += 1 + return max(counts) / min(len(aa), len(bb)) diff --git a/am_deduper/cli.py b/am_deduper/cli.py index a6163e0..70c545b 100644 --- a/am_deduper/cli.py +++ b/am_deduper/cli.py @@ -1,18 +1,44 @@ from __future__ import annotations + import argparse from pathlib import Path + from .exact import exact_dedupe -from .scanner import scan from .l2 import acoustic_dedupe +from .scanner import scan from .watcher import watch -def main(): - p=argparse.ArgumentParser(prog='am-deduper'); sub=p.add_subparsers(dest='cmd',required=True) - for name in ('scan','dedupe','reconcile','watch'): - sp=sub.add_parser(name); sp.add_argument('root',type=Path); sp.add_argument('--db',type=Path,default=None) - a=p.parse_args(); db=a.db or a.root/'.dedupe'/'dedupe.db' - if a.cmd=='watch': watch(a.root,db); return - print(f'indexed={scan(a.root,db)} db={db}') - if a.cmd in ('dedupe','reconcile'): - ex=exact_dedupe(db,a.root); print(f'exact_hardlinked={len(ex)}') - l2=acoustic_dedupe(db,a.root); print(f'l2={l2}') + +def main() -> None: + parser = argparse.ArgumentParser(prog="am-deduper") + sub = parser.add_subparsers(dest="cmd", required=True) + + def add_common(sp): + sp.add_argument("root", type=Path) + sp.add_argument("--db", type=Path, default=None) + sp.add_argument("--dry-run", action="store_true", help="report actions without changing files") + + add_common(sub.add_parser("scan")) + add_common(sub.add_parser("dedupe")) + add_common(sub.add_parser("reconcile")) + watch_parser = sub.add_parser("watch") + watch_parser.add_argument("root", type=Path) + watch_parser.add_argument("--db", type=Path, default=None) + watch_parser.add_argument("--l2", action="store_true", help="enable acoustic matching in the watcher") + + args = parser.parse_args() + db = args.db or (Path("/data/dedupe.db") if args.cmd != "watch" and str(args.root).startswith("/music/") else args.root / ".dedupe" / "dedupe.db") + + if args.cmd == "watch": + watch(args.root, db, l2=args.l2) + return + + indexed = scan(args.root, db) + print(f"indexed={indexed} db={db}") + if args.cmd == "scan": + return + + exact = exact_dedupe(db, args.root, dry_run=args.dry_run) + print(f"exact_hardlinked={len(exact)} dry_run={args.dry_run}") + l2 = acoustic_dedupe(db, args.root, dry_run=args.dry_run) + print(f"l2={l2} dry_run={args.dry_run}") diff --git a/am_deduper/exact.py b/am_deduper/exact.py index 549bc3d..e86fa60 100644 --- a/am_deduper/exact.py +++ b/am_deduper/exact.py @@ -39,7 +39,7 @@ def choose_winner(rows): return min(rows,key=key) -def exact_dedupe(db_path: Path, root: Path) -> list[tuple[str,str]]: +def exact_dedupe(db_path: Path, root: Path, dry_run: bool = False) -> list[tuple[str,str]]: actions=[] with connect(db_path) as conn: rows=conn.execute("SELECT * FROM audio_tracks WHERE status IN ('active','hardlinked')").fetchall() @@ -62,7 +62,8 @@ def exact_dedupe(db_path: Path, root: Path) -> list[tuple[str,str]]: if r["id"]==winner["id"]: continue lp=Path(r["file_path"]) if not lp.exists(): continue - if lp.stat().st_ino != wp.stat().st_ino: hardlink_duplicate(lp,wp) - conn.execute("UPDATE audio_tracks SET status='hardlinked',updated_at=CURRENT_TIMESTAMP WHERE id=?",(r["id"],)) + if lp.stat().st_ino != wp.stat().st_ino and not dry_run: + hardlink_duplicate(lp,wp) + conn.execute("UPDATE audio_tracks SET status='hardlinked',updated_at=CURRENT_TIMESTAMP WHERE id=?",(r["id"],)) actions.append((str(lp),str(wp))) return actions diff --git a/am_deduper/l2.py b/am_deduper/l2.py index 7e6ca62..ad1679e 100644 --- a/am_deduper/l2.py +++ b/am_deduper/l2.py @@ -3,6 +3,7 @@ from __future__ import annotations from pathlib import Path from .acoustic import chromaprint, similarity from .arbitration import choose_winner, is_version_conflict, score +from .exact import hardlink_duplicate from .db import connect from .constants import QUARANTINE_SUBDIR, TRASH_SUBDIR @@ -22,7 +23,7 @@ def find_candidates(conn,row): return conn.execute("SELECT * FROM audio_tracks WHERE id!=? AND status IN ('active','hardlinked') AND duration IS NOT NULL AND duration BETWEEN ? AND ?", (row["id"],d-DURATION_DELTA,d+DURATION_DELTA)).fetchall() -def acoustic_dedupe(db_path: Path, root: Path) -> dict: +def acoustic_dedupe(db_path: Path, root: Path, dry_run: bool = False) -> dict: stats={"checked":0,"equivalent":0,"review":0,"unique":0,"errors":0} with connect(db_path) as conn: rows=conn.execute("SELECT * FROM audio_tracks WHERE status='active' ORDER BY id").fetchall() @@ -47,12 +48,11 @@ def acoustic_dedupe(db_path: Path, root: Path) -> dict: winner=choose_winner([row,c]); loser=c if winner["id"]==row["id"] else row lp=Path(loser["file_path"]); wp=Path(winner["file_path"]) if lp.exists() and wp.exists(): - trash=root/TRASH_SUBDIR - trash.mkdir(parents=True,exist_ok=True) - target=trash/lp.name - n=1 - while target.exists(): target=trash/f"{lp.stem}.{n}{lp.suffix}"; n+=1 - lp.replace(target) - conn.execute("UPDATE audio_tracks SET status='trashed',file_path=?,trashed_at=CURRENT_TIMESTAMP,review_status=?,updated_at=CURRENT_TIMESTAMP WHERE id=?",(str(target),f"equivalent={sim:.3f};winner={wp}",loser["id"])) + # Confirmed same-version acoustic match keeps the user's logical + # path. The physical data is shared via a hard link. + if lp.stat().st_ino != wp.stat().st_ino: + if not dry_run: + hardlink_duplicate(lp, wp) + conn.execute("UPDATE audio_tracks SET status='hardlinked',review_status=?,updated_at=CURRENT_TIMESTAMP WHERE id=?", (f"acoustic={sim:.3f};winner={wp}", loser["id"])) stats["equivalent"]+=1 return stats diff --git a/am_deduper/metadata.py b/am_deduper/metadata.py index 9fed84d..e6204f4 100644 --- a/am_deduper/metadata.py +++ b/am_deduper/metadata.py @@ -28,6 +28,6 @@ def read_metadata(path: Path) -> dict: "sample_rate": int(tag.samplerate) if tag.samplerate else None, "bit_depth": int(tag.bitdepth) if tag.bitdepth else None, }) - text = " ".join(str(data.get(k) or "") for k in ("title", "album", "file_name")) + text = " ".join(str(data.get(k) or "") for k in ("title", "album")) + " " + path.name data["version_flags"] = [m for m in VERSION_MARKERS if re.search(rf"(? None: path = Path(filename) - if path.is_symlink() or path.suffix.lower() not in MUSIC_EXTENSIONS: + if path.is_symlink() or path.suffix.lower() not in MUSIC_EXTENSIONS or ".dedupe" in path.parts: return try: previous = path.stat().st_size @@ -33,7 +36,11 @@ class Handler(FileSystemEventHandler): stable = 0 with connect(self.db_path) as conn: index_file(conn, path) - except (FileNotFoundError, PermissionError): + # Exact dedupe is byte-for-byte safe. Acoustic dedupe is opt-in. + exact_dedupe(self.db_path, self.root) + if self.l2: + acoustic_dedupe(self.db_path, self.root) + except (FileNotFoundError, PermissionError, OSError): return def on_created(self, event): @@ -45,9 +52,11 @@ class Handler(FileSystemEventHandler): self._process(event.dest_path) -def watch(root: Path, db_path: Path) -> None: + +def watch(root: Path, db_path: Path, l2: bool = False) -> None: + root = root.resolve() observer = Observer() - observer.schedule(Handler(root, db_path), str(root), recursive=True) + observer.schedule(Handler(root, db_path, l2=l2), str(root), recursive=True) observer.start() try: while True: diff --git a/docker-compose.yml b/docker-compose.yml index 86936e2..a5901d3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,6 @@ services: am-deduper: - build: . + image: ${AM_DEDUPER_IMAGE:?Set AM_DEDUPER_IMAGE to the pushed Gitea registry image} container_name: am-deduper restart: unless-stopped command: ["watch", "/music/AllMusic"] @@ -8,5 +8,6 @@ services: TZ: Asia/Shanghai PYTHONUNBUFFERED: "1" volumes: - - /your/nas/path/AllMusic:/music/AllMusic + - ${ALLMUSIC_PATH:?Set ALLMUSIC_PATH to the NAS AllMusic directory}:/music/AllMusic + - ${DEDUPE_DATA_PATH:?Set DEDUPE_DATA_PATH to the NAS app-data directory}:/data stop_grace_period: 30s diff --git a/pyproject.toml b/pyproject.toml index 6478b7e..8d44f98 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,6 +6,8 @@ requires-python = ">=3.10" dependencies = [ "tinytag>=2.1.1,<3", "watchdog>=6,<7", + "pyacoustid>=1.3,<2", + "pychromaprint>=1.2,<2", ] [project.optional-dependencies]