feat: bootstrap NAS music deduper

.dockerignore

@@ -0,0 +1,5 @@
+.git
+.dedupe
+__pycache__
+.pytest_cache
+.venv

.gitignore

@@ -0,0 +1,5 @@
+.dedupe/
+*.db
+__pycache__/
+.pytest_cache/
+.venv/

Dockerfile

@@ -0,0 +1,12 @@
+FROM python:3.12-slim
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends chromaprint-tools \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+COPY pyproject.toml README.md ./
+COPY am_deduper ./am_deduper
+RUN pip install --no-cache-dir .
+
+ENTRYPOINT ["am-deduper"]

README.md

@@ -0,0 +1,39 @@
+# AM Music Deduper
+
+轻量级家庭 NAS 音乐库去重服务，面向 `AllMusic/{username}/` 场景。
+
+## 设计目标
+
+- 本地 SQLite 元数据库
+- L1 极速候选：文件大小 + 头/尾 64 KiB hash
+- Exact Duplicate 最终使用完整 SHA-256 确认
+- L2 使用 Chromaprint 做音频相似度候选
+- duration 预过滤，降低 CPU/I/O
+- 同一版本才允许自动择优
+- loser 进入 `.dedupe/trash/`，默认保留 30 天
+- 不使用软链接；物理去重使用 hard link
+- inotify 增量处理 + 低优先级全量 reconcile
+- 任何不确定或异常默认保留原文件
+
+## 当前阶段
+
+第一阶段先完成安全的扫描、SQLite 索引、L1/Exact 去重、trash/quarantine 和测试。
+L2 Chromaprint 比对与生产 NAS 监听在后续阶段接入。
+
+## 安全原则
+
+1. 不修改正在写入的文件。
+2. L1 命中必须再做完整 SHA-256 才能判定 exact duplicate。
+3. 不根据歌名/歌手/专辑单独判重。
+4. 不同格式或音频规格不会因为 metadata 相似而直接合并。
+5. 自动处理前保留可恢复路径；失败不删除原文件。
+6. 数据库只是索引，文件系统才是最终事实。
+
+## CLI 预期
+
+```bash
+am-deduper scan /music/AllMusic
+am-deduper dedupe /music/AllMusic
+am-deduper reconcile /music/AllMusic
+am-deduper trash-prune /music/AllMusic --days 30
+```

am_deduper/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

am_deduper/cli.py

@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+
+from .exact import exact_dedupe
+from .scanner import scan
+from .watcher import watch
+
+
+def main() -> None:
+    p = argparse.ArgumentParser(prog="am-deduper")
+    sub = p.add_subparsers(dest="cmd", required=True)
+    for name in ("scan", "dedupe", "reconcile", "watch"):
+        sp = sub.add_parser(name)
+        sp.add_argument("root", type=Path)
+        sp.add_argument("--db", type=Path, default=None)
+    args = p.parse_args()
+    db = args.db or (args.root / ".dedupe" / "dedupe.db")
+    if args.cmd == "watch":
+        watch(args.root, db)
+        return
+    if args.cmd in {"scan", "reconcile"}:
+        print(f"indexed={scan(args.root, db)} db={db}")
+    if args.cmd == "dedupe":
+        scan(args.root, db)
+        actions = exact_dedupe(db, args.root)
+        print(f"hardlinked={len(actions)}")

am_deduper/constants.py

@@ -0,0 +1,11 @@
+from pathlib import Path
+
+MUSIC_EXTENSIONS = frozenset({
+    ".flac", ".mp3", ".m4a", ".aac", ".ogg", ".opus", ".wav", ".alac",
+})
+FAST_HASH_CHUNK = 64 * 1024
+STABILITY_CHECKS = 2
+STABILITY_INTERVAL_SECONDS = 2
+TRASH_DIRNAME = ".dedupe"
+TRASH_SUBDIR = "trash"
+QUARANTINE_SUBDIR = "quarantine"

am_deduper/db.py

@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+import sqlite3
+from pathlib import Path
+
+SCHEMA = """
+CREATE TABLE IF NOT EXISTS audio_tracks (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    file_path TEXT UNIQUE NOT NULL,
+    file_size INTEGER NOT NULL,
+    fast_hash TEXT NOT NULL,
+    sha256 TEXT,
+    duration REAL,
+    artist TEXT,
+    title TEXT,
+    album TEXT,
+    format TEXT,
+    bitrate INTEGER,
+    sample_rate INTEGER,
+    bit_depth INTEGER,
+    chromaprint TEXT,
+    score INTEGER,
+    status TEXT NOT NULL DEFAULT 'active',
+    inode INTEGER,
+    device INTEGER,
+    created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    trashed_at TEXT,
+    review_status TEXT
+);
+CREATE INDEX IF NOT EXISTS idx_fast_hash ON audio_tracks(fast_hash);
+CREATE INDEX IF NOT EXISTS idx_sha256 ON audio_tracks(sha256);
+CREATE INDEX IF NOT EXISTS idx_duration ON audio_tracks(duration);
+CREATE INDEX IF NOT EXISTS idx_status ON audio_tracks(status);
+"""
+
+
+def connect(path: Path) -> sqlite3.Connection:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    conn = sqlite3.connect(path)
+    conn.row_factory = sqlite3.Row
+    conn.execute("PRAGMA journal_mode=WAL")
+    conn.execute("PRAGMA foreign_keys=ON")
+    conn.executescript(SCHEMA)
+    return conn

am_deduper/exact.py

@@ -0,0 +1,101 @@
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+from .db import connect
+from .hashing import sha256
+
+
+def confirm_sha256(conn, row) -> str | None:
+    path = Path(row["file_path"])
+    try:
+        st = path.stat()
+    except FileNotFoundError:
+        return None
+    if st.st_size != row["file_size"]:
+        return None
+    # If the file changed after indexing, refresh the fast hash first and refuse
+    # this pass. The next incremental/reconcile pass will safely retry it.
+    digest = sha256(path)
+    conn.execute(
+        "UPDATE audio_tracks SET sha256=?, updated_at=CURRENT_TIMESTAMP WHERE id=?",
+        (digest, row["id"]),
+    )
+    return digest
+
+
+def hardlink_duplicate(conn, loser: Path, winner: Path) -> None:
+    """Make loser a hard link to winner, atomically replacing loser."""
+    tmp = loser.with_name(loser.name + ".dedupe-linking")
+    if tmp.exists():
+        tmp.unlink()
+    os.link(winner, tmp)
+    os.replace(tmp, loser)
+
+
+def choose_winner(rows):
+    """Prefer public admin paths, then older records, then lexical path."""
+    def key(r):
+        path = r["file_path"]
+        is_admin = "/admin/" in path or path.endswith("/admin")
+        return (0 if is_admin else 1, r["created_at"], path)
+    return min(rows, key=key)
+
+
+def exact_dedupe(db_path: Path, root: Path) -> list[tuple[str, str]]:
+    """Hard-link byte-identical files while preserving every logical user path."""
+    actions = []
+    with connect(db_path) as conn:
+        candidates = conn.execute(
+            """
+            SELECT * FROM audio_tracks
+            WHERE status='active'
+            ORDER BY id
+            """
+        ).fetchall()
+        # Only full-hash files that share the cheap L1 key need to be compared.
+        groups = {}
+        for row in candidates:
+            groups.setdefault((row["file_size"], row["fast_hash"]), []).append(row)
+        for rows in groups.values():
+            if len(rows) < 2:
+                # L1 has no collision candidate. Do not read the whole file;
+                # the full SHA-256 will be computed only when a future file
+                # lands in the same L1 bucket.
+                continue
+            for row in rows:
+                if row["sha256"] is None:
+                    confirm_sha256(conn, row)
+            refreshed = conn.execute(
+                "SELECT * FROM audio_tracks WHERE id IN (%s)" % ",".join("?" * len(rows)),
+                [r["id"] for r in rows],
+            ).fetchall()
+            by_hash = {}
+            for row in refreshed:
+                if row["sha256"]:
+                    by_hash.setdefault(row["sha256"], []).append(row)
+            for same in by_hash.values():
+                if len(same) < 2:
+                    continue
+                winner = choose_winner(same)
+                winpath = Path(winner["file_path"])
+                if not winpath.exists():
+                    continue
+                for row in same:
+                    if row["id"] == winner["id"]:
+                        continue
+                    loser = Path(row["file_path"])
+                    if not loser.exists():
+                        continue
+                    # Preserve the user's logical path. Exact duplicates are NOT
+                    # moved to trash: all users should continue seeing/downloading
+                    # their own path, while the inode/data is shared.
+                    if loser.stat().st_ino != winpath.stat().st_ino:
+                        hardlink_duplicate(conn, loser, winpath)
+                    conn.execute(
+                        "UPDATE audio_tracks SET status='hardlinked', updated_at=CURRENT_TIMESTAMP WHERE id=?",
+                        (row["id"],),
+                    )
+                    actions.append((str(loser), str(winpath)))
+    return actions

am_deduper/hashing.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+import hashlib
+from pathlib import Path
+
+from .constants import FAST_HASH_CHUNK
+
+
+def fast_hash(path: Path) -> str:
+    """Hash size + first/last 64 KiB; used only as a candidate key."""
+    size = path.stat().st_size
+    with path.open("rb") as f:
+        head = f.read(FAST_HASH_CHUNK)
+        if size > FAST_HASH_CHUNK:
+            f.seek(max(0, size - FAST_HASH_CHUNK))
+        tail = f.read(FAST_HASH_CHUNK)
+    h = hashlib.sha256()
+    h.update(str(size).encode())
+    h.update(b"\0")
+    h.update(head)
+    h.update(b"\0")
+    h.update(tail)
+    return h.hexdigest()
+
+
+def sha256(path: Path, chunk_size: int = 1024 * 1024) -> str:
+    h = hashlib.sha256()
+    with path.open("rb") as f:
+        while True:
+            chunk = f.read(chunk_size)
+            if not chunk:
+                break
+            h.update(chunk)
+    return h.hexdigest()

am_deduper/metadata.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+try:
+    from tinytag import TinyTag
+except ImportError:  # optional until metadata phase is invoked
+    TinyTag = None
+
+
+def read_metadata(path: Path) -> dict:
+    if TinyTag is None:
+        return {}
+    tag = TinyTag.get(str(path), image=False)
+    return {
+        "duration": tag.duration,
+        "artist": tag.artist,
+        "title": tag.title,
+        "album": tag.album,
+        "format": path.suffix.lower().lstrip("."),
+        "bitrate": int(tag.bitrate) if tag.bitrate else None,
+        "sample_rate": int(tag.samplerate) if tag.samplerate else None,
+        "bit_depth": int(tag.bitdepth) if tag.bitdepth else None,
+    }

am_deduper/scanner.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Iterator
+
+from .constants import MUSIC_EXTENSIONS
+from .db import connect
+from .hashing import fast_hash
+
+
+def iter_music(root: Path) -> Iterator[Path]:
+    for path in root.rglob("*"):
+        if not path.is_file() or path.is_symlink():
+            continue
+        if path.name.startswith(".") or ".dedupe" in path.parts:
+            continue
+        if path.suffix.lower() in MUSIC_EXTENSIONS:
+            yield path
+
+
+def index_file(conn, path: Path) -> None:
+    st = path.stat()
+    fh = fast_hash(path)
+    conn.execute(
+        """
+        INSERT INTO audio_tracks(file_path, file_size, fast_hash, inode, device, status)
+        VALUES (?, ?, ?, ?, ?, 'active')
+        ON CONFLICT(file_path) DO UPDATE SET
+          file_size=excluded.file_size,
+          fast_hash=excluded.fast_hash,
+          inode=excluded.inode,
+          device=excluded.device,
+          updated_at=CURRENT_TIMESTAMP,
+          status='active'
+        """,
+        (str(path), st.st_size, fh, st.st_ino, st.st_dev),
+    )
+
+
+def scan(root: Path, db_path: Path) -> int:
+    count = 0
+    with connect(db_path) as conn:
+        for path in iter_music(root):
+            index_file(conn, path)
+            count += 1
+    return count

am_deduper/watcher.py

@@ -0,0 +1,57 @@
+from __future__ import annotations
+
+import time
+from pathlib import Path
+
+from watchdog.events import FileSystemEventHandler
+from watchdog.observers import Observer
+
+from .constants import MUSIC_EXTENSIONS, STABILITY_CHECKS, STABILITY_INTERVAL_SECONDS
+from .db import connect
+from .scanner import index_file
+
+
+class Handler(FileSystemEventHandler):
+    def __init__(self, root: Path, db_path: Path):
+        self.root = root
+        self.db_path = db_path
+
+    def _process(self, filename: str) -> None:
+        path = Path(filename)
+        if path.is_symlink() or path.suffix.lower() not in MUSIC_EXTENSIONS:
+            return
+        try:
+            previous = path.stat().st_size
+            stable = 0
+            while stable < STABILITY_CHECKS - 1:
+                time.sleep(STABILITY_INTERVAL_SECONDS)
+                size = path.stat().st_size
+                if size == previous:
+                    stable += 1
+                else:
+                    previous = size
+                    stable = 0
+            with connect(self.db_path) as conn:
+                index_file(conn, path)
+        except (FileNotFoundError, PermissionError):
+            return
+
+    def on_created(self, event):
+        if not event.is_directory:
+            self._process(event.src_path)
+
+    def on_moved(self, event):
+        if not event.is_directory:
+            self._process(event.dest_path)
+
+
+def watch(root: Path, db_path: Path) -> None:
+    observer = Observer()
+    observer.schedule(Handler(root, db_path), str(root), recursive=True)
+    observer.start()
+    try:
+        while True:
+            time.sleep(1)
+    finally:
+        observer.stop()
+        observer.join()

docker-compose.yml

@@ -0,0 +1,11 @@
+services:
+  am-deduper:
+    build: .
+    container_name: am-deduper
+    restart: unless-stopped
+    command: ["watch", "/music/AllMusic"]
+    environment:
+      TZ: Asia/Shanghai
+    volumes:
+      - /your/nas/path/AllMusic:/music/AllMusic
+      - ./data:/music/AllMusic/.dedupe

pyproject.toml

@@ -0,0 +1,18 @@
+[project]
+name = "am-music-deduper"
+version = "0.1.0"
+description = "Lightweight NAS music deduplication service for AllMusic"
+requires-python = ">=3.10"
+dependencies = [
+  "tinytag>=2.1.1,<3",
+  "watchdog>=6,<7",
+]
+
+[project.optional-dependencies]
+test = ["pytest>=8,<9"]
+
+[project.scripts]
+am-deduper = "am_deduper.cli:main"
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]

tests/test_exact.py

@@ -0,0 +1,22 @@
+from pathlib import Path
+
+from am_deduper.db import connect
+from am_deduper.exact import exact_dedupe
+from am_deduper.scanner import scan
+
+
+def test_exact_dedupe_creates_hardlink(tmp_path: Path):
+    root = tmp_path / "AllMusic"
+    (root / "admin").mkdir(parents=True)
+    (root / "alice").mkdir(parents=True)
+    a = root / "admin" / "song.flac"
+    b = root / "alice" / "song.flac"
+    data = b"fake flac content" * 10000
+    a.write_bytes(data)
+    b.write_bytes(data)
+    db = root / ".dedupe" / "dedupe.db"
+    assert scan(root, db) == 2
+    actions = exact_dedupe(db, root)
+    assert len(actions) == 1
+    assert a.stat().st_ino == b.stat().st_ino
+    assert a.read_bytes() == b.read_bytes()

tests/test_hashing.py

@@ -0,0 +1,21 @@
+from pathlib import Path
+
+from am_deduper.hashing import fast_hash, sha256
+
+
+def test_fast_hash_same_content(tmp_path: Path):
+    a = tmp_path / "a.bin"
+    b = tmp_path / "b.bin"
+    data = b"x" * 200_000
+    a.write_bytes(data)
+    b.write_bytes(data)
+    assert fast_hash(a) == fast_hash(b)
+    assert sha256(a) == sha256(b)
+
+
+def test_fast_hash_uses_size(tmp_path: Path):
+    a = tmp_path / "a.bin"
+    b = tmp_path / "b.bin"
+    a.write_bytes(b"x" * 100)
+    b.write_bytes(b"x" * 101)
+    assert fast_hash(a) != fast_hash(b)