feat: bootstrap NAS music deduper
This commit is contained in:
OpenClaw Agent
@@ -0,0 +1 @@
|
||||
__version__ = "0.1.0"
|
||||
@@ -0,0 +1,28 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
from .exact import exact_dedupe
|
||||
from .scanner import scan
|
||||
from .watcher import watch
|
||||
|
||||
|
||||
def main() -> None:
|
||||
p = argparse.ArgumentParser(prog="am-deduper")
|
||||
sub = p.add_subparsers(dest="cmd", required=True)
|
||||
for name in ("scan", "dedupe", "reconcile", "watch"):
|
||||
sp = sub.add_parser(name)
|
||||
sp.add_argument("root", type=Path)
|
||||
sp.add_argument("--db", type=Path, default=None)
|
||||
args = p.parse_args()
|
||||
db = args.db or (args.root / ".dedupe" / "dedupe.db")
|
||||
if args.cmd == "watch":
|
||||
watch(args.root, db)
|
||||
return
|
||||
if args.cmd in {"scan", "reconcile"}:
|
||||
print(f"indexed={scan(args.root, db)} db={db}")
|
||||
if args.cmd == "dedupe":
|
||||
scan(args.root, db)
|
||||
actions = exact_dedupe(db, args.root)
|
||||
print(f"hardlinked={len(actions)}")
|
||||
@@ -0,0 +1,11 @@
|
||||
from pathlib import Path
|
||||
|
||||
MUSIC_EXTENSIONS = frozenset({
|
||||
".flac", ".mp3", ".m4a", ".aac", ".ogg", ".opus", ".wav", ".alac",
|
||||
})
|
||||
FAST_HASH_CHUNK = 64 * 1024
|
||||
STABILITY_CHECKS = 2
|
||||
STABILITY_INTERVAL_SECONDS = 2
|
||||
TRASH_DIRNAME = ".dedupe"
|
||||
TRASH_SUBDIR = "trash"
|
||||
QUARANTINE_SUBDIR = "quarantine"
|
||||
@@ -0,0 +1,45 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS audio_tracks (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
file_path TEXT UNIQUE NOT NULL,
|
||||
file_size INTEGER NOT NULL,
|
||||
fast_hash TEXT NOT NULL,
|
||||
sha256 TEXT,
|
||||
duration REAL,
|
||||
artist TEXT,
|
||||
title TEXT,
|
||||
album TEXT,
|
||||
format TEXT,
|
||||
bitrate INTEGER,
|
||||
sample_rate INTEGER,
|
||||
bit_depth INTEGER,
|
||||
chromaprint TEXT,
|
||||
score INTEGER,
|
||||
status TEXT NOT NULL DEFAULT 'active',
|
||||
inode INTEGER,
|
||||
device INTEGER,
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
trashed_at TEXT,
|
||||
review_status TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_fast_hash ON audio_tracks(fast_hash);
|
||||
CREATE INDEX IF NOT EXISTS idx_sha256 ON audio_tracks(sha256);
|
||||
CREATE INDEX IF NOT EXISTS idx_duration ON audio_tracks(duration);
|
||||
CREATE INDEX IF NOT EXISTS idx_status ON audio_tracks(status);
|
||||
"""
|
||||
|
||||
|
||||
def connect(path: Path) -> sqlite3.Connection:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
conn = sqlite3.connect(path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
conn.execute("PRAGMA foreign_keys=ON")
|
||||
conn.executescript(SCHEMA)
|
||||
return conn
|
||||
@@ -0,0 +1,101 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from .db import connect
|
||||
from .hashing import sha256
|
||||
|
||||
|
||||
def confirm_sha256(conn, row) -> str | None:
|
||||
path = Path(row["file_path"])
|
||||
try:
|
||||
st = path.stat()
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
if st.st_size != row["file_size"]:
|
||||
return None
|
||||
# If the file changed after indexing, refresh the fast hash first and refuse
|
||||
# this pass. The next incremental/reconcile pass will safely retry it.
|
||||
digest = sha256(path)
|
||||
conn.execute(
|
||||
"UPDATE audio_tracks SET sha256=?, updated_at=CURRENT_TIMESTAMP WHERE id=?",
|
||||
(digest, row["id"]),
|
||||
)
|
||||
return digest
|
||||
|
||||
|
||||
def hardlink_duplicate(conn, loser: Path, winner: Path) -> None:
|
||||
"""Make loser a hard link to winner, atomically replacing loser."""
|
||||
tmp = loser.with_name(loser.name + ".dedupe-linking")
|
||||
if tmp.exists():
|
||||
tmp.unlink()
|
||||
os.link(winner, tmp)
|
||||
os.replace(tmp, loser)
|
||||
|
||||
|
||||
def choose_winner(rows):
|
||||
"""Prefer public admin paths, then older records, then lexical path."""
|
||||
def key(r):
|
||||
path = r["file_path"]
|
||||
is_admin = "/admin/" in path or path.endswith("/admin")
|
||||
return (0 if is_admin else 1, r["created_at"], path)
|
||||
return min(rows, key=key)
|
||||
|
||||
|
||||
def exact_dedupe(db_path: Path, root: Path) -> list[tuple[str, str]]:
|
||||
"""Hard-link byte-identical files while preserving every logical user path."""
|
||||
actions = []
|
||||
with connect(db_path) as conn:
|
||||
candidates = conn.execute(
|
||||
"""
|
||||
SELECT * FROM audio_tracks
|
||||
WHERE status='active'
|
||||
ORDER BY id
|
||||
"""
|
||||
).fetchall()
|
||||
# Only full-hash files that share the cheap L1 key need to be compared.
|
||||
groups = {}
|
||||
for row in candidates:
|
||||
groups.setdefault((row["file_size"], row["fast_hash"]), []).append(row)
|
||||
for rows in groups.values():
|
||||
if len(rows) < 2:
|
||||
# L1 has no collision candidate. Do not read the whole file;
|
||||
# the full SHA-256 will be computed only when a future file
|
||||
# lands in the same L1 bucket.
|
||||
continue
|
||||
for row in rows:
|
||||
if row["sha256"] is None:
|
||||
confirm_sha256(conn, row)
|
||||
refreshed = conn.execute(
|
||||
"SELECT * FROM audio_tracks WHERE id IN (%s)" % ",".join("?" * len(rows)),
|
||||
[r["id"] for r in rows],
|
||||
).fetchall()
|
||||
by_hash = {}
|
||||
for row in refreshed:
|
||||
if row["sha256"]:
|
||||
by_hash.setdefault(row["sha256"], []).append(row)
|
||||
for same in by_hash.values():
|
||||
if len(same) < 2:
|
||||
continue
|
||||
winner = choose_winner(same)
|
||||
winpath = Path(winner["file_path"])
|
||||
if not winpath.exists():
|
||||
continue
|
||||
for row in same:
|
||||
if row["id"] == winner["id"]:
|
||||
continue
|
||||
loser = Path(row["file_path"])
|
||||
if not loser.exists():
|
||||
continue
|
||||
# Preserve the user's logical path. Exact duplicates are NOT
|
||||
# moved to trash: all users should continue seeing/downloading
|
||||
# their own path, while the inode/data is shared.
|
||||
if loser.stat().st_ino != winpath.stat().st_ino:
|
||||
hardlink_duplicate(conn, loser, winpath)
|
||||
conn.execute(
|
||||
"UPDATE audio_tracks SET status='hardlinked', updated_at=CURRENT_TIMESTAMP WHERE id=?",
|
||||
(row["id"],),
|
||||
)
|
||||
actions.append((str(loser), str(winpath)))
|
||||
return actions
|
||||
@@ -0,0 +1,34 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
|
||||
from .constants import FAST_HASH_CHUNK
|
||||
|
||||
|
||||
def fast_hash(path: Path) -> str:
|
||||
"""Hash size + first/last 64 KiB; used only as a candidate key."""
|
||||
size = path.stat().st_size
|
||||
with path.open("rb") as f:
|
||||
head = f.read(FAST_HASH_CHUNK)
|
||||
if size > FAST_HASH_CHUNK:
|
||||
f.seek(max(0, size - FAST_HASH_CHUNK))
|
||||
tail = f.read(FAST_HASH_CHUNK)
|
||||
h = hashlib.sha256()
|
||||
h.update(str(size).encode())
|
||||
h.update(b"\0")
|
||||
h.update(head)
|
||||
h.update(b"\0")
|
||||
h.update(tail)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def sha256(path: Path, chunk_size: int = 1024 * 1024) -> str:
|
||||
h = hashlib.sha256()
|
||||
with path.open("rb") as f:
|
||||
while True:
|
||||
chunk = f.read(chunk_size)
|
||||
if not chunk:
|
||||
break
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
@@ -0,0 +1,24 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from tinytag import TinyTag
|
||||
except ImportError: # optional until metadata phase is invoked
|
||||
TinyTag = None
|
||||
|
||||
|
||||
def read_metadata(path: Path) -> dict:
|
||||
if TinyTag is None:
|
||||
return {}
|
||||
tag = TinyTag.get(str(path), image=False)
|
||||
return {
|
||||
"duration": tag.duration,
|
||||
"artist": tag.artist,
|
||||
"title": tag.title,
|
||||
"album": tag.album,
|
||||
"format": path.suffix.lower().lstrip("."),
|
||||
"bitrate": int(tag.bitrate) if tag.bitrate else None,
|
||||
"sample_rate": int(tag.samplerate) if tag.samplerate else None,
|
||||
"bit_depth": int(tag.bitdepth) if tag.bitdepth else None,
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Iterator
|
||||
|
||||
from .constants import MUSIC_EXTENSIONS
|
||||
from .db import connect
|
||||
from .hashing import fast_hash
|
||||
|
||||
|
||||
def iter_music(root: Path) -> Iterator[Path]:
|
||||
for path in root.rglob("*"):
|
||||
if not path.is_file() or path.is_symlink():
|
||||
continue
|
||||
if path.name.startswith(".") or ".dedupe" in path.parts:
|
||||
continue
|
||||
if path.suffix.lower() in MUSIC_EXTENSIONS:
|
||||
yield path
|
||||
|
||||
|
||||
def index_file(conn, path: Path) -> None:
|
||||
st = path.stat()
|
||||
fh = fast_hash(path)
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO audio_tracks(file_path, file_size, fast_hash, inode, device, status)
|
||||
VALUES (?, ?, ?, ?, ?, 'active')
|
||||
ON CONFLICT(file_path) DO UPDATE SET
|
||||
file_size=excluded.file_size,
|
||||
fast_hash=excluded.fast_hash,
|
||||
inode=excluded.inode,
|
||||
device=excluded.device,
|
||||
updated_at=CURRENT_TIMESTAMP,
|
||||
status='active'
|
||||
""",
|
||||
(str(path), st.st_size, fh, st.st_ino, st.st_dev),
|
||||
)
|
||||
|
||||
|
||||
def scan(root: Path, db_path: Path) -> int:
|
||||
count = 0
|
||||
with connect(db_path) as conn:
|
||||
for path in iter_music(root):
|
||||
index_file(conn, path)
|
||||
count += 1
|
||||
return count
|
||||
@@ -0,0 +1,57 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from watchdog.events import FileSystemEventHandler
|
||||
from watchdog.observers import Observer
|
||||
|
||||
from .constants import MUSIC_EXTENSIONS, STABILITY_CHECKS, STABILITY_INTERVAL_SECONDS
|
||||
from .db import connect
|
||||
from .scanner import index_file
|
||||
|
||||
|
||||
class Handler(FileSystemEventHandler):
|
||||
def __init__(self, root: Path, db_path: Path):
|
||||
self.root = root
|
||||
self.db_path = db_path
|
||||
|
||||
def _process(self, filename: str) -> None:
|
||||
path = Path(filename)
|
||||
if path.is_symlink() or path.suffix.lower() not in MUSIC_EXTENSIONS:
|
||||
return
|
||||
try:
|
||||
previous = path.stat().st_size
|
||||
stable = 0
|
||||
while stable < STABILITY_CHECKS - 1:
|
||||
time.sleep(STABILITY_INTERVAL_SECONDS)
|
||||
size = path.stat().st_size
|
||||
if size == previous:
|
||||
stable += 1
|
||||
else:
|
||||
previous = size
|
||||
stable = 0
|
||||
with connect(self.db_path) as conn:
|
||||
index_file(conn, path)
|
||||
except (FileNotFoundError, PermissionError):
|
||||
return
|
||||
|
||||
def on_created(self, event):
|
||||
if not event.is_directory:
|
||||
self._process(event.src_path)
|
||||
|
||||
def on_moved(self, event):
|
||||
if not event.is_directory:
|
||||
self._process(event.dest_path)
|
||||
|
||||
|
||||
def watch(root: Path, db_path: Path) -> None:
|
||||
observer = Observer()
|
||||
observer.schedule(Handler(root, db_path), str(root), recursive=True)
|
||||
observer.start()
|
||||
try:
|
||||
while True:
|
||||
time.sleep(1)
|
||||
finally:
|
||||
observer.stop()
|
||||
observer.join()
|
||||
Reference in New Issue
Block a user