from __future__ import annotations import os from pathlib import Path from .db import connect from .hashing import sha256 def confirm_sha256(conn, row) -> str | None: path = Path(row["file_path"]) try: st = path.stat() except FileNotFoundError: return None if st.st_size != row["file_size"]: return None # If the file changed after indexing, refresh the fast hash first and refuse # this pass. The next incremental/reconcile pass will safely retry it. digest = sha256(path) conn.execute( "UPDATE audio_tracks SET sha256=?, updated_at=CURRENT_TIMESTAMP WHERE id=?", (digest, row["id"]), ) return digest def hardlink_duplicate(conn, loser: Path, winner: Path) -> None: """Make loser a hard link to winner, atomically replacing loser.""" tmp = loser.with_name(loser.name + ".dedupe-linking") if tmp.exists(): tmp.unlink() os.link(winner, tmp) os.replace(tmp, loser) def choose_winner(rows): """Prefer public admin paths, then older records, then lexical path.""" def key(r): path = r["file_path"] is_admin = "/admin/" in path or path.endswith("/admin") return (0 if is_admin else 1, r["created_at"], path) return min(rows, key=key) def exact_dedupe(db_path: Path, root: Path) -> list[tuple[str, str]]: """Hard-link byte-identical files while preserving every logical user path.""" actions = [] with connect(db_path) as conn: candidates = conn.execute( """ SELECT * FROM audio_tracks WHERE status='active' ORDER BY id """ ).fetchall() # Only full-hash files that share the cheap L1 key need to be compared. groups = {} for row in candidates: groups.setdefault((row["file_size"], row["fast_hash"]), []).append(row) for rows in groups.values(): if len(rows) < 2: # L1 has no collision candidate. Do not read the whole file; # the full SHA-256 will be computed only when a future file # lands in the same L1 bucket. continue for row in rows: if row["sha256"] is None: confirm_sha256(conn, row) refreshed = conn.execute( "SELECT * FROM audio_tracks WHERE id IN (%s)" % ",".join("?" * len(rows)), [r["id"] for r in rows], ).fetchall() by_hash = {} for row in refreshed: if row["sha256"]: by_hash.setdefault(row["sha256"], []).append(row) for same in by_hash.values(): if len(same) < 2: continue winner = choose_winner(same) winpath = Path(winner["file_path"]) if not winpath.exists(): continue for row in same: if row["id"] == winner["id"]: continue loser = Path(row["file_path"]) if not loser.exists(): continue # Preserve the user's logical path. Exact duplicates are NOT # moved to trash: all users should continue seeing/downloading # their own path, while the inode/data is shared. if loser.stat().st_ino != winpath.stat().st_ino: hardlink_duplicate(conn, loser, winpath) conn.execute( "UPDATE audio_tracks SET status='hardlinked', updated_at=CURRENT_TIMESTAMP WHERE id=?", (row["id"],), ) actions.append((str(loser), str(winpath))) return actions