#!/usr/bin/env python3 """CSV deduplication CLI — configurable key columns, summary report.""" from __future__ import annotations import argparse import csv import sys from pathlib import Path def dedup(rows: list[dict], keys: list[str]) -> tuple[list[dict], int]: seen: set[tuple] = set() out: list[dict] = [] removed = 0 for row in rows: sig = tuple(row.get(k, "") for k in keys) if sig in seen: removed += 1 continue seen.add(sig) out.append(row) return out, removed def main() -> int: p = argparse.ArgumentParser( description="Deduplicate CSV rows by configurable column keys and write a summary." ) p.add_argument("input", type=Path, help="Input CSV path") p.add_argument("-o", "--output", type=Path, required=True, help="Output CSV path") p.add_argument( "-k", "--keys", required=True, help="Comma-separated column names used as the dedup key", ) p.add_argument( "--summary", type=Path, default=None, help="Optional path for a text summary report (default: stdout)", ) args = p.parse_args() keys = [k.strip() for k in args.keys.split(",") if k.strip()] if not keys: print("error: at least one key column required", file=sys.stderr) return 2 with args.input.open(newline="", encoding="utf-8") as f: reader = csv.DictReader(f) if not reader.fieldnames: print("error: empty or headerless CSV", file=sys.stderr) return 2 missing = [k for k in keys if k not in reader.fieldnames] if missing: print(f"error: missing columns: {missing}", file=sys.stderr) return 2 rows = list(reader) kept, removed = dedup(rows, keys) args.output.parent.mkdir(parents=True, exist_ok=True) with args.output.open("w", newline="", encoding="utf-8") as f: writer = csv.DictWriter(f, fieldnames=rows[0].keys() if rows else keys) writer.writeheader() writer.writerows(kept) report = ( f"input_rows={len(rows)}\n" f"output_rows={len(kept)}\n" f"removed_duplicates={removed}\n" f"key_columns={','.join(keys)}\n" f"output_file={args.output}\n" ) if args.summary: args.summary.write_text(report, encoding="utf-8") print(report, end="") return 0 if __name__ == "__main__": raise SystemExit(main())