#!/usr/bin/env python3
import argparse
import json
from pathlib import Path


def iter_jsonl(path):
    with Path(path).open(encoding="utf-8") as handle:
        for line in handle:
            if line.strip():
                yield json.loads(line)


def dump(obj):
    return json.dumps(obj, ensure_ascii=False, separators=(",", ":"))


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--inputs", nargs="+", required=True)
    parser.add_argument("--output", required=True)
    parser.add_argument("--report", required=True)
    args = parser.parse_args()

    output = Path(args.output)
    report_path = Path(args.report)
    output.parent.mkdir(parents=True, exist_ok=True)
    report_path.parent.mkdir(parents=True, exist_ok=True)

    seen = {}
    duplicates = []
    validation_problem_count = 0
    source_counts = {}
    for path_str in args.inputs:
        path = Path(path_str)
        if not path.exists():
            continue
        source_counts[str(path)] = 0
        for obj in iter_jsonl(path):
            key = obj.get("cache_key") or obj.get("response_id")
            source_counts[str(path)] += 1
            if key in seen:
                duplicates.append(key)
                continue
            if obj.get("validation_problems"):
                validation_problem_count += 1
            seen[key] = obj

    with output.open("w", encoding="utf-8") as out:
        for obj in sorted(seen.values(), key=lambda x: (x.get("report_id", ""), x.get("persona_id", ""))):
            out.write(dump(obj) + "\n")

    report = {
        "inputs": args.inputs,
        "output": str(output),
        "source_counts": source_counts,
        "merged_count": len(seen),
        "duplicate_count": len(duplicates),
        "duplicates_sample": duplicates[:20],
        "records_with_validation_problems": validation_problem_count,
    }
    report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
    print(json.dumps(report, ensure_ascii=False, indent=2))


if __name__ == "__main__":
    main()