from __future__ import annotations

from pathlib import Path
import sys

ROOT = Path(__file__).resolve().parents[1]
SRC = ROOT / "src"
sys.path.insert(0, str(SRC))

from ai_archive.utils import dump_jsonl


def _read_jsonl(path: Path) -> list[dict]:
    if not path.exists():
        return []
    records: list[dict] = []
    with path.open("r", encoding="utf-8") as fh:
        for line in fh:
            if line.strip():
                records.append(__import__("json").loads(line))
    return records


def main() -> int:
    normalized = ROOT / "normalized"
    source_files = [normalized / "chatgpt_conversations.jsonl", normalized / "claude_conversations.jsonl"]
    conversations: list[dict] = []
    messages: list[dict] = []
    seen = set()
    for path in source_files:
        for convo in _read_jsonl(path):
            key = (convo.get("source"), convo.get("conversation_id"))
            if key in seen:
                continue
            seen.add(key)
            conversations.append(convo)
            for msg in convo.get("messages") or []:
                messages.append({
                    "message_id": msg.get("message_id"),
                    "conversation_id": convo.get("conversation_id"),
                    "source": convo.get("source"),
                    "role": msg.get("role"),
                    "content": msg.get("content"),
                    "created_at": msg.get("created_at"),
                    "metadata": msg.get("metadata", {}),
                })
    dump_jsonl(normalized / "conversations.jsonl", conversations)
    dump_jsonl(normalized / "messages.jsonl", messages)
    print(f"wrote {len(conversations)} conversations and {len(messages)} messages")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())