from __future__ import annotations

from pathlib import Path
import sys

ROOT = Path(__file__).resolve().parents[1]
SRC = ROOT / "src"
sys.path.insert(0, str(SRC))

from ai_archive.chunking import chunk_conversation
from ai_archive.utils import dump_jsonl


def _read_jsonl(path: Path) -> list[dict]:
    if not path.exists():
        return []
    records: list[dict] = []
    with path.open("r", encoding="utf-8") as fh:
        for line in fh:
            if line.strip():
                records.append(__import__("json").loads(line))
    return records


def main() -> int:
    normalized = ROOT / "normalized"
    conversations = _read_jsonl(normalized / "conversations.jsonl")
    chunks: list[dict] = []
    for convo in conversations:
        chunks.extend(chunk_conversation(convo))
    dump_jsonl(normalized / "chunks.jsonl", chunks)
    print(f"wrote {len(chunks)} chunks")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())