from zbt.domains.knowledge.chunking import DocumentChunker def test_chunker_preserves_short_paragraph_boundaries() -> None: chunker = DocumentChunker(max_chars=36, overlap_chars=8) chunks = chunker.split( "等待期为30天。\n\n意外伤害不受等待期限制。\n\n报销需提供发票和诊断证明。" ) assert chunks == [ "等待期为30天。", "意外伤害不受等待期限制。", "报销需提供发票和诊断证明。", ] assert all(len(chunk) <= 36 for chunk in chunks) def test_chunker_splits_long_paragraph_with_context_overlap() -> None: chunker = DocumentChunker(max_chars=24, overlap_chars=6) content = "第一条说明等待期规则,疾病医疗等待期为三十天。第二条说明意外伤害不受等待期限制。" chunks = chunker.split(content) assert len(chunks) >= 2 assert all(len(chunk) <= 24 for chunk in chunks) assert chunks[0][-6:] == chunks[1][:6] assert "".join([chunks[0], *[chunk[6:] for chunk in chunks[1:]]]) == content def test_chunker_rejects_blank_document() -> None: chunker = DocumentChunker() assert chunker.split(" \n\n ") == []