|
|
|
|
|
import asyncio |
|
|
|
|
|
async def stream_in_chunks(publish, exchange: str, llm_obj_builder, text: str, |
|
|
batch_size: int = 3, max_chars: int = 100, |
|
|
base_delay_ms: int = 30, per_char_ms: int = 2) -> None: |
|
|
seps = set(" ,!?{}.:;\n") |
|
|
buf, parts, count = [], [], 0 |
|
|
for ch in text: |
|
|
parts.append(ch) |
|
|
if ch in seps: |
|
|
buf.append("".join(parts)); parts.clear(); count += 1 |
|
|
if count >= batch_size or sum(len(x) for x in buf) >= max_chars: |
|
|
o = llm_obj_builder("".join(buf)) |
|
|
await publish(exchange, o) |
|
|
await asyncio.sleep((base_delay_ms + per_char_ms * sum(len(x) for x in buf))/1000) |
|
|
buf.clear(); count = 0 |
|
|
if parts: buf.append("".join(parts)) |
|
|
if buf: |
|
|
await publish(exchange, llm_obj_builder("".join(buf))) |
|
|
|
