.

app.py

@@ -7,7 +7,8 @@ from rabbit_base import RabbitBase
 from listener import RabbitListenerBase
 from rabbit_repo import RabbitRepo
 from oa_server import OpenAIServers
-from vllm_backend import VLLMChatBackend, StubImagesBackend   # ✅ use your backend module
+from vllm_backend import VLLMChatBackend, StubImagesBackend   # ✅ our backend
+import state   # holds vllm_engine reference
 
 # ---- vLLM imports ----
 from vllm.engine.async_llm_engine import AsyncLLMEngine
@@ -28,14 +29,10 @@ except Exception:
     def gpu_entrypoint() -> str: 
         return "gpu: not available (CPU only)"
 
-# ----------------- vLLM globals -----------------
-vllm_engine: AsyncLLMEngine | None = None
-
+# ----------------- vLLM init -----------------
 async def init_vllm():
-    """Initialize vLLM engine with a Hugging Face model."""
-    global vllm_engine
-    if vllm_engine is not None:
-        return vllm_engine
+    if state.vllm_engine is not None:
+        return state.vllm_engine
 
     model_id = getattr(settings, "LlmHFModelID", "Qwen/Qwen2.5-7B-Instruct")
     log.info(f"Loading vLLM model: {model_id}")
@@ -45,8 +42,8 @@ async def init_vllm():
         trust_remote_code=True,
         max_model_len=getattr(settings, "LlmOpenAICtxSize", 32768),
     )
-    vllm_engine = AsyncLLMEngine.from_engine_args(args)
-    return vllm_engine
+    state.vllm_engine = AsyncLLMEngine.from_engine_args(args)
+    return state.vllm_engine
 
 # ----------------- RabbitMQ wiring -----------------
 publisher = RabbitRepo(external_source="openai.mq.server")
@@ -55,7 +52,7 @@ base = RabbitBase(exchange_type_resolver=resolver)
 
 servers = OpenAIServers(
     publisher,
-    chat_backend=VLLMChatBackend(),    # ✅ now from llm_backend.py
+    chat_backend=VLLMChatBackend(),
     images_backend=StubImagesBackend()
 )
 

state.py

@@ -0,0 +1,5 @@
+# state.py
+from typing import Optional
+from vllm.engine.async_llm_engine import AsyncLLMEngine
+
+vllm_engine: Optional[AsyncLLMEngine] = None

vllm_backend.py

@@ -1,10 +1,10 @@
 # vllm_backend.py
-import time, json, logging
+import time, logging
 from typing import Any, Dict, AsyncIterable
 
 from vllm.sampling_params import SamplingParams
 from oa_server import ChatBackend, ImagesBackend
-from app import vllm_engine  # global vLLM engine created in app.py
+from state import vllm_engine   # ✅ the single source of truth
 
 logger = logging.getLogger(__name__)
 
@@ -17,14 +17,14 @@ class VLLMChatBackend(ChatBackend):
         if vllm_engine is None:
             raise RuntimeError("vLLM engine not initialized")
 
-        # Naive: just grab the last user message for now
+        # For now: just grab the last user message
         messages = request.get("messages", [])
         prompt = messages[-1]["content"] if messages else "(empty)"
 
         params = SamplingParams(
             temperature=float(request.get("temperature", 0.7)),
             max_tokens=int(request.get("max_tokens", 512)),
-            stream=True
+            stream=True,
         )
 
         rid = f"chatcmpl-local-{int(time.time())}"
@@ -47,7 +47,7 @@ class VLLMChatBackend(ChatBackend):
             logger.exception("vLLM generation failed")
             raise
 
-        # final stop signal
+        # Final stop signal
         yield {
             "id": rid,
             "object": "chat.completion.chunk",
@@ -63,4 +63,5 @@ class StubImagesBackend(ImagesBackend):
     """
     async def generate_b64(self, request: Dict[str, Any]) -> str:
         logger.warning("Image generation not supported in local vLLM backend.")
+        # 1x1 transparent PNG
         return "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR4nGP4BwQACfsD/etCJH0AAAAASUVORK5CYII="