.

app.py

@@ -8,11 +8,6 @@ from listener import RabbitListenerBase
 from rabbit_repo import RabbitRepo
 from oa_server import OpenAIServers
 from vllm_backend import VLLMChatBackend, StubImagesBackend
-import state   # holds vllm_engine reference
-
-# ---- vLLM imports ----
-from vllm.engine.async_llm_engine import AsyncLLMEngine
-from vllm.engine.arg_utils import AsyncEngineArgs
 
 logging.basicConfig(
     level=logging.INFO,
@@ -28,40 +23,10 @@ try:
     def gpu_entrypoint() -> str:
         return "gpu: ready"
 
-    @spaces.GPU(duration=60)
-    def _build_vllm_engine_on_gpu(model_id: str, max_len: int):
-        args = AsyncEngineArgs(
-            model=model_id,
-            trust_remote_code=True,
-            max_model_len=max_len,
-        )
-        return AsyncLLMEngine.from_engine_args(args)
-
 except Exception:
     def gpu_entrypoint() -> str:
         return "gpu: not available (CPU only)"
 
-    def _build_vllm_engine_on_gpu(model_id: str, max_len: int):
-        args = AsyncEngineArgs(
-            model=model_id,
-            trust_remote_code=True,
-            max_model_len=max_len,
-        )
-        return AsyncLLMEngine.from_engine_args(args)
-
-# ----------------- vLLM init -----------------
-async def init_vllm():
-    if state.vllm_engine is not None:
-        return state.vllm_engine
-
-    model_id = getattr(settings, "LlmHFModelID", "Qwen/Qwen2.5-7B-Instruct")
-    max_len = int(getattr(settings, "LlmOpenAICtxSize", 32768))
-    log.info(f"Loading vLLM model: {model_id}")
-
-    # Build inside a GPU context so Spaces ZeroGPU exposes CUDA
-    state.vllm_engine = _build_vllm_engine_on_gpu(model_id, max_len)
-    return state.vllm_engine
-
 # ----------------- RabbitMQ wiring -----------------
 publisher = RabbitRepo(external_source="openai.mq.server")
 resolver = (lambda name: "direct" if name.startswith("oa.") else settings.RABBIT_EXCHANGE_TYPE)
@@ -90,7 +55,6 @@ listener = RabbitListenerBase(base, instance_name=settings.RABBIT_INSTANCE_NAME,
 # ----------------- Startup init -----------------
 async def _startup_init():
     try:
-        await init_vllm()             # load vLLM model
         await base.connect()          # connect to RabbitMQ
         await listener.start(DECLS)   # start queue listeners
         return "OpenAI MQ + vLLM: ready"

vllm_backend.py

@@ -4,57 +4,75 @@ from typing import Any, Dict, AsyncIterable
 
 from vllm.sampling_params import SamplingParams
 from backends_base import ChatBackend, ImagesBackend
-from state import vllm_engine   # ✅ the single source of truth
 
 logger = logging.getLogger(__name__)
 
+try:
+    import spaces
+except ImportError:
+    spaces = None
+
+
 class VLLMChatBackend(ChatBackend):
     """
-    Streams completions from a local vLLM engine.
-    Produces OpenAI-compatible ChatCompletionChunk dicts.
+    On ZeroGPU: build vLLM engine per request (no persistent state).
+    Returns a single ChatCompletionChunk with the full text.
     """
-    async def stream(self, request: Dict[str, Any]) -> AsyncIterable[Dict[str, Any]]:
-        if vllm_engine is None:
-            raise RuntimeError("vLLM engine not initialized")
 
-        # For now: just grab the last user message
+    async def stream(self, request: Dict[str, Any]) -> AsyncIterable[Dict[str, Any]]:
         messages = request.get("messages", [])
         prompt = messages[-1]["content"] if messages else "(empty)"
 
         params = SamplingParams(
             temperature=float(request.get("temperature", 0.7)),
             max_tokens=int(request.get("max_tokens", 512)),
-            stream=True,
+            stream=False,  # we want full text only
         )
 
         rid = f"chatcmpl-local-{int(time.time())}"
         now = int(time.time())
         model_name = request.get("model", "local-vllm")
 
+        # GPU wrapper for ZeroGPU
+        if spaces:
+            @spaces.GPU(duration=60)
+            def run_once(prompt: str) -> str:
+                from vllm.engine.async_llm_engine import AsyncLLMEngine
+                from vllm.engine.arg_utils import AsyncEngineArgs
+
+                args = AsyncEngineArgs(model=model_name, trust_remote_code=True)
+                engine = AsyncLLMEngine.from_engine_args(args)
+
+                # synchronous generate
+                outputs = list(engine.generate(prompt, params, request_id=rid))
+                return outputs[-1].outputs[0].text if outputs else ""
+
+        else:
+            def run_once(prompt: str) -> str:
+                from vllm.engine.async_llm_engine import AsyncLLMEngine
+                from vllm.engine.arg_utils import AsyncEngineArgs
+
+                args = AsyncEngineArgs(model=model_name, trust_remote_code=True)
+                engine = AsyncLLMEngine.from_engine_args(args)
+
+                outputs = list(engine.generate(prompt, params, request_id=rid))
+                return outputs[-1].outputs[0].text if outputs else ""
+
         try:
-            async for output in vllm_engine.generate(prompt, params, request_id=rid):
-                text_piece = output.outputs[0].text
-                yield {
-                    "id": rid,
-                    "object": "chat.completion.chunk",
-                    "created": now,
-                    "model": model_name,
-                    "choices": [
-                        {"index": 0, "delta": {"content": text_piece}, "finish_reason": None}
-                    ],
-                }
+            text = run_once(prompt)
+            yield {
+                "id": rid,
+                "object": "chat.completion.chunk",
+                "created": now,
+                "model": model_name,
+                "choices": [
+                    {"index": 0, "delta": {"content": text}, "finish_reason": "stop"}
+                ],
+            }
         except Exception:
-            logger.exception("vLLM generation failed")
+            logger.exception("vLLM inference failed")
             raise
 
-        # Final stop signal
-        yield {
-            "id": rid,
-            "object": "chat.completion.chunk",
-            "created": now,
-            "model": model_name,
-            "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
-        }
 
 class StubImagesBackend(ImagesBackend):
     """
@@ -63,5 +81,6 @@ class StubImagesBackend(ImagesBackend):
     """
     async def generate_b64(self, request: Dict[str, Any]) -> str:
         logger.warning("Image generation not supported in local vLLM backend.")
-        # 1x1 transparent PNG
-        return "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR4nGP4BwQACfsD/etCJH0AAAAASUVORK5CYII="
+        return (
+            "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR4nGP4BwQACfsD/etCJH0AAAAASUVORK5CYII="
+        )