Update app.py

app.py

@@ -1,4 +1,4 @@
-# app.py — MCP POC using local Hugging Face model (flan-t5 or other) or rule-based fallback.
+# app.py — MCP POC using OpenRouter for LLM (replaces local HF model)
 # Place this file next to config.py. Do NOT store secrets here.
 
 from mcp.server.fastmcp import FastMCP
@@ -12,14 +12,6 @@ import traceback
 import inspect
 import re
 
-# Optional transformers imports — load only if available
-TRANSFORMERS_AVAILABLE = False
-try:
-    from transformers import pipeline, AutoTokenizer, AutoModelForSeq2SeqLM, AutoModelForCausalLM
-    TRANSFORMERS_AVAILABLE = True
-except Exception:
-    TRANSFORMERS_AVAILABLE = False
-
 # ----------------------------
 # Load config
 # ----------------------------
@@ -29,13 +21,18 @@ try:
         CLIENT_SECRET,
         REFRESH_TOKEN,
         API_BASE,
-        LOCAL_MODEL,      # e.g. "google/flan-t5-base" or None
+        OPENROUTER_API_KEY,   # your OpenRouter API key (put this in config.py)
+        OPENROUTER_MODEL      # e.g. "gpt-4o-mini" or any model name routed by OpenRouter
     )
 except Exception:
     raise SystemExit(
-        "Make sure config.py exists with CLIENT_ID, CLIENT_SECRET, REFRESH_TOKEN, API_BASE, LOCAL_MODEL (or leave LOCAL_MODEL=None)."
+        "Make sure config.py exists with CLIENT_ID, CLIENT_SECRET, REFRESH_TOKEN, API_BASE, "
+        "OPENROUTER_API_KEY and OPENROUTER_MODEL (or set OPENROUTER_MODEL to your preferred model)."
     )
 
+# OpenRouter endpoint (public OpenRouter cloud endpoint)
+OPENROUTER_BASE_URL = "https://api.openrouter.ai/v1"
+
 # ----------------------------
 # Initialize FastMCP
 # ----------------------------
@@ -87,71 +84,79 @@ def _log_llm_call(confidence: Optional[float] = None):
 _init_analytics()
 
 # ----------------------------
-# Local LLM pipeline initialization
+# OpenRouter wrapper
 # ----------------------------
-LLM_PIPELINE = None
-TOKENIZER = None
+def _openrouter_headers():
+    return {"Authorization": f"Bearer {OPENROUTER_API_KEY}", "Content-Type": "application/json"}
 
-def init_local_model():
+def openrouter_generate(system_prompt: str, user_prompt: str, history: Optional[List[Tuple[str,str]]] = None, max_tokens: int = 512) -> Dict[str, Any]:
     """
-    Initialize local HF model pipeline depending on LOCAL_MODEL.
-    Supports seq2seq (flan/t5) and causal models.
-    If transformers is unavailable or LOCAL_MODEL is None, leaves LLM_PIPELINE as None.
+    Call OpenRouter chat completions endpoint with messages built from system + history + user prompt.
+    Returns dict: {'text': <str>, 'raw': <resp_json>, 'confidence': Optional[float]}
     """
-    global LLM_PIPELINE, TOKENIZER
+    messages = []
+    # system
+    if system_prompt:
+        messages.append({"role": "system", "content": system_prompt})
+    # history (list of (user,assistant))
+    history = history or []
+    for pair in history:
+        try:
+            u, a = pair[0], pair[1]
+            if u:
+                messages.append({"role": "user", "content": u})
+            if a:
+                messages.append({"role": "assistant", "content": a})
+        except Exception:
+            continue
+    # current user
+    messages.append({"role": "user", "content": user_prompt})
+
+    body = {
+        "model": OPENROUTER_MODEL,
+        "messages": messages,
+        "max_tokens": max_tokens,
+        # "temperature": 0.0,   # set if you want deterministic responses
+    }
 
-    if not LOCAL_MODEL:
-        print("LOCAL_MODEL is None — using rule-based fallback.")
-        LLM_PIPELINE = None
-        return
+    url = f"{OPENROUTER_BASE_URL}/chat/completions"
+    try:
+        r = requests.post(url, headers=_openrouter_headers(), json=body, timeout=60)
+    except Exception as e:
+        raise RuntimeError(f"OpenRouter request failed: {e}")
 
-    if not TRANSFORMERS_AVAILABLE:
-        print("transformers not installed — using rule-based fallback.")
-        LLM_PIPELINE = None
-        return
+    if r.status_code not in (200, 201):
+        # surface the error for debugging
+        raise RuntimeError(f"OpenRouter API error {r.status_code}: {r.text}")
 
+    resp_json = r.json()
+    # Parse response for text; different routers may vary slightly
+    text = ""
+    confidence = None
     try:
-        tokenizer_name = LOCAL_TOKENIZER or LOCAL_MODEL
-
-        # Detect seq2seq family (T5/Flan)
-        if any(x in LOCAL_MODEL.lower() for x in ["flan", "t5", "seq2seq"]):
-            TOKENIZER = AutoTokenizer.from_pretrained(tokenizer_name, use_fast=True)
-            model = AutoModelForSeq2SeqLM.from_pretrained(LOCAL_MODEL)
-            LLM_PIPELINE = pipeline("text2text-generation", model=model, tokenizer=TOKENIZER)
-            print(f"Loaded seq2seq model pipeline for {LOCAL_MODEL}")
+        # typical shape: choices[0].message.content or choices[0].message
+        choice = resp_json.get("choices", [{}])[0]
+        message = choice.get("message", {}) or {}
+        if isinstance(message, dict):
+            text = message.get("content") or message.get("content", "")
+            # sometimes content is a dict mapping types -> text
+            if isinstance(text, dict):
+                # join possible parts
+                text = text.get("text") or next(iter(text.values()), "")
         else:
-            # causal model path
-            TOKENIZER = AutoTokenizer.from_pretrained(tokenizer_name, use_fast=True)
-            model = AutoModelForCausalLM.from_pretrained(LOCAL_MODEL)
-            LLM_PIPELINE = pipeline("text-generation", model=model, tokenizer=TOKENIZER)
-            print(f"Loaded causal model pipeline for {LOCAL_MODEL}")
-
-    except Exception as e:
-        print("Failed to load local model:", e)
-        traceback.print_exc()
-        LLM_PIPELINE = None
+            text = str(message)
+        # some providers include scores
+        confidence = choice.get("finish_reason_score") or choice.get("score") or None
+    except Exception:
+        text = json.dumps(resp_json)
 
-# Try to init model at startup (may be slow)
-init_local_model()
+    _log_llm_call(confidence)
+    return {"text": text, "raw": resp_json, "confidence": confidence}
 
 # ----------------------------
-# Rule-based fallback responder
-# ----------------------------
-def rule_based_response(message: str) -> str:
-    msg = (message or "").strip().lower()
-    if msg.startswith("create record") or msg.startswith("create contact"):
-        return "To create a record, use the command: create_record MODULE_NAME {\"Field\": \"value\"}"
-    if msg.startswith("create_invoice"):
-        return "To create invoice: create_invoice {\"customer_id\": \"...\", \"line_items\": [...]} (JSON)"
-    if msg.startswith("help") or msg.startswith("what can you do"):
-        return "I can create/update/delete records in Zoho (create_record/update_record/delete_record) or process local files by pasting their path (/mnt/data/...)."
-    return "(fallback) No local LLM loaded. Use explicit commands like `create_record` or paste a /mnt/data/ path."
-
-# ----------------------------
-# Zoho token refresh & headers helper
+# Zoho token refresh & headers
 # ----------------------------
 def _get_valid_token_headers() -> dict:
-    # Note: region-specific account host may need .com or .eu — ensure API_BASE matches services used.
     token_url = "https://accounts.zoho.in/oauth/v2/token"
     params = {
         "refresh_token": REFRESH_TOKEN,
@@ -167,7 +172,7 @@ def _get_valid_token_headers() -> dict:
         raise RuntimeError(f"Failed to refresh Zoho token: {r.status_code} {r.text}")
 
 # ----------------------------
-# MCP tools: Zoho CRM & Books (CRUD + document processing)
+# MCP tools: Zoho CRUD & process_document (unchanged)
 # ----------------------------
 @mcp.tool()
 def authenticate_zoho() -> str:
@@ -243,10 +248,6 @@ def delete_record(module_name: str, record_id: str) -> str:
 
 @mcp.tool()
 def create_invoice(data: dict) -> str:
-    """
-    Creates an invoice in Zoho Books.
-    NOTE: Ensure API_BASE points to the Books base (e.g. https://books.zoho.in/api/v3) when calling invoices.
-    """
     try:
         headers = _get_valid_token_headers()
         url = f"{API_BASE}/invoices"
@@ -270,7 +271,6 @@ def process_document(file_path: str, target_module: Optional[str] = "Contacts")
     try:
         if os.path.exists(file_path):
             file_url = f"file://{file_path}"
-            # Placeholder extraction — replace with OCR + parsing logic
             extracted = {
                 "Name": "ACME Corp (simulated)",
                 "Email": "[email protected]",
@@ -294,15 +294,9 @@ def process_document(file_path: str, target_module: Optional[str] = "Contacts")
         return {"status": "error", "error": str(e)}
 
 # ----------------------------
-# Simple local command parser to call tools explicitly from chat (POC)
+# Simple local command parser
 # ----------------------------
 def try_parse_and_invoke_command(text: str):
-    """
-    Very small parser for explicit chat commands:
-      create_record MODULE {json}
-      create_invoice {json}
-      process_document /mnt/data/...
-    """
     text = text.strip()
     # create_record
     m = re.match(r"^create_record\s+(\w+)\s+(.+)$", text, re.I)
@@ -334,45 +328,56 @@ def try_parse_and_invoke_command(text: str):
     return None
 
 # ----------------------------
-# Local LLM / fallback generator
+# OpenRouter-based chat handler
 # ----------------------------
-def local_llm_generate(prompt: str) -> str:
-    if LLM_PIPELINE is not None:
-        try:
-            # For seq2seq (text2text) the pipeline returns 'generated_text'
-            out = LLM_PIPELINE(prompt, max_new_tokens=256)
-            if isinstance(out, list) and len(out) > 0:
-                # text2text pipelines often provide 'generated_text'
-                if isinstance(out[0], dict):
-                    return out[0].get("generated_text") or out[0].get("text") or str(out[0])
-                return str(out[0])
-            return str(out)
-        except Exception as e:
-            print("LLM pipeline error:", e)
-            traceback.print_exc()
-            return rule_based_response(prompt)
-    else:
-        return rule_based_response(prompt)
+def openrouter_response(message: str, history: Optional[List[Tuple[str,str]]] = None) -> str:
+    history = history or []
+    system_prompt = (
+        "You are Zoho Assistant. Keep responses concise. When appropriate, output a JSON object with keys 'tool' and 'args' "
+        "so the server can automatically call the corresponding MCP tool. Example:\n"
+        '{"tool":"create_record","args":{"module_name":"Contacts","record_data":{"Last_Name":"Doe","Email":"[email protected]"}}}\n'
+        "If not invoking tools, answer conversationally."
+    )
+    try:
+        resp = openrouter_generate(system_prompt, message, history)
+        text = resp.get("text", "")
+        # If LLM returned JSON indicating a tool invocation, attempt to parse & run
+        parsed = None
+        payload = text.strip()
+        if payload.startswith("{") or payload.startswith("["):
+            try:
+                parsed = json.loads(payload)
+            except Exception:
+                parsed = None
+        if isinstance(parsed, dict) and "tool" in parsed:
+            tool_name = parsed.get("tool")
+            args = parsed.get("args", {})
+            # Try call local tool by name if exists
+            if tool_name in globals() and callable(globals()[tool_name]):
+                try:
+                    result = globals()[tool_name](**args) if isinstance(args, dict) else globals()[tool_name](args)
+                    return f"Invoked tool '{tool_name}'. Result:\n{result}"
+                except Exception as e:
+                    return f"Tool invocation error: {e}"
+            else:
+                return f"Requested tool '{tool_name}' not found locally."
+        return text
+    except Exception as e:
+        return f"(OpenRouter error) {e}"
 
 # ----------------------------
-# Gradio chat handler (accepts message, history)
+# Gradio chat handler
 # ----------------------------
 def chat_handler(message, history):
-    """
-    Gradio ChatInterface calls this with (message, history).
-    If the message is a local file path (starting with /mnt/data/), we pass it unchanged
-    to process_document() — this satisfies the developer instruction to send the path as the URL.
-    Otherwise, try explicit commands, then local LLM fallback.
-    """
     history = history or []
     trimmed = (message or "").strip()
 
-    # 1) explicit commands
+    # Explicit POC commands
     cmd = try_parse_and_invoke_command(trimmed)
     if cmd is not None:
         return cmd
 
-    # 2) developer convenience: local path handling
+    # Developer convenience: local path handling (send unchanged)
     if trimmed.startswith("/mnt/data/"):
         try:
             doc = process_document(trimmed)
@@ -380,31 +385,8 @@ def chat_handler(message, history):
         except Exception as e:
             return f"Error processing document: {e}"
 
-    # 3) otherwise call local LLM (if available) or fallback
-    # build a compact prompt including a short system instruction and history
-    history_text = ""
-    for pair in history:
-        try:
-            user_turn, assistant_turn = pair[0], pair[1]
-        except Exception:
-            if isinstance(pair, dict):
-                user_turn = pair.get("user", "")
-                assistant_turn = pair.get("assistant", "")
-            else:
-                user_turn, assistant_turn = "", ""
-        if user_turn:
-            history_text += f"User: {user_turn}\n"
-        if assistant_turn:
-            history_text += f"Assistant: {assistant_turn}\n"
-
-    system = "You are a Zoho assistant that can call local MCP tools when asked. Keep replies short and actionable."
-    prompt = f"{system}\n{history_text}\nUser: {trimmed}\nAssistant:"
-    try:
-        resp = local_llm_generate(prompt)
-        _log_llm_call(None)
-        return resp
-    except Exception as e:
-        return f"LLM error: {e}"
+    # Otherwise, call OpenRouter
+    return openrouter_response(trimmed, history)
 
 # ----------------------------
 # Gradio UI
@@ -412,13 +394,13 @@ def chat_handler(message, history):
 def chat_interface():
     return gr.ChatInterface(
         fn=chat_handler,
-        textbox=gr.Textbox(placeholder="Ask me to create contacts, invoices, or paste /mnt/data/... for dev.")
+        textbox=gr.Textbox(placeholder="Ask me to create contacts, invoices, upload docs (or paste /mnt/data/... for dev).")
     )
 
 # ----------------------------
 # Entrypoint
 # ----------------------------
 if __name__ == "__main__":
-    print("[startup] Launching Gradio UI + FastMCP server (local LLM mode).")
+    print("[startup] Launching Gradio UI + FastMCP server (OpenRouter mode).")
     demo = chat_interface()
     demo.launch(server_name="0.0.0.0", server_port=7860)