Update app.py

app.py

@@ -266,48 +266,106 @@ For more information on `huggingface_hub` Inference API support, please check th
 # if __name__ == "__main__":
 #     demo.launch()
 
+# import gradio as gr
+# from transformers import AutoTokenizer, AutoModelForCausalLM
+# import torch
+
+# # Load once globally
+# tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-Coder-32B-Instruct")
+# model = AutoModelForCausalLM.from_pretrained(
+#     "Qwen/Qwen2.5-Coder-32B-Instruct",
+#     device_map="auto",
+#     torch_dtype=torch.float16,
+# )
+
+# def respond(message, history):
+#     system_prompt = (
+#         "You are a helpful coding assistant specialized in web development. "
+#         "Provide complete code snippets for HTML, CSS, JS, Flask, Node.js etc."
+#     )
+#     # Build input prompt including chat history
+#     chat_history = ""
+#     for user_msg, bot_msg in history:
+#         chat_history += f"User: {user_msg}\nAssistant: {bot_msg}\n"
+#     prompt = f"{system_prompt}\n{chat_history}User: {message}\nAssistant:"
+
+#     inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
+#     outputs = model.generate(
+#         **inputs,
+#         max_new_tokens=512,
+#         temperature=0.7,
+#         do_sample=True,
+#         top_p=0.95,
+#         eos_token_id=tokenizer.eos_token_id,
+#     )
+#     generated_text = tokenizer.decode(outputs[0], skip_special_tokens=True)
+
+#     # Extract only the new response part after the prompt
+#     response = generated_text[len(prompt):].strip()
+
+#     # Append current Q/A to history
+#     history.append((message, response))
+#     return "", history
+
+# demo = gr.ChatInterface(respond, type="messages")
+
+# if __name__ == "__main__":
+#     demo.launch()
+import os
 import gradio as gr
-from transformers import AutoTokenizer, AutoModelForCausalLM
-import torch
-
-# Load once globally
-tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-Coder-32B-Instruct")
-model = AutoModelForCausalLM.from_pretrained(
-    "Qwen/Qwen2.5-Coder-32B-Instruct",
-    device_map="auto",
-    torch_dtype=torch.float16,
+from huggingface_hub import InferenceClient
+from dotenv import load_dotenv
+
+# Load .env variables (make sure to have HF_TOKEN in .env or set as env var)
+load_dotenv()
+HF_TOKEN = os.getenv("HF_TOKEN")  # or directly assign your token here as string
+
+# Initialize InferenceClient with Hugging Face API token
+client = InferenceClient(
+    model="Qwen/Qwen2.5-Coder-32B-Instruct",
+    token=HF_TOKEN
 )
 
 def respond(message, history):
-    system_prompt = (
-        "You are a helpful coding assistant specialized in web development. "
-        "Provide complete code snippets for HTML, CSS, JS, Flask, Node.js etc."
-    )
-    # Build input prompt including chat history
-    chat_history = ""
-    for user_msg, bot_msg in history:
-        chat_history += f"User: {user_msg}\nAssistant: {bot_msg}\n"
-    prompt = f"{system_prompt}\n{chat_history}User: {message}\nAssistant:"
-
-    inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
-    outputs = model.generate(
-        **inputs,
-        max_new_tokens=512,
-        temperature=0.7,
-        do_sample=True,
-        top_p=0.95,
-        eos_token_id=tokenizer.eos_token_id,
+    """
+    Chat response generator function streaming from Hugging Face Inference API.
+    """
+    system_message = (
+        "You are a helpful and experienced coding assistant specialized in web development. "
+        "Help the user by generating complete and functional code for building websites. "
+        "You can provide HTML, CSS, JavaScript, and backend code (like Flask, Node.js, etc.) "
+        "based on their requirements."
     )
-    generated_text = tokenizer.decode(outputs[0], skip_special_tokens=True)
-
-    # Extract only the new response part after the prompt
-    response = generated_text[len(prompt):].strip()
-
-    # Append current Q/A to history
-    history.append((message, response))
-    return "", history
-
-demo = gr.ChatInterface(respond, type="messages")
+    max_tokens = 2048
+    temperature = 0.7
+    top_p = 0.95
+
+    # Prepare messages in OpenAI chat format
+    messages = [{"role": "system", "content": system_message}]
+    for user_msg, assistant_msg in history:
+        if user_msg:
+            messages.append({"role": "user", "content": user_msg})
+        if assistant_msg:
+            messages.append({"role": "assistant", "content": assistant_msg})
+    messages.append({"role": "user", "content": message})
+
+    response = ""
+    # Stream response tokens from Hugging Face Inference API
+    for chunk in client.chat.completions.create(
+        model="Qwen/Qwen2.5-Coder-32B-Instruct",
+        messages=messages,
+        max_tokens=max_tokens,
+        stream=True,
+        temperature=temperature,
+        top_p=top_p,
+    ):
+        token = chunk.choices[0].delta.get("content", "")
+        response += token
+        yield response
+
+# Create Gradio chat interface
+demo = gr.ChatInterface(fn=respond, title="Coding Assistant", 
+                       description="Ask for web development code help!")
 
 if __name__ == "__main__":
     demo.launch()
@@ -318,3 +376,4 @@ if __name__ == "__main__":
 
 
 
+