Recommended params

app.py

@@ -115,10 +115,19 @@ async def predict(q: str):
             tokenize=False,
             add_generation_prompt=True,
         )
-        model_inputs = tokenizer([text], return_tensors="pt").to(model.device)
+        model_inputs = tokenizer(
+            [text], 
+            return_tensors="pt",
+            add_special_tokens=False
+        ).to(model.device)
 
         # Generate the output
-        generated_ids = model.generate(**model_inputs, max_new_tokens=32768)
+        generated_ids = model.generate(
+            **model_inputs,
+            top_p=0.9,
+            temperature=0.8,
+            max_new_tokens=512
+        )
 
         # Get and decode the output
         output_ids = generated_ids[0][len(model_inputs.input_ids[0]) :]