backend/llm_service.py

import modal
import os
import json
import urllib.request
import traceback

# Define a Modal App
# You can give it a more specific name if you like, e.g., "buffetbot-llm-service"
app = modal.App("buffetbot-llm")

@app.function(
    secrets=[modal.Secret.from_name("buffetbot-vllm-config")],
    gpu="H100:1",  # Add GPU specification
    timeout=1500  # Increased timeout to 25 minutes
)
def generate_llm_response(prompt: str, system_message: str) -> str:
    """Generates a response using our deployed vLLM service."""
    print(f"Received prompt for LLM: {prompt[:200]}...") # Log the prompt for debugging

    # Get the URL of our deployed vLLM service
    vllm_url = os.environ.get("VLLM_SERVICE_URL")
    if not vllm_url:
        raise ValueError("VLLM_SERVICE_URL environment variable not set")

    # Prepare the request with system message for better context
    messages = [
        {
            "role": "system",
            "content": "You are BuffetBot, a concise stock analyst. Provide clear recommendations based on key metrics and economic moat. Use 'BuffetBot recommends'."
        },
        {"role": "user", "content": prompt}
    ]
    
    headers = {
        "Authorization": f"Bearer {os.environ['API_KEY']}",
        "Content-Type": "application/json",
    }
    payload = json.dumps({
        "messages": messages,
        "model": "neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w4a16",
        "temperature": 0.3,  # Lower temperature for more focused responses
        "max_tokens": 500,   # Reduce max tokens for faster responses
        "top_p": 0.9,        # Add top_p for better quality/speed balance
        "frequency_penalty": 0.0,  # Add frequency penalty to reduce repetition
        "presence_penalty": 0.0    # Add presence penalty to encourage diversity
    })

    # Make the request to our vLLM service
    req = urllib.request.Request(
        f"{vllm_url}/v1/chat/completions",
        data=payload.encode("utf-8"),
        headers=headers,
        method="POST",
    )

    try:
        with urllib.request.urlopen(req, timeout=1200) as response:  # Increased timeout to 20 minutes
            result = json.loads(response.read().decode())
            return result["choices"][0]["message"]["content"]
    except Exception as e:
        traceback.print_exc()
        print(f"Error calling vLLM service: {str(e)}")
        # Fallback to a simple response if the LLM service fails
        return "I apologize, but I'm having trouble accessing the LLM service at the moment. Please try again later."

@app.local_entrypoint()
def main():
    # Test with a prompt similar to what stock_analyzer.py sends
    test_prompt = """
As an AI-powered stock analyst, provide a comprehensive investment insight for AAPL. Focus on explaining what the key numbers mean, how the financial health and growth metrics contribute to the overall picture, and the significance of its economic moat.

Company Name: Apple Inc.
Sector: Technology
Industry: Consumer Electronics
Full-time Employees: 161000

## Current Market Status
- Current Price: $170.00
- 52 Week High: $180.00
- 52 Week Low: $120.00
- Market Cap: $2,600,000,000,000.00

## Key Valuation Metrics
- P/E Ratio: 28.0
- Forward P/E: 26.0
- PEG Ratio: 2.1
- Dividend Yield: 0.5%

## Growth and Profitability
- Revenue Growth (YoY): 0.05
- Profit Margins: 0.25
- Operating Margins: 0.30

## Financial Health Indicators
- Current Ratio: 1.2
- Debt to Equity: 1.5
- Return on Equity: 1.5
- Debt to Asset Ratio: 0.45

#### Recent Financials (Latest Available)
- Latest Revenue: $90,000,000,000.00
- Latest Free Cash Flow: $25,000,000,000.00
- Latest Shares Outstanding: 15,500,000,000.00

#### Key Balance Sheet Items
- Total Assets: $350,000,000,000.00
- Total Liabilities: $250,000,000,000.00
- Total Debt: $100,000,000,000.00
- Total Cash: $50,000,000,000.00
- Total Equity: $100,000,000,000.00
- Debt to Equity: 1.0

#### Key Income Statement Items
- Gross Profits: $40,000,000,000.00
- Operating Income: $30,000,000,000.00
- Net Income: $20,000,000,000.00
- Earnings Growth (YoY): 0.08

## Economic Moat Analysis
Moat Rating: Wide Moat

Based on the detailed financial data and moat analysis provided above, give a comprehensive, easy-to-understand summary of the investment potential of AAPL. Explain the significance of key numbers and their implications. Conclude with a clear overall sentiment regarding investment potential.
"""
    print(f"Testing LLM with prompt: {test_prompt[:100]}...")
    response = generate_llm_response.remote(test_prompt, "You are a professional stock analyst providing clear, concise, and accurate investment analysis. Focus on explaining the numbers and their implications. Avoid any garbled or unclear text.")
    print(f"LLM Response: {response}")

    # Test with a simple prompt that includes the stock data
    simple_prompt = """
Based on the following data for Apple Inc. (AAPL), summarize the key investment considerations:

Company: Apple Inc. (AAPL)
Current Price: $170.00
P/E Ratio: 28.0
Revenue Growth: 5%
Profit Margin: 25%
Moat Rating: Wide Moat

Please provide a concise summary of the investment considerations.
"""
    print(f"Testing LLM with simple prompt: {simple_prompt[:100]}...")
    response_simple = generate_llm_response.remote(simple_prompt, "You are a professional stock analyst providing clear, concise, and accurate investment analysis. Focus on explaining the numbers and their implications. Avoid any garbled or unclear text.")
    print(f"LLM Response (simple): {response_simple}")