Spaces:
Running
on
Zero
Running
on
Zero
Update app.py
Browse files
app.py
CHANGED
|
@@ -6,744 +6,532 @@ import torch
|
|
| 6 |
import re
|
| 7 |
import json
|
| 8 |
import pandas as pd
|
| 9 |
-
import
|
| 10 |
-
import
|
| 11 |
-
import spaces # Import the spaces library
|
| 12 |
|
| 13 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 14 |
model_cache = {}
|
|
|
|
| 15 |
|
|
|
|
| 16 |
HF_TOKEN = os.environ.get("HF_TOKEN")
|
| 17 |
|
| 18 |
# --- Constants for Benchmarks ---
|
| 19 |
MMLU_DATASET = "cais/mmlu"
|
| 20 |
MMLU_PRO_DATASET = "TIGER-Lab/MMLU-Pro"
|
|
|
|
|
|
|
|
|
|
|
|
|
| 21 |
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
all_options = {}
|
| 25 |
-
gr_dropdown_options = [] # This is for initial display only, not used for dynamic updates directly
|
| 26 |
-
|
| 27 |
-
# Get subjects for MMLU
|
| 28 |
-
try:
|
| 29 |
-
mmlu_subjects = get_dataset_config_names(MMLU_DATASET, token=HF_TOKEN)
|
| 30 |
-
all_options[MMLU_DATASET] = ["ALL"] + mmlu_subjects
|
| 31 |
-
except Exception as e:
|
| 32 |
-
print(f"Warning: Could not load MMLU dataset configs. Error: {e}")
|
| 33 |
-
all_options[MMLU_DATASET] = []
|
| 34 |
-
|
| 35 |
-
# Get subjects for MMLU-Pro
|
| 36 |
-
try:
|
| 37 |
-
mmlu_pro_subjects = get_dataset_config_names(MMLU_PRO_DATASET, token=HF_TOKEN)
|
| 38 |
-
all_options[MMLU_PRO_DATASET] = ["ALL"] + mmlu_pro_subjects
|
| 39 |
-
except Exception as e:
|
| 40 |
-
print(f"Warning: Could not load MMLU-Pro dataset configs. It might not be accessible or available. Error: {e}")
|
| 41 |
-
all_options[MMLU_PRO_DATASET] = []
|
| 42 |
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 54 |
def load_model(model_id):
|
| 55 |
"""
|
| 56 |
-
Loads a Hugging Face model and
|
| 57 |
-
Uses a cache to avoid
|
| 58 |
-
Provides Gradio Info/Error messages for user feedback.
|
| 59 |
-
Raises an exception if model loading fails.
|
| 60 |
"""
|
|
|
|
|
|
|
|
|
|
| 61 |
gr.Info(f"Attempting to load model: {model_id}...")
|
| 62 |
if model_id in model_cache:
|
| 63 |
-
gr.Info(f"Model '{model_id}'
|
| 64 |
return model_cache[model_id]
|
|
|
|
| 65 |
try:
|
| 66 |
-
#
|
|
|
|
|
|
|
| 67 |
tokenizer = AutoTokenizer.from_pretrained(model_id, token=HF_TOKEN, trust_remote_code=True)
|
| 68 |
model = AutoModelForCausalLM.from_pretrained(
|
| 69 |
model_id,
|
| 70 |
token=HF_TOKEN,
|
| 71 |
-
torch_dtype=
|
| 72 |
trust_remote_code=True
|
| 73 |
).to("cuda" if torch.cuda.is_available() else "cpu")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 74 |
|
| 75 |
-
# Create a text-generation pipeline
|
| 76 |
-
generator = pipeline("text-generation", model=model, tokenizer=tokenizer, device=0 if torch.cuda.is_available() else -1)
|
| 77 |
-
|
| 78 |
-
# Cache the loaded generator
|
| 79 |
model_cache[model_id] = generator
|
| 80 |
gr.Info(f"Model '{model_id}' loaded successfully.")
|
| 81 |
return generator
|
| 82 |
except Exception as e:
|
| 83 |
-
#
|
| 84 |
-
raise
|
|
|
|
| 85 |
|
|
|
|
| 86 |
|
| 87 |
def format_prompt(item):
|
|
|
|
|
|
|
|
|
|
| 88 |
|
| 89 |
-
|
| 90 |
-
A.
|
| 91 |
-
|
| 92 |
-
C. {item['choices'][2]}
|
| 93 |
-
D. {item['choices'][3]}
|
| 94 |
-
Answer:"""
|
| 95 |
-
return prompt, item['answer'] # Returns the prompt string and the correct choice index (0-3)
|
| 96 |
|
| 97 |
-
def
|
| 98 |
"""
|
| 99 |
-
Extracts the
|
| 100 |
-
It
|
| 101 |
"""
|
| 102 |
-
|
| 103 |
-
match
|
| 104 |
-
if match:
|
| 105 |
-
return match.group(1).upper() # Ensure it's uppercase
|
| 106 |
-
|
| 107 |
-
# Fallback: look for a single capital letter A-D anywhere in the output
|
| 108 |
-
match = re.search(r"\b([ABCD])\b", output.strip())
|
| 109 |
-
if match:
|
| 110 |
-
return match.group(1)
|
| 111 |
-
|
| 112 |
-
return None # Return None if no valid choice letter is found
|
| 113 |
-
|
| 114 |
-
def get_choice_letter(index):
|
| 115 |
-
"""Converts a numerical choice index (0-3) to a capital letter (A-D)."""
|
| 116 |
-
if 0 <= index <= 3:
|
| 117 |
-
return chr(ord('A') + index)
|
| 118 |
-
return None # Return None for invalid indices
|
| 119 |
|
| 120 |
def evaluate_single_subject(generator, dataset_id, subject, sample_count, progress):
|
| 121 |
"""
|
| 122 |
-
Evaluates a
|
| 123 |
-
|
| 124 |
-
Args:
|
| 125 |
-
generator: The Hugging Face pipeline for text generation.
|
| 126 |
-
dataset_id (str): The ID of the dataset (e.g., "cais/mmlu", "cais/mmlu_pro").
|
| 127 |
-
subject (str): The specific subject/config name within the dataset.
|
| 128 |
-
sample_count (int): The maximum number of samples to evaluate.
|
| 129 |
-
progress (gr.Progress): Gradio progress tracker.
|
| 130 |
-
|
| 131 |
-
Returns:
|
| 132 |
-
tuple: (accuracy, list_of_detailed_results)
|
| 133 |
-
Raises:
|
| 134 |
-
Exception: If dataset loading fails.
|
| 135 |
"""
|
| 136 |
-
gr.Info(f"Loading dataset: {dataset_id}
|
| 137 |
try:
|
| 138 |
-
# Load the
|
| 139 |
-
dataset = load_dataset(dataset_id, subject, token=HF_TOKEN
|
| 140 |
except Exception as e:
|
| 141 |
-
# Re-raise the exception to be caught by the outer run_evaluation try-except
|
| 142 |
raise RuntimeError(f"Failed to load dataset '{dataset_id}' for subject '{subject}'. Error: {e}")
|
| 143 |
|
| 144 |
-
#
|
| 145 |
-
|
| 146 |
-
dataset = dataset.shuffle(seed=42).select(range(
|
| 147 |
|
| 148 |
-
|
| 149 |
-
|
| 150 |
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
expected_letter = get_choice_letter(answer_idx)
|
| 155 |
-
|
| 156 |
-
# Generate only 1 new token for the answer (A, B, C, D)
|
| 157 |
-
# do_sample=False ensures deterministic output for a given prompt (greedy decoding)
|
| 158 |
-
output_raw = generator(prompt, max_new_tokens=1, do_sample=False)[0]["generated_text"]
|
| 159 |
|
| 160 |
-
#
|
| 161 |
-
|
|
|
|
| 162 |
|
| 163 |
-
|
| 164 |
-
predicted_letter = extract_choice_letter(output_raw)
|
| 165 |
-
|
| 166 |
is_correct = (predicted_letter == expected_letter)
|
| 167 |
-
correct_count += is_correct
|
| 168 |
|
| 169 |
-
|
| 170 |
-
|
|
|
|
|
|
|
| 171 |
"question": item['question'],
|
| 172 |
"choices": item['choices'],
|
| 173 |
-
"
|
| 174 |
-
"
|
| 175 |
-
"
|
| 176 |
"is_correct": is_correct,
|
| 177 |
-
"is_reasoning_model_output": is_reasoning_model_output # Store the flag
|
| 178 |
})
|
| 179 |
-
|
| 180 |
-
# Calculate accuracy for the current subject
|
| 181 |
-
accuracy = (correct_count / len(dataset)) * 100 if len(dataset) > 0 else 0
|
| 182 |
-
return accuracy, subject_results
|
| 183 |
|
| 184 |
-
|
| 185 |
-
|
| 186 |
-
"""
|
| 187 |
-
Main function to orchestrate the evaluation process.
|
| 188 |
-
Handles single subject or 'ALL' subjects evaluation for MMLU/MMLU-Pro.
|
| 189 |
-
Returns Gradio.update objects to control UI component visibility and content.
|
| 190 |
-
"""
|
| 191 |
-
gr.Info("Starting evaluation...")
|
| 192 |
-
if not model_id:
|
| 193 |
-
gr.Warning("Please enter a Hugging Face Model ID before running the evaluation.")
|
| 194 |
-
# Return updates to hide logs/debug and show empty results
|
| 195 |
-
return "", gr.update(value="", visible=False), gr.update(visible=False), \
|
| 196 |
-
gr.update(visible=False), gr.update(visible=False), gr.update(value="", visible=False)
|
| 197 |
-
|
| 198 |
-
dataset_id_map = {
|
| 199 |
-
"MMLU": MMLU_DATASET,
|
| 200 |
-
"MMLU-Pro": MMLU_PRO_DATASET
|
| 201 |
-
}
|
| 202 |
-
current_dataset_id = dataset_id_map.get(benchmark_category)
|
| 203 |
|
| 204 |
-
if not current_dataset_id:
|
| 205 |
-
gr.Error(f"Unknown benchmark category selected: {benchmark_category}. This should not happen.")
|
| 206 |
-
return "", gr.update(value="", visible=False), gr.update(visible=False), \
|
| 207 |
-
gr.update(visible=False), gr.update(visible=False), gr.update(value="", visible=False)
|
| 208 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 209 |
try:
|
| 210 |
-
|
|
|
|
| 211 |
|
| 212 |
-
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
|
| 216 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 217 |
if subject_name == "ALL":
|
| 218 |
-
|
| 219 |
-
if "ALL" in
|
| 220 |
-
|
| 221 |
-
|
| 222 |
-
|
| 223 |
-
gr.Warning(f"No subjects found to evaluate for '{benchmark_category}'.")
|
| 224 |
-
return "", gr.update(value="", visible=False), gr.update(visible=False), \
|
| 225 |
-
gr.update(visible=False), gr.update(visible=False), gr.update(value="", visible=False)
|
| 226 |
-
|
| 227 |
-
for i, sub in enumerate(progress.tqdm(subjects_to_evaluate, desc=f"Evaluating ALL {benchmark_category} subjects")):
|
| 228 |
-
gr.Info(f"Evaluating {benchmark_category} - {sub} ({i+1}/{len(subjects_to_evaluate)})...")
|
| 229 |
-
try:
|
| 230 |
-
accuracy, subject_details = evaluate_single_subject(generator, current_dataset_id, sub, sample_count, progress)
|
| 231 |
-
all_evaluation_results.extend(subject_details)
|
| 232 |
-
|
| 233 |
-
num_evaluated_samples = len(subject_details)
|
| 234 |
-
num_correct_in_subject = sum(d['is_correct'] for d in subject_details)
|
| 235 |
-
|
| 236 |
-
total_correct_overall += num_correct_in_subject
|
| 237 |
-
total_samples_overall += num_evaluated_samples
|
| 238 |
-
eval_summary_lines.append(f"- {benchmark_category} - {sub}: {accuracy:.2f}% ({num_correct_in_subject}/{num_evaluated_samples} samples)")
|
| 239 |
-
except Exception as e:
|
| 240 |
-
gr.Error(f"Skipping {benchmark_category} - {sub} due to an error: {e}")
|
| 241 |
-
eval_summary_lines.append(f"- {benchmark_category} - {sub}: Error during evaluation.")
|
| 242 |
-
continue
|
| 243 |
-
|
| 244 |
-
overall_accuracy = (total_correct_overall / total_samples_overall) * 100 if total_samples_overall > 0 else 0
|
| 245 |
-
score_string = f"Overall Average Accuracy for {benchmark_category}: {overall_accuracy:.2f}% across {total_samples_overall} total samples.\n\n"
|
| 246 |
-
score_string += "Detailed breakdown:\n" + "\n".join(eval_summary_lines)
|
| 247 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 248 |
else:
|
| 249 |
-
|
| 250 |
-
|
| 251 |
-
|
| 252 |
-
|
| 253 |
-
|
| 254 |
-
|
| 255 |
-
#
|
| 256 |
-
formatted_details = "\n\n".join([
|
| 257 |
-
(
|
| 258 |
-
f"### Question:\n{item['question']}\n\n"
|
| 259 |
-
+ f"**Choices:**\n" + "\n".join([f"{get_choice_letter(i)}. {c}" for i, c in enumerate(item['choices'])]) + "\n\n"
|
| 260 |
-
+ (f"**Note:** Reasoning models are currently not fully supported for single-letter extraction. The original model output followed:\n" if item.get('is_reasoning_model_output') else "")
|
| 261 |
-
+ f"**Model Raw Output:** {item['model_raw_output']}\n"
|
| 262 |
-
+ f"**Expected Answer:** {item['expected_answer_letter']}\n"
|
| 263 |
-
+ f"**Predicted Answer:** {item['predicted_answer_letter']}\n"
|
| 264 |
-
+ f"**Correct:** {'Yes' if item['is_correct'] else 'No'}"
|
| 265 |
-
)
|
| 266 |
-
for item in all_evaluation_results
|
| 267 |
-
])
|
| 268 |
-
|
| 269 |
-
# Record the evaluation result to a JSONL file for the leaderboard
|
| 270 |
record = {
|
| 271 |
"model_id": model_id,
|
| 272 |
"benchmark": benchmark_category,
|
| 273 |
-
"subject": subject_name,
|
| 274 |
"accuracy": overall_accuracy,
|
| 275 |
-
"
|
|
|
|
| 276 |
"timestamp": pd.Timestamp.now().isoformat()
|
| 277 |
}
|
| 278 |
-
with open(
|
| 279 |
f.write(json.dumps(record) + "\n")
|
| 280 |
-
|
| 281 |
gr.Info("Evaluation completed successfully!")
|
| 282 |
-
|
| 283 |
-
|
| 284 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 285 |
|
| 286 |
except Exception as e:
|
| 287 |
-
error_message =
|
| 288 |
-
|
| 289 |
-
gr.Error(
|
| 290 |
|
| 291 |
-
# Return updates for
|
| 292 |
-
return
|
| 293 |
-
|
| 294 |
-
|
| 295 |
-
|
| 296 |
-
|
| 297 |
-
|
| 298 |
-
|
| 299 |
-
|
| 300 |
-
|
| 301 |
-
|
| 302 |
-
|
| 303 |
-
|
| 304 |
-
|
| 305 |
-
|
| 306 |
-
except Exception as e:
|
| 307 |
-
gr.Error(f"Error saving file: {e}")
|
| 308 |
-
return None
|
| 309 |
|
| 310 |
def load_leaderboard(benchmark_filter):
|
| 311 |
"""
|
| 312 |
-
Loads evaluation data
|
| 313 |
-
|
| 314 |
"""
|
| 315 |
try:
|
| 316 |
-
|
| 317 |
-
|
| 318 |
-
|
| 319 |
-
df
|
| 320 |
-
df = df.dropna(subset=['accuracy'])
|
| 321 |
-
|
| 322 |
if df.empty:
|
| 323 |
-
|
| 324 |
-
return pd.DataFrame(columns=["Model ID", "Average Accuracy (%)"]).to_dict('records')
|
| 325 |
-
|
| 326 |
-
# Filter data based on the selected benchmark
|
| 327 |
-
df_filtered = df[df['benchmark'] == benchmark_filter]
|
| 328 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 329 |
if df_filtered.empty:
|
| 330 |
-
|
| 331 |
-
|
| 332 |
-
|
| 333 |
-
|
| 334 |
-
|
| 335 |
-
|
| 336 |
-
|
| 337 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 338 |
|
| 339 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 340 |
|
| 341 |
-
except FileNotFoundError:
|
| 342 |
-
gr.Warning("No evaluation data found yet. Run an evaluation to populate the leaderboard!")
|
| 343 |
-
return pd.DataFrame(columns=["Model ID", "Average Accuracy (%)"]).to_dict('records')
|
| 344 |
except Exception as e:
|
| 345 |
gr.Error(f"Error loading leaderboard: {e}")
|
| 346 |
-
traceback.print_exc()
|
| 347 |
-
return pd.DataFrame(columns=["Model ID", "
|
| 348 |
-
|
| 349 |
-
def update_subject_dropdown_choices(benchmark_category):
|
| 350 |
-
"""
|
| 351 |
-
Updates the choices for the subject dropdown based on the selected benchmark category.
|
| 352 |
-
"""
|
| 353 |
-
dataset_id_map = {
|
| 354 |
-
"MMLU": MMLU_DATASET,
|
| 355 |
-
"MMLU-Pro": MMLU_PRO_DATASET
|
| 356 |
-
}
|
| 357 |
-
selected_dataset_id = dataset_id_map.get(benchmark_category)
|
| 358 |
-
|
| 359 |
-
if selected_dataset_id and selected_dataset_id in ALL_BENCHMARK_SUBJECTS:
|
| 360 |
-
new_choices = ALL_BENCHMARK_SUBJECTS[selected_dataset_id]
|
| 361 |
-
# Set default value to "ALL" if available, otherwise the first subject
|
| 362 |
-
default_value = "ALL" if "ALL" in new_choices else (new_choices[0] if new_choices else None)
|
| 363 |
-
return gr.update(choices=new_choices, value=default_value)
|
| 364 |
-
else:
|
| 365 |
-
return gr.update(choices=[], value=None)
|
| 366 |
|
| 367 |
|
| 368 |
# --- Gradio Interface Definition ---
|
| 369 |
-
with gr.Blocks(css="""
|
| 370 |
-
/* Import Google Font - Inter */
|
| 371 |
-
@import url('https://fonts.com/css2?family=Inter:wght@300;400;500;600;700&display=swap');
|
| 372 |
-
|
| 373 |
-
/* General body and container styling */
|
| 374 |
-
body {
|
| 375 |
-
font-family: 'Inter', sans-serif;
|
| 376 |
-
background-color: #eef2f6; /* Lighter background */
|
| 377 |
-
margin: 0;
|
| 378 |
-
padding: 20px;
|
| 379 |
-
}
|
| 380 |
-
.gradio-container {
|
| 381 |
-
max-width: 1200px;
|
| 382 |
-
margin: 20px auto;
|
| 383 |
-
padding: 40px; /* Increased padding */
|
| 384 |
-
box-shadow: 0 10px 25px rgba(0,0,0,0.1); /* Softer, larger shadow */
|
| 385 |
-
border-radius: 15px; /* More rounded corners */
|
| 386 |
-
background-color: #ffffff;
|
| 387 |
-
border: 1px solid #e0e6ed; /* Subtle border */
|
| 388 |
-
}
|
| 389 |
-
|
| 390 |
-
/* Headings */
|
| 391 |
-
h1 {
|
| 392 |
-
color: #1a202c; /* Darker, more professional heading color */
|
| 393 |
-
text-align: center;
|
| 394 |
-
margin-bottom: 30px;
|
| 395 |
-
font-size: 2.8em; /* Slightly larger H1 */
|
| 396 |
-
font-weight: 700;
|
| 397 |
-
letter-spacing: -0.03em;
|
| 398 |
-
text-shadow: 1px 1px 2px rgba(0,0,0,0.05); /* Subtle text shadow */
|
| 399 |
-
}
|
| 400 |
-
h3 {
|
| 401 |
-
color: #2d3748;
|
| 402 |
-
font-size: 1.3em; /* Slightly larger H3 */
|
| 403 |
-
margin-bottom: 15px;
|
| 404 |
-
font-weight: 600;
|
| 405 |
-
}
|
| 406 |
|
| 407 |
-
|
| 408 |
-
|
| 409 |
-
|
| 410 |
-
|
| 411 |
-
|
| 412 |
-
|
| 413 |
-
|
| 414 |
-
|
| 415 |
-
|
| 416 |
-
font-size:
|
| 417 |
-
max-width: 800px; /* Constrain width for readability */
|
| 418 |
-
margin: 0 auto;
|
| 419 |
-
}
|
| 420 |
-
|
| 421 |
-
/* Buttons */
|
| 422 |
-
.gr-button {
|
| 423 |
-
background-color: #2f80ed; /* A vibrant, professional blue */
|
| 424 |
-
color: white;
|
| 425 |
-
border: none;
|
| 426 |
-
padding: 14px 30px; /* More padding */
|
| 427 |
-
border-radius: 10px; /* More rounded */
|
| 428 |
-
cursor: pointer;
|
| 429 |
-
transition: background-color 0.3s ease, transform 0.2s ease, box-shadow 0.2s ease;
|
| 430 |
-
font-size: 1.15em; /* Slightly larger font */
|
| 431 |
-
font-weight: 600;
|
| 432 |
-
box-shadow: 0 5px 15px rgba(0, 123, 255, 0.2); /* Enhanced shadow for primary button */
|
| 433 |
-
margin: 5px; /* Add some margin for spacing between buttons */
|
| 434 |
-
}
|
| 435 |
-
.gr-button:hover {
|
| 436 |
-
background-color: #1a6dcd; /* Darker blue on hover */
|
| 437 |
-
transform: translateY(-3px); /* More pronounced lift effect */
|
| 438 |
-
box-shadow: 0 8px 20px rgba(0, 123, 255, 0.3);
|
| 439 |
-
}
|
| 440 |
-
.gr-button:active {
|
| 441 |
-
transform: translateY(0);
|
| 442 |
-
box-shadow: 0 2px 5px rgba(0,0,0,0.1);
|
| 443 |
-
}
|
| 444 |
-
/* Specific button styling for debug/show details */
|
| 445 |
-
#debug-button, #show-details-button {
|
| 446 |
-
background-color: #718096; /* Professional grey */
|
| 447 |
-
box-shadow: 0 3px 10px rgba(113, 128, 150, 0.2);
|
| 448 |
-
}
|
| 449 |
-
#debug-button:hover, #show-details-button:hover {
|
| 450 |
-
background-color: #5d6d81;
|
| 451 |
-
box-shadow: 0 5px 12px rgba(113, 128, 150, 0.3);
|
| 452 |
-
}
|
| 453 |
-
#download-button {
|
| 454 |
-
background-color: #38a169; /* Muted green for download */
|
| 455 |
-
box-shadow: 0 3px 10px rgba(56, 161, 105, 0.2);
|
| 456 |
-
}
|
| 457 |
-
#download-button:hover {
|
| 458 |
-
background-color: #277e50;
|
| 459 |
-
box-shadow: 0 5px 12px rgba(56, 161, 105, 0.3);
|
| 460 |
-
}
|
| 461 |
-
|
| 462 |
-
/* Input/Output Boxes (Containers) */
|
| 463 |
-
.gr-box {
|
| 464 |
-
border: 1px solid #cbd5e0; /* Lighter, subtle border */
|
| 465 |
-
border-radius: 12px;
|
| 466 |
-
padding: 25px; /* Increased padding */
|
| 467 |
-
margin-bottom: 25px;
|
| 468 |
-
background-color: #f8fafc; /* Very light background */
|
| 469 |
-
box-shadow: inset 0 2px 5px rgba(0,0,0,0.03); /* Subtle inner shadow */
|
| 470 |
-
}
|
| 471 |
-
/* Specific text output boxes (the content inside the containers) */
|
| 472 |
-
.gr-output-text {
|
| 473 |
-
white-space: pre-wrap;
|
| 474 |
-
word-wrap: break-word;
|
| 475 |
-
background-color: #ffffff; /* White background for readability */
|
| 476 |
-
border: 1px solid #e2e8f0;
|
| 477 |
-
border-radius: 8px;
|
| 478 |
-
padding: 18px; /* More padding */
|
| 479 |
-
min-height: 120px; /* Ensure a minimum height */
|
| 480 |
-
box-shadow: 0 2px 8px rgba(0,0,0,0.05); /* Small shadow for depth */
|
| 481 |
-
color: #2d3748; /* Darker text for readability */
|
| 482 |
-
font-size: 0.95em;
|
| 483 |
-
line-height: 1.6;
|
| 484 |
-
}
|
| 485 |
-
/* Specific error output style */
|
| 486 |
-
#error-message-output {
|
| 487 |
-
background-color: #ffe0e6; /* Light red */
|
| 488 |
-
border-color: #ff99aa; /* Slightly darker red border */
|
| 489 |
-
color: #c53030; /* Stronger red text */
|
| 490 |
-
font-weight: 500;
|
| 491 |
-
padding: 20px;
|
| 492 |
-
}
|
| 493 |
-
|
| 494 |
-
|
| 495 |
-
/* Labels for inputs */
|
| 496 |
-
.gr-textbox label, .gr-dropdown label, .gr-slider label {
|
| 497 |
-
font-weight: 600;
|
| 498 |
-
color: #2d3748; /* Darker label text */
|
| 499 |
-
margin-bottom: 10px;
|
| 500 |
-
display: block;
|
| 501 |
-
font-size: 1.05em; /* Slightly larger label font */
|
| 502 |
-
}
|
| 503 |
-
|
| 504 |
-
/* Tabs styling */
|
| 505 |
-
.gr-tabs-nav button {
|
| 506 |
-
font-weight: 600;
|
| 507 |
-
font-size: 1.1em;
|
| 508 |
-
padding: 12px 25px; /* More padding for tabs */
|
| 509 |
-
border-top-left-radius: 10px;
|
| 510 |
-
border-top-right-radius: 10px;
|
| 511 |
-
background-color: #ebf4f8; /* Light blueish tab background */
|
| 512 |
-
color: #4a5568;
|
| 513 |
-
border: 1px solid #cce0eb; /* Subtle border for tabs */
|
| 514 |
-
border-bottom: none;
|
| 515 |
-
transition: background-color 0.3s ease, color 0.3s ease;
|
| 516 |
-
}
|
| 517 |
-
.gr-tabs-nav button.selected {
|
| 518 |
-
background-color: #ffffff; /* White for selected tab */
|
| 519 |
-
color: #2f80ed; /* Blue for selected text */
|
| 520 |
-
border-color: #2f80ed;
|
| 521 |
-
border-bottom: 1px solid #ffffff; /* Hide bottom border to merge with content */
|
| 522 |
-
}
|
| 523 |
-
|
| 524 |
-
/* Leaderboard specific table styling (general for all leaderboard tables) */
|
| 525 |
-
.leaderboard-table {
|
| 526 |
-
border-radius: 12px;
|
| 527 |
-
box-shadow: 0 4px 15px rgba(0,0,0,0.08);
|
| 528 |
-
overflow: hidden;
|
| 529 |
-
margin-bottom: 25px; /* Space between tables */
|
| 530 |
-
}
|
| 531 |
-
.leaderboard-table table {
|
| 532 |
-
border-collapse: separate;
|
| 533 |
-
border-spacing: 0;
|
| 534 |
-
width: 100%;
|
| 535 |
-
background-color: #ffffff;
|
| 536 |
-
}
|
| 537 |
-
.leaderboard-table thead th {
|
| 538 |
-
background-color: #edf2f7; /* Light grey header */
|
| 539 |
-
color: #2d3748;
|
| 540 |
font-weight: 700;
|
| 541 |
-
|
| 542 |
-
|
| 543 |
-
|
| 544 |
-
}
|
| 545 |
-
.leaderboard-table tbody tr {
|
| 546 |
-
transition: background-color 0.2s ease;
|
| 547 |
-
}
|
| 548 |
-
.leaderboard-table tbody tr:nth-child(odd) {
|
| 549 |
-
background-color: #f7fafc; /* Zebra striping */
|
| 550 |
-
}
|
| 551 |
-
.leaderboard-table tbody tr:hover {
|
| 552 |
-
background-color: #e6fffa; /* Light teal on hover for rows */
|
| 553 |
-
}
|
| 554 |
-
.leaderboard-table tbody td {
|
| 555 |
-
padding: 12px 20px;
|
| 556 |
-
border-bottom: 1px solid #ebf4f8;
|
| 557 |
-
color: #4a5568;
|
| 558 |
}
|
| 559 |
-
.
|
| 560 |
-
|
| 561 |
}
|
| 562 |
-
|
| 563 |
-
|
| 564 |
-
|
| 565 |
-
|
| 566 |
-
|
| 567 |
-
|
| 568 |
-
|
| 569 |
-
}
|
| 570 |
-
|
| 571 |
-
|
| 572 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 573 |
display: flex;
|
| 574 |
-
|
| 575 |
-
padding: 0px 0px 20px 0px; /* Reduced padding for more compact look */
|
| 576 |
}
|
| 577 |
-
#leaderboard-toggle
|
| 578 |
-
|
| 579 |
-
|
| 580 |
-
color: #2d3748;
|
| 581 |
-
padding: 10px 20px;
|
| 582 |
border-radius: 8px;
|
| 583 |
-
background-color: #edf2f7; /* Light background for unselected */
|
| 584 |
-
border: 1px solid #e2e8f0;
|
| 585 |
cursor: pointer;
|
| 586 |
-
transition:
|
| 587 |
-
|
| 588 |
-
|
| 589 |
-
|
| 590 |
-
|
| 591 |
-
|
| 592 |
-
|
| 593 |
-
|
| 594 |
-
|
| 595 |
-
|
| 596 |
-
|
| 597 |
-
|
| 598 |
-
#leaderboard-toggle label.gr-radio-label:hover {
|
| 599 |
-
background-color: #e2e8f0; /* Lighter grey on hover */
|
| 600 |
}
|
| 601 |
-
|
| 602 |
-
/*
|
| 603 |
-
|
| 604 |
-
|
| 605 |
-
|
| 606 |
-
margin-bottom: 20px; /* Space above dropdown */
|
| 607 |
}
|
| 608 |
-
|
| 609 |
-
|
| 610 |
-
|
| 611 |
-
|
| 612 |
-
|
| 613 |
-
|
| 614 |
-
|
| 615 |
-
border: 1px solid #d9e3ed;
|
| 616 |
-
cursor: pointer;
|
| 617 |
-
transition: all 0.3s ease;
|
| 618 |
-
margin: 0 5px;
|
| 619 |
}
|
| 620 |
-
|
| 621 |
-
background-color: #
|
| 622 |
-
color: white;
|
| 623 |
-
border-color: #48bb78;
|
| 624 |
-
box-shadow: 0 2px 8px rgba(72, 187, 120, 0.2);
|
| 625 |
}
|
| 626 |
-
|
| 627 |
-
background-color: #
|
| 628 |
}
|
| 629 |
-
|
| 630 |
-
|
|
|
|
| 631 |
}
|
| 632 |
|
| 633 |
-
|
|
|
|
|
|
|
|
|
|
| 634 |
""") as demo:
|
| 635 |
-
gr.Markdown(""
|
| 636 |
-
|
| 637 |
-
""")
|
| 638 |
|
| 639 |
with gr.Tabs():
|
|
|
|
| 640 |
with gr.TabItem("π Run Evaluation"):
|
| 641 |
-
gr.
|
| 642 |
-
|
| 643 |
-
|
| 644 |
-
|
| 645 |
-
|
| 646 |
-
|
| 647 |
-
|
| 648 |
-
|
| 649 |
-
|
| 650 |
-
|
| 651 |
-
|
| 652 |
-
|
| 653 |
-
|
| 654 |
-
|
| 655 |
-
|
| 656 |
-
|
| 657 |
-
|
| 658 |
-
|
| 659 |
-
|
| 660 |
-
|
| 661 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 662 |
interactive=True,
|
| 663 |
-
|
| 664 |
-
|
| 665 |
)
|
| 666 |
-
|
| 667 |
-
|
| 668 |
-
|
| 669 |
-
|
| 670 |
-
|
| 671 |
-
|
| 672 |
-
interactive=True,
|
| 673 |
-
min_width=400,
|
| 674 |
-
visible=False
|
| 675 |
-
)
|
| 676 |
-
sample_count_slider = gr.Slider(
|
| 677 |
-
label="Number of Samples per Subject (1-100)",
|
| 678 |
-
minimum=1,
|
| 679 |
-
maximum=100,
|
| 680 |
-
value=100,
|
| 681 |
-
step=1,
|
| 682 |
-
interactive=True,
|
| 683 |
-
min_width=200,
|
| 684 |
-
visible=False
|
| 685 |
-
)
|
| 686 |
-
run_button = gr.Button("Run Evaluation", elem_classes="gr-button")
|
| 687 |
-
|
| 688 |
-
gr.Markdown("<hr>") # Visual separator
|
| 689 |
-
|
| 690 |
-
with gr.Column(elem_classes="gr-box"):
|
| 691 |
-
acc_output = gr.Textbox(
|
| 692 |
-
label="Benchmark Accuracy Results",
|
| 693 |
-
interactive=False,
|
| 694 |
-
elem_classes="gr-output-text",
|
| 695 |
-
lines=5,
|
| 696 |
-
placeholder="Evaluation results will appear here."
|
| 697 |
)
|
| 698 |
-
|
| 699 |
-
# Define button click actions
|
| 700 |
-
run_button.click(
|
| 701 |
-
run_evaluation,
|
| 702 |
-
inputs=[model_id_input, benchmark_selection_radio, benchmark_subject_dropdown, sample_count_slider], # Updated inputs
|
| 703 |
-
outputs=[
|
| 704 |
-
acc_output
|
| 705 |
-
]
|
| 706 |
-
)
|
| 707 |
-
|
| 708 |
-
# Link benchmark selection radio to subject dropdown
|
| 709 |
-
benchmark_selection_radio.change(
|
| 710 |
-
update_subject_dropdown_choices,
|
| 711 |
-
inputs=[benchmark_selection_radio],
|
| 712 |
-
outputs=[benchmark_subject_dropdown]
|
| 713 |
-
)
|
| 714 |
|
| 715 |
-
|
| 716 |
-
|
| 717 |
-
|
| 718 |
-
|
| 719 |
-
|
| 720 |
-
|
| 721 |
-
|
| 722 |
-
|
| 723 |
-
|
| 724 |
-
|
| 725 |
-
|
| 726 |
-
|
| 727 |
-
|
| 728 |
-
|
| 729 |
-
|
| 730 |
-
|
| 731 |
-
|
| 732 |
-
|
| 733 |
-
|
| 734 |
-
|
| 735 |
-
|
| 736 |
-
|
| 737 |
-
|
| 738 |
-
|
| 739 |
-
|
| 740 |
-
|
| 741 |
-
|
| 742 |
-
|
| 743 |
-
|
| 744 |
-
|
| 745 |
-
|
| 746 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 747 |
|
| 748 |
# Launch the Gradio app
|
| 749 |
-
|
|
|
|
|
|
| 6 |
import re
|
| 7 |
import json
|
| 8 |
import pandas as pd
|
| 9 |
+
import traceback
|
| 10 |
+
import spaces
|
|
|
|
| 11 |
|
| 12 |
+
# --- Environment and Caching ---
|
| 13 |
+
|
| 14 |
+
# It's good practice to ensure the cache directory exists.
|
| 15 |
+
CACHE_DIR = "evaluation_cache"
|
| 16 |
+
os.makedirs(CACHE_DIR, exist_ok=True)
|
| 17 |
+
EVAL_FILE = os.path.join(CACHE_DIR, "eval.jsonl")
|
| 18 |
+
|
| 19 |
+
# Cache to avoid reloading models and dataset configs
|
| 20 |
model_cache = {}
|
| 21 |
+
benchmark_subject_cache = {}
|
| 22 |
|
| 23 |
+
# Use environment variable for the Hugging Face token
|
| 24 |
HF_TOKEN = os.environ.get("HF_TOKEN")
|
| 25 |
|
| 26 |
# --- Constants for Benchmarks ---
|
| 27 |
MMLU_DATASET = "cais/mmlu"
|
| 28 |
MMLU_PRO_DATASET = "TIGER-Lab/MMLU-Pro"
|
| 29 |
+
BENCHMARK_MAP = {
|
| 30 |
+
"MMLU": MMLU_DATASET,
|
| 31 |
+
"MMLU-Pro": MMLU_PRO_DATASET
|
| 32 |
+
}
|
| 33 |
|
| 34 |
+
# --- Data Loading and Preparation ---
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
|
| 36 |
+
def get_all_benchmark_options():
|
| 37 |
+
"""
|
| 38 |
+
Fetches and caches the available subjects (configs) for each benchmark dataset.
|
| 39 |
+
This function now populates a global cache to avoid repeated API calls.
|
| 40 |
+
"""
|
| 41 |
+
if benchmark_subject_cache:
|
| 42 |
+
return benchmark_subject_cache
|
| 43 |
+
|
| 44 |
+
print("Fetching benchmark configurations for the first time...")
|
| 45 |
+
for key, dataset_id in BENCHMARK_MAP.items():
|
| 46 |
+
try:
|
| 47 |
+
# Fetching dataset configurations requires authentication if the dataset is private
|
| 48 |
+
subjects = get_dataset_config_names(dataset_id, token=HF_TOKEN)
|
| 49 |
+
benchmark_subject_cache[key] = ["ALL"] + subjects
|
| 50 |
+
except Exception as e:
|
| 51 |
+
print(f"Warning: Could not load configs for {key} ({dataset_id}). It might be private or unavailable. Error: {e}")
|
| 52 |
+
benchmark_subject_cache[key] = []
|
| 53 |
+
print("Benchmark configurations cached.")
|
| 54 |
+
return benchmark_subject_cache
|
| 55 |
+
|
| 56 |
+
# Initialize the cache on startup
|
| 57 |
+
ALL_BENCHMARK_SUBJECTS = get_all_benchmark_options()
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
@spaces.GPU()
|
| 61 |
def load_model(model_id):
|
| 62 |
"""
|
| 63 |
+
Loads a Hugging Face model and tokenizer, creating a text-generation pipeline.
|
| 64 |
+
Uses a cache to avoid reloading models.
|
|
|
|
|
|
|
| 65 |
"""
|
| 66 |
+
if not model_id:
|
| 67 |
+
raise ValueError("Model ID cannot be empty.")
|
| 68 |
+
|
| 69 |
gr.Info(f"Attempting to load model: {model_id}...")
|
| 70 |
if model_id in model_cache:
|
| 71 |
+
gr.Info(f"Model '{model_id}' found in cache.")
|
| 72 |
return model_cache[model_id]
|
| 73 |
+
|
| 74 |
try:
|
| 75 |
+
# Use bfloat16 for better performance on modern GPUs
|
| 76 |
+
dtype = torch.bfloat16 if torch.cuda.is_available() and torch.cuda.is_bf16_supported() else torch.float32
|
| 77 |
+
|
| 78 |
tokenizer = AutoTokenizer.from_pretrained(model_id, token=HF_TOKEN, trust_remote_code=True)
|
| 79 |
model = AutoModelForCausalLM.from_pretrained(
|
| 80 |
model_id,
|
| 81 |
token=HF_TOKEN,
|
| 82 |
+
torch_dtype=dtype,
|
| 83 |
trust_remote_code=True
|
| 84 |
).to("cuda" if torch.cuda.is_available() else "cpu")
|
| 85 |
+
|
| 86 |
+
# Create the pipeline for text generation
|
| 87 |
+
generator = pipeline(
|
| 88 |
+
"text-generation",
|
| 89 |
+
model=model,
|
| 90 |
+
tokenizer=tokenizer,
|
| 91 |
+
device=0 if torch.cuda.is_available() else -1
|
| 92 |
+
)
|
| 93 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 94 |
model_cache[model_id] = generator
|
| 95 |
gr.Info(f"Model '{model_id}' loaded successfully.")
|
| 96 |
return generator
|
| 97 |
except Exception as e:
|
| 98 |
+
# Raise a more specific error to be caught by the main evaluation function
|
| 99 |
+
raise RuntimeError(f"Failed to load model '{model_id}'. Please verify the model ID and your Hugging Face token (if required). Error: {e}")
|
| 100 |
+
|
| 101 |
|
| 102 |
+
# --- Evaluation Logic ---
|
| 103 |
|
| 104 |
def format_prompt(item):
|
| 105 |
+
"""Formats the MMLU question and choices into a standardized prompt."""
|
| 106 |
+
prompt = f"Question: {item['question']}\n\nChoices:\nA. {item['choices'][0]}\nB. {item['choices'][1]}\nC. {item['choices'][2]}\nD. {item['choices'][3]}\n\nAnswer:"
|
| 107 |
+
return prompt, item['answer']
|
| 108 |
|
| 109 |
+
def get_choice_letter(index):
|
| 110 |
+
"""Converts a numerical choice index (0-3) to a letter (A-D)."""
|
| 111 |
+
return chr(ord('A') + index) if 0 <= index <= 3 else None
|
|
|
|
|
|
|
|
|
|
|
|
|
| 112 |
|
| 113 |
+
def extract_predicted_letter(output_text):
|
| 114 |
"""
|
| 115 |
+
Extracts the predicted letter from the model's output.
|
| 116 |
+
It looks for a letter (A, B, C, D) immediately following 'Answer:'.
|
| 117 |
"""
|
| 118 |
+
match = re.search(r"Answer:\s*([ABCD])", output_text, re.IGNORECASE)
|
| 119 |
+
return match.group(1).upper() if match else None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 120 |
|
| 121 |
def evaluate_single_subject(generator, dataset_id, subject, sample_count, progress):
|
| 122 |
"""
|
| 123 |
+
Evaluates a model on a specific subject from a dataset.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 124 |
"""
|
| 125 |
+
gr.Info(f"Loading dataset: {dataset_id} ({subject})...")
|
| 126 |
try:
|
| 127 |
+
# Load the 'test' split as it's standard for MMLU evaluation
|
| 128 |
+
dataset = load_dataset(dataset_id, subject, token=HF_TOKEN, split="test")
|
| 129 |
except Exception as e:
|
|
|
|
| 130 |
raise RuntimeError(f"Failed to load dataset '{dataset_id}' for subject '{subject}'. Error: {e}")
|
| 131 |
|
| 132 |
+
# Shuffle and select a subset of samples for evaluation
|
| 133 |
+
num_samples = min(sample_count, len(dataset))
|
| 134 |
+
dataset = dataset.shuffle(seed=42).select(range(num_samples))
|
| 135 |
|
| 136 |
+
correct_predictions = 0
|
| 137 |
+
results_details = []
|
| 138 |
|
| 139 |
+
for item in progress.tqdm(dataset, desc=f"Evaluating {subject}"):
|
| 140 |
+
prompt, correct_answer_idx = format_prompt(item)
|
| 141 |
+
expected_letter = get_choice_letter(correct_answer_idx)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 142 |
|
| 143 |
+
# Generate a short response, aiming for a single letter answer.
|
| 144 |
+
# do_sample=False (greedy decoding) is crucial for reproducibility.
|
| 145 |
+
raw_output = generator(prompt, max_new_tokens=5, do_sample=False)[0]["generated_text"]
|
| 146 |
|
| 147 |
+
predicted_letter = extract_predicted_letter(raw_output)
|
|
|
|
|
|
|
| 148 |
is_correct = (predicted_letter == expected_letter)
|
|
|
|
| 149 |
|
| 150 |
+
if is_correct:
|
| 151 |
+
correct_predictions += 1
|
| 152 |
+
|
| 153 |
+
results_details.append({
|
| 154 |
"question": item['question'],
|
| 155 |
"choices": item['choices'],
|
| 156 |
+
"raw_output": raw_output.strip(),
|
| 157 |
+
"expected_letter": expected_letter,
|
| 158 |
+
"predicted_letter": predicted_letter,
|
| 159 |
"is_correct": is_correct,
|
|
|
|
| 160 |
})
|
|
|
|
|
|
|
|
|
|
|
|
|
| 161 |
|
| 162 |
+
accuracy = (correct_predictions / num_samples) * 100 if num_samples > 0 else 0
|
| 163 |
+
return accuracy, results_details
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 164 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 165 |
|
| 166 |
+
@spaces.GPU()
|
| 167 |
+
def run_evaluation(model_id, benchmark_category, subject_name, sample_count, progress=gr.Progress(track_tqdm=True)):
|
| 168 |
+
"""
|
| 169 |
+
Main function to orchestrate the entire evaluation process.
|
| 170 |
+
Handles single subject or 'ALL' subjects evaluation.
|
| 171 |
+
Returns updates for Gradio UI components.
|
| 172 |
+
"""
|
| 173 |
try:
|
| 174 |
+
gr.Info("Starting evaluation...")
|
| 175 |
+
generator = load_model(model_id)
|
| 176 |
|
| 177 |
+
dataset_id = BENCHMARK_MAP.get(benchmark_category)
|
| 178 |
+
if not dataset_id:
|
| 179 |
+
raise ValueError(f"Invalid benchmark category: {benchmark_category}")
|
| 180 |
+
|
| 181 |
+
all_results_details = []
|
| 182 |
+
summary_lines = []
|
| 183 |
+
total_correct = 0
|
| 184 |
+
total_samples = 0
|
| 185 |
+
|
| 186 |
+
subjects_to_run = []
|
| 187 |
if subject_name == "ALL":
|
| 188 |
+
subjects_to_run = ALL_BENCHMARK_SUBJECTS.get(benchmark_category, [])
|
| 189 |
+
if "ALL" in subjects_to_run:
|
| 190 |
+
subjects_to_run.remove("ALL") # Remove 'ALL' from the list of subjects to run
|
| 191 |
+
else:
|
| 192 |
+
subjects_to_run = [subject_name]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 193 |
|
| 194 |
+
if not subjects_to_run:
|
| 195 |
+
gr.Warning(f"No subjects found for '{benchmark_category}'.")
|
| 196 |
+
return "", "", "", pd.DataFrame().to_dict('records')
|
| 197 |
+
|
| 198 |
+
for i, subject in enumerate(subjects_to_run):
|
| 199 |
+
gr.Info(f"Evaluating {benchmark_category} - {subject} ({i+1}/{len(subjects_to_run)})...")
|
| 200 |
+
try:
|
| 201 |
+
accuracy, subject_details = evaluate_single_subject(generator, dataset_id, subject, sample_count, progress)
|
| 202 |
+
|
| 203 |
+
all_results_details.extend(subject_details)
|
| 204 |
+
num_correct = sum(d['is_correct'] for d in subject_details)
|
| 205 |
+
num_evaluated = len(subject_details)
|
| 206 |
+
|
| 207 |
+
total_correct += num_correct
|
| 208 |
+
total_samples += num_evaluated
|
| 209 |
+
summary_lines.append(f"- **{subject}**: {accuracy:.2f}% ({num_correct}/{num_evaluated})")
|
| 210 |
+
|
| 211 |
+
except Exception as e:
|
| 212 |
+
gr.Error(f"Skipping {subject} due to an error: {e}")
|
| 213 |
+
summary_lines.append(f"- **{subject}**: Evaluation failed.")
|
| 214 |
+
continue
|
| 215 |
+
|
| 216 |
+
overall_accuracy = (total_correct / total_samples) * 100 if total_samples > 0 else 0
|
| 217 |
+
|
| 218 |
+
# --- Prepare Outputs ---
|
| 219 |
+
if subject_name == "ALL":
|
| 220 |
+
result_summary = f"### Overall Average Accuracy for {benchmark_category}: {overall_accuracy:.2f}%\n"
|
| 221 |
+
result_summary += "across {:,} total samples.\n\n---\n\n**Breakdown by Subject:**\n".format(total_samples)
|
| 222 |
+
result_summary += "\n".join(summary_lines)
|
| 223 |
else:
|
| 224 |
+
result_summary = f"### Accuracy for {benchmark_category} - {subject_name}: {overall_accuracy:.2f}%\n"
|
| 225 |
+
result_summary += "({:,}/{:,} correct)".format(total_correct, total_samples)
|
| 226 |
+
|
| 227 |
+
# Create a detailed DataFrame for inspection
|
| 228 |
+
df_details = pd.DataFrame(all_results_details)
|
| 229 |
+
|
| 230 |
+
# Save results for leaderboard
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 231 |
record = {
|
| 232 |
"model_id": model_id,
|
| 233 |
"benchmark": benchmark_category,
|
|
|
|
| 234 |
"accuracy": overall_accuracy,
|
| 235 |
+
"subject": subject_name,
|
| 236 |
+
"sample_count": total_samples,
|
| 237 |
"timestamp": pd.Timestamp.now().isoformat()
|
| 238 |
}
|
| 239 |
+
with open(EVAL_FILE, "a") as f:
|
| 240 |
f.write(json.dumps(record) + "\n")
|
| 241 |
+
|
| 242 |
gr.Info("Evaluation completed successfully!")
|
| 243 |
+
|
| 244 |
+
# Return updates for the UI
|
| 245 |
+
return (
|
| 246 |
+
gr.update(value=result_summary, visible=True),
|
| 247 |
+
gr.update(value="", visible=False), # Hide error message
|
| 248 |
+
gr.update(value="", visible=False), # Hide error details
|
| 249 |
+
gr.update(value=df_details.to_dict('records'), visible=True) # Show detailed results table
|
| 250 |
+
)
|
| 251 |
|
| 252 |
except Exception as e:
|
| 253 |
+
error_message = f"An unexpected error occurred: {e}"
|
| 254 |
+
error_details = traceback.format_exc()
|
| 255 |
+
gr.Error(error_message)
|
| 256 |
|
| 257 |
+
# Return error updates for the UI
|
| 258 |
+
return (
|
| 259 |
+
gr.update(value="", visible=False), # Hide results summary
|
| 260 |
+
gr.update(value=error_message, visible=True),
|
| 261 |
+
gr.update(value=error_details, visible=True),
|
| 262 |
+
gr.update(value=pd.DataFrame().to_dict('records'), visible=False) # Hide detailed results
|
| 263 |
+
)
|
| 264 |
+
|
| 265 |
+
# --- UI Helper Functions ---
|
| 266 |
+
|
| 267 |
+
def update_subject_dropdown(benchmark_category):
|
| 268 |
+
"""Updates the subject dropdown choices based on the selected benchmark."""
|
| 269 |
+
choices = ALL_BENCHMARK_SUBJECTS.get(benchmark_category, [])
|
| 270 |
+
default_value = "ALL" if "ALL" in choices else (choices[0] if choices else None)
|
| 271 |
+
return gr.update(choices=choices, value=default_value)
|
|
|
|
|
|
|
|
|
|
| 272 |
|
| 273 |
def load_leaderboard(benchmark_filter):
|
| 274 |
"""
|
| 275 |
+
Loads and processes evaluation data to display on the leaderboard.
|
| 276 |
+
It now correctly averages scores for models that were evaluated on 'ALL' subjects.
|
| 277 |
"""
|
| 278 |
try:
|
| 279 |
+
if not os.path.exists(EVAL_FILE):
|
| 280 |
+
return pd.DataFrame(columns=["Model ID", "Avg. Accuracy (%)", "Total Samples"]).to_dict('records')
|
| 281 |
+
|
| 282 |
+
df = pd.read_json(EVAL_FILE, lines=True)
|
|
|
|
|
|
|
| 283 |
if df.empty:
|
| 284 |
+
return pd.DataFrame(columns=["Model ID", "Avg. Accuracy (%)", "Total Samples"]).to_dict('records')
|
|
|
|
|
|
|
|
|
|
|
|
|
| 285 |
|
| 286 |
+
# Coerce accuracy to numeric and filter valid entries
|
| 287 |
+
df['accuracy'] = pd.to_numeric(df['accuracy'], errors='coerce')
|
| 288 |
+
df.dropna(subset=['accuracy'], inplace=True)
|
| 289 |
+
|
| 290 |
+
# Filter by the selected benchmark (e.g., MMLU or MMLU-Pro)
|
| 291 |
+
df_filtered = df[df['benchmark'] == benchmark_filter].copy()
|
| 292 |
+
|
| 293 |
if df_filtered.empty:
|
| 294 |
+
return pd.DataFrame(columns=["Model ID", "Avg. Accuracy (%)", "Total Samples"]).to_dict('records')
|
| 295 |
+
|
| 296 |
+
# We are interested in the 'ALL' subject evaluations for the main leaderboard
|
| 297 |
+
df_all = df_filtered[df_filtered['subject'] == 'ALL'].copy()
|
| 298 |
+
|
| 299 |
+
if df_all.empty:
|
| 300 |
+
return pd.DataFrame(columns=["Model ID", "Avg. Accuracy (%)", "Total Samples"]).to_dict('records')
|
| 301 |
+
|
| 302 |
+
# Find the latest evaluation for each model
|
| 303 |
+
df_all['timestamp'] = pd.to_datetime(df_all['timestamp'])
|
| 304 |
+
latest_evals = df_all.loc[df_all.groupby('model_id')['timestamp'].idxmax()]
|
| 305 |
+
|
| 306 |
+
leaderboard_df = latest_evals[['model_id', 'accuracy', 'sample_count']].copy()
|
| 307 |
+
leaderboard_df.columns = ["Model ID", "Avg. Accuracy (%)", "Total Samples"]
|
| 308 |
|
| 309 |
+
# Format accuracy to 2 decimal places
|
| 310 |
+
leaderboard_df["Avg. Accuracy (%)"] = leaderboard_df["Avg. Accuracy (%)"].map('{:.2f}'.format)
|
| 311 |
+
|
| 312 |
+
# Sort by accuracy
|
| 313 |
+
leaderboard_df = leaderboard_df.sort_values(by="Avg. Accuracy (%)", ascending=False)
|
| 314 |
+
|
| 315 |
+
return leaderboard_df.to_dict('records')
|
| 316 |
|
|
|
|
|
|
|
|
|
|
| 317 |
except Exception as e:
|
| 318 |
gr.Error(f"Error loading leaderboard: {e}")
|
| 319 |
+
traceback.print_exc()
|
| 320 |
+
return pd.DataFrame(columns=["Model ID", "Avg. Accuracy (%)", "Total Samples"]).to_dict('records')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 321 |
|
| 322 |
|
| 323 |
# --- Gradio Interface Definition ---
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 324 |
|
| 325 |
+
with gr.Blocks(theme=gr.themes.Soft(), css="""
|
| 326 |
+
/* --- Global & Layout --- */
|
| 327 |
+
body { font-family: 'Inter', sans-serif; background-color: #f8f9fa; }
|
| 328 |
+
.gradio-container { max-width: 1280px !important; margin: auto; }
|
| 329 |
+
.gr-box { border-radius: 12px !important; box-shadow: 0 4px 12px rgba(0,0,0,0.05) !important; border: 1px solid #e9ecef !important; }
|
| 330 |
+
|
| 331 |
+
/* --- Typography --- */
|
| 332 |
+
h1 {
|
| 333 |
+
text-align: center;
|
| 334 |
+
font-size: 2.5rem !important;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 335 |
font-weight: 700;
|
| 336 |
+
color: #212529;
|
| 337 |
+
margin-bottom: 0.5rem;
|
| 338 |
+
letter-spacing: -1px;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 339 |
}
|
| 340 |
+
.subtitle {
|
| 341 |
+
text-align: center; color: #6c757d; font-size: 1.1rem; margin-bottom: 2.5rem;
|
| 342 |
}
|
| 343 |
+
|
| 344 |
+
/* --- Buttons & Inputs --- */
|
| 345 |
+
.gr-button {
|
| 346 |
+
border-radius: 8px !important;
|
| 347 |
+
font-weight: 600 !important;
|
| 348 |
+
padding: 10px 20px !important;
|
| 349 |
+
transition: all 0.2s ease;
|
| 350 |
+
}
|
| 351 |
+
.gr-button-primary { box-shadow: 0 4px 10px rgba(0, 123, 255, 0.2); }
|
| 352 |
+
.gr-button-primary:hover { transform: translateY(-2px); box-shadow: 0 6px 15px rgba(0, 123, 255, 0.3); }
|
| 353 |
+
|
| 354 |
+
/* --- Custom Radio Buttons (Segmented Control) --- */
|
| 355 |
+
#leaderboard-toggle, #eval-benchmark-selection {
|
| 356 |
+
background-color: #e9ecef;
|
| 357 |
+
padding: 5px;
|
| 358 |
+
border-radius: 10px;
|
| 359 |
+
display: inline-flex;
|
| 360 |
+
margin: auto;
|
| 361 |
+
}
|
| 362 |
+
#leaderboard-toggle div.gr-form, #eval-benchmark-selection div.gr-form {
|
| 363 |
display: flex;
|
| 364 |
+
gap: 5px;
|
|
|
|
| 365 |
}
|
| 366 |
+
#leaderboard-toggle input[type='radio'], #eval-benchmark-selection input[type='radio'] { display: none; }
|
| 367 |
+
#leaderboard-toggle label, #eval-benchmark-selection label {
|
| 368 |
+
padding: 8px 16px;
|
|
|
|
|
|
|
| 369 |
border-radius: 8px;
|
|
|
|
|
|
|
| 370 |
cursor: pointer;
|
| 371 |
+
transition: background-color 0.3s, color 0.3s, box-shadow 0.3s;
|
| 372 |
+
font-weight: 500;
|
| 373 |
+
color: #495057;
|
| 374 |
+
background: transparent;
|
| 375 |
+
border: none;
|
| 376 |
+
box-shadow: none;
|
| 377 |
+
}
|
| 378 |
+
#leaderboard-toggle input[type='radio']:checked + label, #eval-benchmark-selection input[type='radio']:checked + label {
|
| 379 |
+
background-color: white;
|
| 380 |
+
color: #007bff;
|
| 381 |
+
font-weight: 600;
|
| 382 |
+
box-shadow: 0 2px 5px rgba(0,0,0,0.1);
|
|
|
|
|
|
|
| 383 |
}
|
| 384 |
+
|
| 385 |
+
/* --- Dataframe / Table Styling --- */
|
| 386 |
+
.leaderboard-table .gr-dataframe table {
|
| 387 |
+
border-collapse: collapse;
|
| 388 |
+
width: 100%;
|
|
|
|
| 389 |
}
|
| 390 |
+
.leaderboard-table .gr-dataframe thead th {
|
| 391 |
+
background-color: #f8f9fa !important;
|
| 392 |
+
color: #495057 !important;
|
| 393 |
+
font-weight: 600 !important;
|
| 394 |
+
text-align: left;
|
| 395 |
+
padding: 12px 15px;
|
| 396 |
+
border-bottom: 2px solid #dee2e6;
|
|
|
|
|
|
|
|
|
|
|
|
|
| 397 |
}
|
| 398 |
+
.leaderboard-table .gr-dataframe tbody tr:nth-of-type(even) {
|
| 399 |
+
background-color: #f8f9fa;
|
|
|
|
|
|
|
|
|
|
| 400 |
}
|
| 401 |
+
.leaderboard-table .gr-dataframe tbody tr:hover {
|
| 402 |
+
background-color: #e9ecef;
|
| 403 |
}
|
| 404 |
+
.leaderboard-table .gr-dataframe tbody td {
|
| 405 |
+
padding: 12px 15px;
|
| 406 |
+
border-bottom: 1px solid #dee2e6;
|
| 407 |
}
|
| 408 |
|
| 409 |
+
/* --- Error & Result Panes --- */
|
| 410 |
+
#error-display-box { background-color: #fff3f3; border-color: #ffc9c9; }
|
| 411 |
+
#error-display-box .gr-label { color: #d9480f !important; font-weight: 600; }
|
| 412 |
+
#result-summary-box { background-color: #f3f9ff; border-color: #cde4ff; }
|
| 413 |
""") as demo:
|
| 414 |
+
gr.Markdown("<h1>π€ Open LLM Evaluator</h1>")
|
| 415 |
+
gr.Markdown("<p class='subtitle'>Benchmark leading models on MMLU and MMLU-Pro. Your results contribute to a live leaderboard.</p>")
|
|
|
|
| 416 |
|
| 417 |
with gr.Tabs():
|
| 418 |
+
# --- Evaluation Tab ---
|
| 419 |
with gr.TabItem("π Run Evaluation"):
|
| 420 |
+
with gr.Row():
|
| 421 |
+
with gr.Column(scale=2):
|
| 422 |
+
with gr.Box():
|
| 423 |
+
gr.Markdown("### 1. Configure Evaluation")
|
| 424 |
+
model_id_input = gr.Textbox(
|
| 425 |
+
label="Hugging Face Model ID",
|
| 426 |
+
placeholder="e.g., meta-llama/Meta-Llama-3-8B-Instruct",
|
| 427 |
+
interactive=True
|
| 428 |
+
)
|
| 429 |
+
with gr.Row():
|
| 430 |
+
benchmark_selection_radio = gr.Radio(
|
| 431 |
+
["MMLU", "MMLU-Pro"],
|
| 432 |
+
label="Benchmark",
|
| 433 |
+
value="MMLU",
|
| 434 |
+
interactive=True,
|
| 435 |
+
elem_id="eval-benchmark-selection",
|
| 436 |
+
container=False
|
| 437 |
+
)
|
| 438 |
+
with gr.Row():
|
| 439 |
+
benchmark_subject_dropdown = gr.Dropdown(
|
| 440 |
+
label="Subject",
|
| 441 |
+
choices=ALL_BENCHMARK_SUBJECTS.get("MMLU", []),
|
| 442 |
+
value="ALL",
|
| 443 |
+
interactive=True
|
| 444 |
+
)
|
| 445 |
+
sample_count_slider = gr.Slider(
|
| 446 |
+
label="Samples per Subject",
|
| 447 |
+
minimum=5, maximum=100, value=25, step=5, interactive=True
|
| 448 |
+
)
|
| 449 |
+
|
| 450 |
+
run_button = gr.Button("Start Evaluation", variant="primary", scale=1)
|
| 451 |
+
|
| 452 |
+
with gr.Column(scale=3):
|
| 453 |
+
gr.Markdown("### 2. View Results")
|
| 454 |
+
|
| 455 |
+
# Panel for displaying the summary of results
|
| 456 |
+
with gr.Box(visible=False, elem_id="result-summary-box") as result_summary_box:
|
| 457 |
+
result_summary_output = gr.Markdown()
|
| 458 |
+
|
| 459 |
+
# Panel for displaying errors
|
| 460 |
+
with gr.Box(visible=False, elem_id="error-display-box") as error_box:
|
| 461 |
+
error_output = gr.Textbox(label="Error Message", interactive=False)
|
| 462 |
+
error_details_output = gr.Textbox(label="Error Details (Traceback)", interactive=False, lines=8)
|
| 463 |
+
|
| 464 |
+
# Panel for detailed, row-by-row results
|
| 465 |
+
with gr.Box(visible=False) as details_box:
|
| 466 |
+
gr.Markdown("#### Detailed Evaluation Log")
|
| 467 |
+
detailed_results_df = gr.Dataframe(
|
| 468 |
+
headers=["Question", "Correct", "Expected", "Predicted", "Raw Output"],
|
| 469 |
+
datatype=["str", "bool", "str", "str", "str"],
|
| 470 |
+
interactive=False,
|
| 471 |
+
row_count=10,
|
| 472 |
+
col_count=5
|
| 473 |
+
)
|
| 474 |
+
|
| 475 |
+
# --- Leaderboard Tab ---
|
| 476 |
+
with gr.TabItem("π Leaderboard"):
|
| 477 |
+
with gr.Column():
|
| 478 |
+
gr.Markdown("<div style='display: flex; justify-content: center; width: 100%; margin-bottom: 20px;'></div>", elem_id="leaderboard-toggle-container")
|
| 479 |
+
leaderboard_type_toggle = gr.Radio(
|
| 480 |
+
["MMLU", "MMLU-Pro"],
|
| 481 |
+
label="Select Benchmark",
|
| 482 |
+
value="MMLU",
|
| 483 |
interactive=True,
|
| 484 |
+
elem_id="leaderboard-toggle",
|
| 485 |
+
container=False
|
| 486 |
)
|
| 487 |
+
leaderboard_table_output = gr.Dataframe(
|
| 488 |
+
headers=["Model ID", "Avg. Accuracy (%)", "Total Samples"],
|
| 489 |
+
interactive=False,
|
| 490 |
+
datatype=["str", "str", "number"],
|
| 491 |
+
row_count=15,
|
| 492 |
+
elem_classes="leaderboard-table"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 493 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 494 |
|
| 495 |
+
# --- Event Handlers & Logic ---
|
| 496 |
+
|
| 497 |
+
# Update subject dropdown when benchmark type changes
|
| 498 |
+
benchmark_selection_radio.change(
|
| 499 |
+
fn=update_subject_dropdown,
|
| 500 |
+
inputs=[benchmark_selection_radio],
|
| 501 |
+
outputs=[benchmark_subject_dropdown]
|
| 502 |
+
)
|
| 503 |
+
|
| 504 |
+
# Main evaluation trigger
|
| 505 |
+
run_button.click(
|
| 506 |
+
fn=run_evaluation,
|
| 507 |
+
inputs=[model_id_input, benchmark_selection_radio, benchmark_subject_dropdown, sample_count_slider],
|
| 508 |
+
outputs=[result_summary_box, error_box, error_details_output, details_box]
|
| 509 |
+
).then(
|
| 510 |
+
# This chained function updates the component values *after* the visibility is set
|
| 511 |
+
lambda r, e, d, df: (r, e, d, df.to_dict('records')),
|
| 512 |
+
inputs=[result_summary_box, error_box, error_details_output, details_box],
|
| 513 |
+
outputs=[result_summary_output, error_output, error_details_output, detailed_results_df]
|
| 514 |
+
)
|
| 515 |
+
|
| 516 |
+
# Leaderboard loading logic
|
| 517 |
+
demo.load(
|
| 518 |
+
fn=load_leaderboard,
|
| 519 |
+
inputs=[leaderboard_type_toggle],
|
| 520 |
+
outputs=[leaderboard_table_output]
|
| 521 |
+
)
|
| 522 |
+
leaderboard_type_toggle.change(
|
| 523 |
+
fn=load_leaderboard,
|
| 524 |
+
inputs=[leaderboard_type_toggle],
|
| 525 |
+
outputs=[leaderboard_table_output]
|
| 526 |
+
)
|
| 527 |
+
|
| 528 |
+
# When the run button is clicked again, refresh the leaderboard
|
| 529 |
+
run_button.click(
|
| 530 |
+
fn=load_leaderboard,
|
| 531 |
+
inputs=[leaderboard_type_toggle],
|
| 532 |
+
outputs=[leaderboard_table_output]
|
| 533 |
+
)
|
| 534 |
|
| 535 |
# Launch the Gradio app
|
| 536 |
+
if __name__ == "__main__":
|
| 537 |
+
demo.launch(debug=True)
|