Initial release: Arabic Function Calling Leaderboard

afcl/app.py

@@ -36,6 +36,7 @@ The **Arabic Function Calling Leaderboard (AFCL)** evaluates Large Language Mode
 LEADERBOARD_COLUMNS = {
     "rank": {"label": "المرتبة", "label_en": "Rank", "type": "number"},
     "model": {"label": "النموذج", "label_en": "Model", "type": "str"},
+    "organization": {"label": "المنظمة", "label_en": "Organization", "type": "str"},
     "overall": {"label": "الدقة الكلية", "label_en": "Overall", "type": "number"},
     "simple": {"label": "بسيط", "label_en": "Simple", "type": "number"},
     "multiple": {"label": "متعدد", "label_en": "Multiple", "type": "number"},
@@ -43,88 +44,11 @@ LEADERBOARD_COLUMNS = {
     "parallel_multiple": {"label": "متوازي متعدد", "label_en": "Parallel Multiple", "type": "number"},
     "irrelevance": {"label": "اللا صلة", "label_en": "Irrelevance", "type": "number"},
     "dialect_handling": {"label": "اللهجات", "label_en": "Dialects", "type": "number"},
+    "status": {"label": "الحالة", "label_en": "Status", "type": "str"},
 }
 
-# Sample leaderboard data (will be replaced with actual results)
-SAMPLE_LEADERBOARD = [
-    {
-        "rank": 1,
-        "model": "GPT-4o",
-        "overall": 78.5,
-        "simple": 85.2,
-        "multiple": 80.1,
-        "parallel": 75.3,
-        "parallel_multiple": 72.4,
-        "irrelevance": 82.0,
-        "dialect_handling": 70.5,
-    },
-    {
-        "rank": 2,
-        "model": "Claude 3.5 Sonnet",
-        "overall": 76.2,
-        "simple": 83.5,
-        "multiple": 78.8,
-        "parallel": 73.2,
-        "parallel_multiple": 70.1,
-        "irrelevance": 80.5,
-        "dialect_handling": 68.2,
-    },
-    {
-        "rank": 3,
-        "model": "Jais-30B",
-        "overall": 72.8,
-        "simple": 78.5,
-        "multiple": 74.2,
-        "parallel": 70.8,
-        "parallel_multiple": 68.5,
-        "irrelevance": 75.2,
-        "dialect_handling": 72.0,
-    },
-    {
-        "rank": 4,
-        "model": "ALLaM-7B",
-        "overall": 68.5,
-        "simple": 75.2,
-        "multiple": 70.5,
-        "parallel": 65.8,
-        "parallel_multiple": 62.3,
-        "irrelevance": 70.8,
-        "dialect_handling": 68.5,
-    },
-    {
-        "rank": 5,
-        "model": "Qwen2.5-72B",
-        "overall": 74.1,
-        "simple": 80.5,
-        "multiple": 76.2,
-        "parallel": 72.5,
-        "parallel_multiple": 69.8,
-        "irrelevance": 77.5,
-        "dialect_handling": 65.2,
-    },
-    {
-        "rank": 6,
-        "model": "SILMA-9B",
-        "overall": 65.2,
-        "simple": 72.8,
-        "multiple": 68.5,
-        "parallel": 62.1,
-        "parallel_multiple": 58.5,
-        "irrelevance": 68.2,
-        "dialect_handling": 62.8,
-    },
-    {
-        "rank": 7,
-        "model": "Llama-3.1-70B",
-        "overall": 71.5,
-        "simple": 78.2,
-        "multiple": 73.5,
-        "parallel": 69.8,
-        "parallel_multiple": 66.2,
-        "irrelevance": 74.5,
-        "dialect_handling": 62.5,
-    },
-]
+# Empty sample - will load from file
+SAMPLE_LEADERBOARD = []
 
 
 def get_leaderboard_data() -> List[Dict]:
@@ -142,6 +66,11 @@ def format_leaderboard_dataframe(data: List[Dict], use_arabic: bool = True) -> p
     """Convert leaderboard data to pandas DataFrame."""
     df = pd.DataFrame(data)
 
+    # Select columns to display
+    display_cols = ["rank", "model", "organization", "overall", "simple", "multiple",
+                    "parallel", "parallel_multiple", "irrelevance", "dialect_handling", "status"]
+    df = df[[c for c in display_cols if c in df.columns]]
+
     # Rename columns based on language preference
     column_mapping = {}
     for col, info in LEADERBOARD_COLUMNS.items():
@@ -151,10 +80,15 @@ def format_leaderboard_dataframe(data: List[Dict], use_arabic: bool = True) -> p
 
     df = df.rename(columns=column_mapping)
 
-    # Format numeric columns
+    # Format numeric columns (show as percentage, but mark 0.0 as "Pending")
     for col in df.columns:
         if df[col].dtype in ['float64', 'float32']:
-            df[col] = df[col].apply(lambda x: f"{x:.1f}%")
+            df[col] = df[col].apply(lambda x: "⏳ Pending" if x == 0.0 else f"{x:.1f}%")
+
+    # Format status column
+    status_col = "الحالة" if use_arabic else "Status"
+    if status_col in df.columns:
+        df[status_col] = df[status_col].apply(lambda x: "⏳ قيد التقييم" if x == "pending" else "✅ مكتمل" if use_arabic else "⏳ Pending" if x == "pending" else "✅ Completed")
 
     return df
 
@@ -375,12 +309,12 @@ def create_app():
             gr.Markdown(f"""
             <div style="text-align: center; padding: 15px; background: #f5f5f5; border-radius: 8px;">
                 <div style="font-size: 2rem; font-weight: bold; color: #1a5f2a;">{len(data)}</div>
-                <div style="color: #666;">Models Evaluated | النماذج المقيّمة</div>
+                <div style="color: #666;">Models to Evaluate | النماذج للتقييم</div>
             </div>
             """)
             gr.Markdown("""
             <div style="text-align: center; padding: 15px; background: #f5f5f5; border-radius: 8px;">
-                <div style="font-size: 2rem; font-weight: bold; color: #1a5f2a;">1,470+</div>
+                <div style="font-size: 2rem; font-weight: bold; color: #1a5f2a;">1,470</div>
                 <div style="color: #666;">Test Samples | عينات الاختبار</div>
             </div>
             """)

data/leaderboard.json

@@ -1,107 +1,394 @@
 [
   {
     "rank": 1,
-    "model": "GPT-4o",
-    "overall": 78.5,
-    "simple": 85.2,
-    "multiple": 80.1,
-    "parallel": 75.3,
-    "parallel_multiple": 72.4,
-    "irrelevance": 82.0,
-    "dialect_handling": 70.5,
-    "java": 76.8,
-    "javascript": 74.2,
-    "rest": 79.5,
-    "sql": 77.3
+    "model": "Jais-30B-Chat",
+    "model_url": "https://huggingface.co/inceptionai/jais-30b-chat-v3",
+    "organization": "Inception AI",
+    "overall": 0.0,
+    "simple": 0.0,
+    "multiple": 0.0,
+    "parallel": 0.0,
+    "parallel_multiple": 0.0,
+    "irrelevance": 0.0,
+    "dialect_handling": 0.0,
+    "status": "pending"
   },
   {
     "rank": 2,
-    "model": "Claude 3.5 Sonnet",
-    "overall": 76.2,
-    "simple": 83.5,
-    "multiple": 78.8,
-    "parallel": 73.2,
-    "parallel_multiple": 70.1,
-    "irrelevance": 80.5,
-    "dialect_handling": 68.2,
-    "java": 75.2,
-    "javascript": 72.8,
-    "rest": 78.2,
-    "sql": 76.5
+    "model": "ALLaM-7B-Instruct",
+    "model_url": "https://huggingface.co/sdaia/allam-1-7b-instruct",
+    "organization": "SDAIA",
+    "overall": 0.0,
+    "simple": 0.0,
+    "multiple": 0.0,
+    "parallel": 0.0,
+    "parallel_multiple": 0.0,
+    "irrelevance": 0.0,
+    "dialect_handling": 0.0,
+    "status": "pending"
   },
   {
     "rank": 3,
-    "model": "Qwen2.5-72B",
-    "overall": 74.1,
-    "simple": 80.5,
-    "multiple": 76.2,
-    "parallel": 72.5,
-    "parallel_multiple": 69.8,
-    "irrelevance": 77.5,
-    "dialect_handling": 65.2,
-    "java": 72.5,
-    "javascript": 70.8,
-    "rest": 75.2,
-    "sql": 73.8
+    "model": "SILMA-9B-Instruct",
+    "model_url": "https://huggingface.co/silma-ai/SILMA-9B-Instruct-v1.0",
+    "organization": "Silma AI",
+    "overall": 0.0,
+    "simple": 0.0,
+    "multiple": 0.0,
+    "parallel": 0.0,
+    "parallel_multiple": 0.0,
+    "irrelevance": 0.0,
+    "dialect_handling": 0.0,
+    "status": "pending"
   },
   {
     "rank": 4,
-    "model": "Jais-30B",
-    "overall": 72.8,
-    "simple": 78.5,
-    "multiple": 74.2,
-    "parallel": 70.8,
-    "parallel_multiple": 68.5,
-    "irrelevance": 75.2,
-    "dialect_handling": 72.0,
-    "java": 68.5,
-    "javascript": 66.2,
-    "rest": 71.8,
-    "sql": 69.5
+    "model": "Fanar-Star-1.2B",
+    "model_url": "https://huggingface.co/QatarComputing/fanar-star-1.2b",
+    "organization": "Qatar Computing Research Institute",
+    "overall": 0.0,
+    "simple": 0.0,
+    "multiple": 0.0,
+    "parallel": 0.0,
+    "parallel_multiple": 0.0,
+    "irrelevance": 0.0,
+    "dialect_handling": 0.0,
+    "status": "pending"
   },
   {
     "rank": 5,
-    "model": "Llama-3.1-70B",
-    "overall": 71.5,
-    "simple": 78.2,
-    "multiple": 73.5,
-    "parallel": 69.8,
-    "parallel_multiple": 66.2,
-    "irrelevance": 74.5,
-    "dialect_handling": 62.5,
-    "java": 70.2,
-    "javascript": 68.5,
-    "rest": 73.5,
-    "sql": 71.2
+    "model": "Yehia-7B-Preview",
+    "model_url": "https://huggingface.co/Kira-Arabic/Yehia-7B-preview",
+    "organization": "Kira Arabic",
+    "overall": 0.0,
+    "simple": 0.0,
+    "multiple": 0.0,
+    "parallel": 0.0,
+    "parallel_multiple": 0.0,
+    "irrelevance": 0.0,
+    "dialect_handling": 0.0,
+    "status": "pending"
   },
   {
     "rank": 6,
-    "model": "ALLaM-7B",
-    "overall": 68.5,
-    "simple": 75.2,
-    "multiple": 70.5,
-    "parallel": 65.8,
-    "parallel_multiple": 62.3,
-    "irrelevance": 70.8,
-    "dialect_handling": 68.5,
-    "java": 62.5,
-    "javascript": 60.2,
-    "rest": 66.8,
-    "sql": 64.5
+    "model": "ArabianGPT-1B",
+    "model_url": "https://huggingface.co/ahmed-samir/arabiangpt-1b",
+    "organization": "Ahmed Samir",
+    "overall": 0.0,
+    "simple": 0.0,
+    "multiple": 0.0,
+    "parallel": 0.0,
+    "parallel_multiple": 0.0,
+    "irrelevance": 0.0,
+    "dialect_handling": 0.0,
+    "status": "pending"
   },
   {
     "rank": 7,
-    "model": "SILMA-9B",
-    "overall": 65.2,
-    "simple": 72.8,
-    "multiple": 68.5,
-    "parallel": 62.1,
-    "parallel_multiple": 58.5,
-    "irrelevance": 68.2,
-    "dialect_handling": 62.8,
-    "java": 58.5,
-    "javascript": 56.2,
-    "rest": 63.2,
-    "sql": 60.8
+    "model": "Qwen2.5-72B-Instruct",
+    "model_url": "https://huggingface.co/Qwen/Qwen2.5-72B-Instruct",
+    "organization": "Alibaba Qwen",
+    "overall": 0.0,
+    "simple": 0.0,
+    "multiple": 0.0,
+    "parallel": 0.0,
+    "parallel_multiple": 0.0,
+    "irrelevance": 0.0,
+    "dialect_handling": 0.0,
+    "status": "pending"
+  },
+  {
+    "rank": 8,
+    "model": "Qwen2.5-32B-Instruct",
+    "model_url": "https://huggingface.co/Qwen/Qwen2.5-32B-Instruct",
+    "organization": "Alibaba Qwen",
+    "overall": 0.0,
+    "simple": 0.0,
+    "multiple": 0.0,
+    "parallel": 0.0,
+    "parallel_multiple": 0.0,
+    "irrelevance": 0.0,
+    "dialect_handling": 0.0,
+    "status": "pending"
+  },
+  {
+    "rank": 9,
+    "model": "Qwen2.5-7B-Instruct",
+    "model_url": "https://huggingface.co/Qwen/Qwen2.5-7B-Instruct",
+    "organization": "Alibaba Qwen",
+    "overall": 0.0,
+    "simple": 0.0,
+    "multiple": 0.0,
+    "parallel": 0.0,
+    "parallel_multiple": 0.0,
+    "irrelevance": 0.0,
+    "dialect_handling": 0.0,
+    "status": "pending"
+  },
+  {
+    "rank": 10,
+    "model": "Llama-3.1-70B-Instruct",
+    "model_url": "https://huggingface.co/meta-llama/Llama-3.1-70B-Instruct",
+    "organization": "Meta",
+    "overall": 0.0,
+    "simple": 0.0,
+    "multiple": 0.0,
+    "parallel": 0.0,
+    "parallel_multiple": 0.0,
+    "irrelevance": 0.0,
+    "dialect_handling": 0.0,
+    "status": "pending"
+  },
+  {
+    "rank": 11,
+    "model": "Llama-3.1-8B-Instruct",
+    "model_url": "https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct",
+    "organization": "Meta",
+    "overall": 0.0,
+    "simple": 0.0,
+    "multiple": 0.0,
+    "parallel": 0.0,
+    "parallel_multiple": 0.0,
+    "irrelevance": 0.0,
+    "dialect_handling": 0.0,
+    "status": "pending"
+  },
+  {
+    "rank": 12,
+    "model": "Gemma-2-27B-IT",
+    "model_url": "https://huggingface.co/google/gemma-2-27b-it",
+    "organization": "Google",
+    "overall": 0.0,
+    "simple": 0.0,
+    "multiple": 0.0,
+    "parallel": 0.0,
+    "parallel_multiple": 0.0,
+    "irrelevance": 0.0,
+    "dialect_handling": 0.0,
+    "status": "pending"
+  },
+  {
+    "rank": 13,
+    "model": "Gemma-2-9B-IT",
+    "model_url": "https://huggingface.co/google/gemma-2-9b-it",
+    "organization": "Google",
+    "overall": 0.0,
+    "simple": 0.0,
+    "multiple": 0.0,
+    "parallel": 0.0,
+    "parallel_multiple": 0.0,
+    "irrelevance": 0.0,
+    "dialect_handling": 0.0,
+    "status": "pending"
+  },
+  {
+    "rank": 14,
+    "model": "Aya-Expanse-32B",
+    "model_url": "https://huggingface.co/CohereForAI/aya-expanse-32b",
+    "organization": "Cohere For AI",
+    "overall": 0.0,
+    "simple": 0.0,
+    "multiple": 0.0,
+    "parallel": 0.0,
+    "parallel_multiple": 0.0,
+    "irrelevance": 0.0,
+    "dialect_handling": 0.0,
+    "status": "pending"
+  },
+  {
+    "rank": 15,
+    "model": "Aya-Expanse-8B",
+    "model_url": "https://huggingface.co/CohereForAI/aya-expanse-8b",
+    "organization": "Cohere For AI",
+    "overall": 0.0,
+    "simple": 0.0,
+    "multiple": 0.0,
+    "parallel": 0.0,
+    "parallel_multiple": 0.0,
+    "irrelevance": 0.0,
+    "dialect_handling": 0.0,
+    "status": "pending"
+  },
+  {
+    "rank": 16,
+    "model": "c4ai-command-r7b-arabic",
+    "model_url": "https://huggingface.co/CohereForAI/c4ai-command-r7b-arabic-02-2025",
+    "organization": "Cohere For AI",
+    "overall": 0.0,
+    "simple": 0.0,
+    "multiple": 0.0,
+    "parallel": 0.0,
+    "parallel_multiple": 0.0,
+    "irrelevance": 0.0,
+    "dialect_handling": 0.0,
+    "status": "pending"
+  },
+  {
+    "rank": 17,
+    "model": "Falcon-180B-Chat",
+    "model_url": "https://huggingface.co/tiiuae/falcon-180B-chat",
+    "organization": "TII UAE",
+    "overall": 0.0,
+    "simple": 0.0,
+    "multiple": 0.0,
+    "parallel": 0.0,
+    "parallel_multiple": 0.0,
+    "irrelevance": 0.0,
+    "dialect_handling": 0.0,
+    "status": "pending"
+  },
+  {
+    "rank": 18,
+    "model": "Falcon-40B-Instruct",
+    "model_url": "https://huggingface.co/tiiuae/falcon-40b-instruct",
+    "organization": "TII UAE",
+    "overall": 0.0,
+    "simple": 0.0,
+    "multiple": 0.0,
+    "parallel": 0.0,
+    "parallel_multiple": 0.0,
+    "irrelevance": 0.0,
+    "dialect_handling": 0.0,
+    "status": "pending"
+  },
+  {
+    "rank": 19,
+    "model": "DeepSeek-V3",
+    "model_url": "https://huggingface.co/deepseek-ai/DeepSeek-V3",
+    "organization": "DeepSeek",
+    "overall": 0.0,
+    "simple": 0.0,
+    "multiple": 0.0,
+    "parallel": 0.0,
+    "parallel_multiple": 0.0,
+    "irrelevance": 0.0,
+    "dialect_handling": 0.0,
+    "status": "pending"
+  },
+  {
+    "rank": 20,
+    "model": "Mistral-Large-Instruct",
+    "model_url": "https://huggingface.co/mistralai/Mistral-Large-Instruct-2411",
+    "organization": "Mistral AI",
+    "overall": 0.0,
+    "simple": 0.0,
+    "multiple": 0.0,
+    "parallel": 0.0,
+    "parallel_multiple": 0.0,
+    "irrelevance": 0.0,
+    "dialect_handling": 0.0,
+    "status": "pending"
+  },
+  {
+    "rank": 21,
+    "model": "Mixtral-8x22B-Instruct",
+    "model_url": "https://huggingface.co/mistralai/Mixtral-8x22B-Instruct-v0.1",
+    "organization": "Mistral AI",
+    "overall": 0.0,
+    "simple": 0.0,
+    "multiple": 0.0,
+    "parallel": 0.0,
+    "parallel_multiple": 0.0,
+    "irrelevance": 0.0,
+    "dialect_handling": 0.0,
+    "status": "pending"
+  },
+  {
+    "rank": 22,
+    "model": "Phi-4",
+    "model_url": "https://huggingface.co/microsoft/phi-4",
+    "organization": "Microsoft",
+    "overall": 0.0,
+    "simple": 0.0,
+    "multiple": 0.0,
+    "parallel": 0.0,
+    "parallel_multiple": 0.0,
+    "irrelevance": 0.0,
+    "dialect_handling": 0.0,
+    "status": "pending"
+  },
+  {
+    "rank": 23,
+    "model": "BLOOM-176B",
+    "model_url": "https://huggingface.co/bigscience/bloom",
+    "organization": "BigScience",
+    "overall": 0.0,
+    "simple": 0.0,
+    "multiple": 0.0,
+    "parallel": 0.0,
+    "parallel_multiple": 0.0,
+    "irrelevance": 0.0,
+    "dialect_handling": 0.0,
+    "status": "pending"
+  },
+  {
+    "rank": 24,
+    "model": "BLOOMZ-7B1",
+    "model_url": "https://huggingface.co/bigscience/bloomz-7b1",
+    "organization": "BigScience",
+    "overall": 0.0,
+    "simple": 0.0,
+    "multiple": 0.0,
+    "parallel": 0.0,
+    "parallel_multiple": 0.0,
+    "irrelevance": 0.0,
+    "dialect_handling": 0.0,
+    "status": "pending"
+  },
+  {
+    "rank": 25,
+    "model": "AceGPT-13B-Chat",
+    "model_url": "https://huggingface.co/FreedomIntelligence/AceGPT-13B-chat",
+    "organization": "FreedomIntelligence",
+    "overall": 0.0,
+    "simple": 0.0,
+    "multiple": 0.0,
+    "parallel": 0.0,
+    "parallel_multiple": 0.0,
+    "irrelevance": 0.0,
+    "dialect_handling": 0.0,
+    "status": "pending"
+  },
+  {
+    "rank": 26,
+    "model": "AraGPT2-Mega",
+    "model_url": "https://huggingface.co/aubmindlab/aragpt2-mega",
+    "organization": "AUB MIND Lab",
+    "overall": 0.0,
+    "simple": 0.0,
+    "multiple": 0.0,
+    "parallel": 0.0,
+    "parallel_multiple": 0.0,
+    "irrelevance": 0.0,
+    "dialect_handling": 0.0,
+    "status": "pending"
+  },
+  {
+    "rank": 27,
+    "model": "Arabic-Llama-3.1-8B",
+    "model_url": "https://huggingface.co/Ammar-Arabi/Arabic-Llama-3.1-8B-Instruct",
+    "organization": "Ammar Arabi",
+    "overall": 0.0,
+    "simple": 0.0,
+    "multiple": 0.0,
+    "parallel": 0.0,
+    "parallel_multiple": 0.0,
+    "irrelevance": 0.0,
+    "dialect_handling": 0.0,
+    "status": "pending"
+  },
+  {
+    "rank": 28,
+    "model": "Llama3-8B-Arabic-Instruct",
+    "model_url": "https://huggingface.co/MahmoudAshraf/Llama3-8B-Arabic-instruct",
+    "organization": "Mahmoud Ashraf",
+    "overall": 0.0,
+    "simple": 0.0,
+    "multiple": 0.0,
+    "parallel": 0.0,
+    "parallel_multiple": 0.0,
+    "irrelevance": 0.0,
+    "dialect_handling": 0.0,
+    "status": "pending"
   }
 ]