{"data":{"category":"multimodal","id":"mmmu","name":"MMMU","rows":[{"conf":"medium","context":"MMMU Pro","model_id":"deepseek-v3-0324","org":"nvidia","rank":1,"root":"deepseek-ai/DeepSeek-V3-0324","score":82,"variant":"DeepSeek-V3-0324-NVFP4"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-26b-a4b","org":"google","rank":2,"root":"google/gemma-4-26B-A4B","score":76.9,"variant":"gemma-4-26B-A4B-it"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-31b","org":"google","rank":3,"root":"google/gemma-4-31B","score":76.9,"variant":"gemma-4-31B-it"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-e4b","org":"google","rank":4,"root":"google/gemma-4-E4B","score":76.9,"variant":"gemma-4-E4B-it"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-e2b","org":"google","rank":5,"root":"google/gemma-4-E2B","score":76.9,"variant":"gemma-4-E2B-it"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-12b","org":"google","rank":6,"root":"google/gemma-4-12B","score":76.9,"variant":"gemma-4-12B-it"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-31b-it-qat-q4-0-unquantized","org":"google","rank":7,"root":"google/gemma-4-31B-it-qat-q4_0-unquantized","score":76.9,"variant":"gemma-4-31B-it-qat-w4a16-ct"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-12b-it-qat-q4-0-unquantized","org":"google","rank":8,"root":"google/gemma-4-12B-it-qat-q4_0-unquantized","score":76.9,"variant":"gemma-4-12B-it-qat-w4a16-ct"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-31b-it-assistant","org":"google","rank":9,"root":"google/gemma-4-31B-it-assistant","score":76.9,"variant":"gemma-4-31B-it-assistant"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-26b-a4b-it","org":"unsloth","rank":10,"root":"google/gemma-4-26B-A4B-it","score":76.9,"variant":"gemma-4-26B-A4B-it-GGUF"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-26b-a4b","org":"google","rank":11,"root":"google/gemma-4-26B-A4B","score":76.9,"variant":"gemma-4-26B-A4B"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-31b","org":"google","rank":12,"root":"google/gemma-4-31B","score":76.9,"variant":"gemma-4-31B"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-12b-it","org":"unsloth","rank":13,"root":"google/gemma-4-12B-it","score":76.9,"variant":"gemma-4-12b-it-GGUF"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-12b-it","org":"google","rank":14,"root":"google/gemma-4-12B-it","score":76.9,"variant":"gemma-4-12B-it-qat-q4_0-unquantized"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-31b-it","org":"unsloth","rank":15,"root":"google/gemma-4-31B-it","score":76.9,"variant":"gemma-4-31B-it-GGUF"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-e4b-it","org":"unsloth","rank":16,"root":"google/gemma-4-E4B-it","score":76.9,"variant":"gemma-4-E4B-it-GGUF"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-e2b-it-qat-q4-0-unquantized","org":"google","rank":17,"root":"google/gemma-4-E2B-it-qat-q4_0-unquantized","score":76.9,"variant":"gemma-4-E2B-it-qat-w4a16-ct"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-26b-a4b-it-qat-q4-0-unquantized","org":"unsloth","rank":18,"root":"google/gemma-4-26B-A4B-it-qat-q4_0-unquantized","score":76.9,"variant":"gemma-4-26B-A4B-it-qat-GGUF"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-e4b-it","org":"unsloth","rank":19,"root":"google/gemma-4-E4B-it","score":76.9,"variant":"gemma-4-E4B-it-unsloth-bnb-4bit"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-e2b","org":"unsloth","rank":20,"root":"google/gemma-4-E2B-it","score":76.9,"variant":"gemma-4-E2B-it-GGUF"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-e4b-it-qat-q4-0-unquantized","org":"google","rank":21,"root":"google/gemma-4-E4B-it-qat-q4_0-unquantized","score":76.9,"variant":"gemma-4-E4B-it-qat-w4a16-ct"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-31b-it","org":"unsloth","rank":22,"root":"google/gemma-4-31B-it","score":76.9,"variant":"gemma-4-31B-it-unsloth-bnb-4bit"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-31b-it-qat-q4-0-unquantized","org":"unsloth","rank":23,"root":"google/gemma-4-31B-it-qat-q4_0-unquantized","score":76.9,"variant":"gemma-4-31B-it-qat-GGUF"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-12b","org":"google","rank":24,"root":"google/gemma-4-12B","score":76.9,"variant":"gemma-4-12B"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-e4b-it-qat-q4-0-unquantized","org":"unsloth","rank":25,"root":"google/gemma-4-E4B-it-qat-q4_0-unquantized","score":76.9,"variant":"gemma-4-E4B-it-qat-GGUF"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-26b-a4b-it-assistant","org":"google","rank":26,"root":"google/gemma-4-26B-A4B-it-assistant","score":76.9,"variant":"gemma-4-26B-A4B-it-assistant"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-12b-it-qat-q4-0-unquantized","org":"unsloth","rank":27,"root":"google/gemma-4-12B-it-qat-q4_0-unquantized","score":76.9,"variant":"gemma-4-12B-it-qat-GGUF"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-12b-it-qat-q4-0-unquantized","org":"google","rank":28,"root":"google/gemma-4-12B-it-qat-q4_0-unquantized","score":76.9,"variant":"gemma-4-12B-it-qat-q4_0-gguf"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-e4b","org":"google","rank":29,"root":"google/gemma-4-E4B","score":76.9,"variant":"gemma-4-E4B"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-e4b-it-qat-q4-0-unquantized","org":"google","rank":30,"root":"google/gemma-4-E4B-it-qat-q4_0-unquantized","score":76.9,"variant":"gemma-4-E4B-it-qat-q4_0-gguf"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-e2b-it-qat-q4-0-unquantized","org":"google","rank":31,"root":"google/gemma-4-E2B-it-qat-q4_0-unquantized","score":76.9,"variant":"gemma-4-E2B-it-qat-q4_0-gguf"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-12b-it-assistant","org":"google","rank":32,"root":"google/gemma-4-12B-it-assistant","score":76.9,"variant":"gemma-4-12B-it-assistant"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-e4b-it","org":"unsloth","rank":33,"root":"google/gemma-4-E4B-it","score":76.9,"variant":"gemma-4-E4B-it"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-12b-it","org":"unsloth","rank":34,"root":"google/gemma-4-12B-it","score":76.9,"variant":"gemma-4-12b-it"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-e2b-it-qat-q4-0-unquantized","org":"unsloth","rank":35,"root":"google/gemma-4-E2B-it-qat-q4_0-unquantized","score":76.9,"variant":"gemma-4-E2B-it-qat-GGUF"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-12b-it-assistant","org":"google","rank":36,"root":"google/gemma-4-12B-it-assistant","score":76.9,"variant":"gemma-4-12B-it-qat-q4_0-unquantized-assistant"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-26b-a4b-it-qat-q4-0-unquantized","org":"google","rank":37,"root":"google/gemma-4-26B-A4B-it-qat-q4_0-unquantized","score":76.9,"variant":"gemma-4-26B-A4B-it-qat-q4_0-gguf"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-e2b","org":"google","rank":38,"root":"google/gemma-4-E2B","score":76.9,"variant":"gemma-4-E2B"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-26b-a4b-it","org":"unsloth","rank":39,"root":"google/gemma-4-26B-A4B-it","score":76.9,"variant":"gemma-4-26B-A4B-it-NVFP4"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-e2b","org":"unsloth","rank":40,"root":"google/gemma-4-E2B-it","score":76.9,"variant":"gemma-4-E2B-it-unsloth-bnb-4bit"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-31b-it-qat-q4-0-unquantized","org":"google","rank":41,"root":"google/gemma-4-31B-it-qat-q4_0-unquantized","score":76.9,"variant":"gemma-4-31B-it-qat-q4_0-gguf"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-e2b","org":"unsloth","rank":42,"root":"google/gemma-4-E2B-it","score":76.9,"variant":"gemma-4-E2B-it"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-31b-it-qat-q4-0-unquantized","org":"unsloth","rank":43,"root":"google/gemma-4-31B-it-qat-q4_0-unquantized","score":76.9,"variant":"gemma-4-31B-it-qat-w4a16"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-e2b-it-assistant","org":"google","rank":44,"root":"google/gemma-4-E2B-it-assistant","score":76.9,"variant":"gemma-4-E2B-it-assistant"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-e4b-it-assistant","org":"google","rank":45,"root":"google/gemma-4-E4B-it-assistant","score":76.9,"variant":"gemma-4-E4B-it-assistant"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-26b-a4b-it","org":"unsloth","rank":46,"root":"google/gemma-4-26B-A4B-it","score":76.9,"variant":"gemma-4-26B-A4B-it"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-e2b-it-qat-mobile-transformers","org":"google","rank":47,"root":"google/gemma-4-E2B-it-qat-mobile-transformers","score":76.9,"variant":"gemma-4-E2B-it-qat-mobile-ct"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-e2b-it-qat-mobile-transformers","org":"unsloth","rank":48,"root":"google/gemma-4-E2B-it-qat-mobile-transformers","score":76.9,"variant":"gemma-4-E2B-it-qat-mobile-GGUF"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-26b-a4b-it","org":"google","rank":49,"root":"google/gemma-4-26B-A4B-it","score":76.9,"variant":"gemma-4-26B-A4B-it-qat-q4_0-unquantized"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-e2b","org":"google","rank":50,"root":"google/gemma-4-E2B-it","score":76.9,"variant":"gemma-4-E2B-it-qat-q4_0-unquantized"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-e4b-it-qat-mobile-transformers","org":"google","rank":51,"root":"google/gemma-4-E4B-it-qat-mobile-transformers","score":76.9,"variant":"gemma-4-E4B-it-qat-mobile-ct"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-12b-it","org":"unsloth","rank":52,"root":"google/gemma-4-12B-it","score":76.9,"variant":"gemma-4-12b-it-NVFP4"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-31b-it","org":"unsloth","rank":53,"root":"google/gemma-4-31B-it","score":76.9,"variant":"gemma-4-31B-it"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-e4b-it-qat-mobile-transformers","org":"unsloth","rank":54,"root":"google/gemma-4-E4B-it-qat-mobile-transformers","score":76.9,"variant":"gemma-4-E4B-it-qat-mobile-GGUF"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-31b-it","org":"unsloth","rank":55,"root":"google/gemma-4-31B-it","score":76.9,"variant":"gemma-4-31B-it-NVFP4"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-12b","org":"unsloth","rank":56,"root":"google/gemma-4-12B","score":76.9,"variant":"gemma-4-12b"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-e2b","org":"google","rank":57,"root":"google/gemma-4-E2B-it","score":76.9,"variant":"gemma-4-E2B-it-qat-mobile-transformers"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-e4b-it","org":"google","rank":58,"root":"google/gemma-4-E4B-it","score":76.9,"variant":"gemma-4-E4B-it-qat-mobile-transformers"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-31b-it-assistant","org":"google","rank":59,"root":"google/gemma-4-31B-it-assistant","score":76.9,"variant":"gemma-4-31B-it-qat-q4_0-unquantized-assistant"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-e4b-it","org":"google","rank":60,"root":"google/gemma-4-E4B-it","score":76.9,"variant":"gemma-4-E4B-it-qat-q4_0-unquantized"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-26b-a4b-it-assistant","org":"google","rank":61,"root":"google/gemma-4-26B-A4B-it-assistant","score":76.9,"variant":"gemma-4-26B-A4B-it-qat-q4_0-unquantized-assistant"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-31b-it","org":"google","rank":62,"root":"google/gemma-4-31B-it","score":76.9,"variant":"gemma-4-31B-it-qat-q4_0-unquantized"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-e4b-it","org":"unsloth","rank":63,"root":"google/gemma-4-E4B-it","score":76.9,"variant":"gemma-4-E4B-it-NVFP4"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-e2b-it-assistant","org":"google","rank":64,"root":"google/gemma-4-E2B-it-assistant","score":76.9,"variant":"gemma-4-E2B-it-qat-q4_0-unquantized-assistant"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-e4b-it-assistant","org":"google","rank":65,"root":"google/gemma-4-E4B-it-assistant","score":76.9,"variant":"gemma-4-E4B-it-qat-q4_0-unquantized-assistant"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-12b-it","org":"unsloth","rank":66,"root":"google/gemma-4-12B-it","score":76.9,"variant":"gemma-4-12B-it-qat-q4_0-unquantized"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-e2b","org":"unsloth","rank":67,"root":"google/gemma-4-E2B","score":76.9,"variant":"gemma-4-E2B"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-e2b","org":"unsloth","rank":68,"root":"google/gemma-4-E2B-it","score":76.9,"variant":"gemma-4-E2B-it-qat-q4_0-unquantized"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-e2b","org":"unsloth","rank":69,"root":"google/gemma-4-E2B-it","score":76.9,"variant":"gemma-4-E2B-it-NVFP4"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-12b-it-qat-q4-0-unquantized","org":"unsloth","rank":70,"root":"google/gemma-4-12B-it-qat-q4_0-unquantized","score":76.9,"variant":"gemma-4-12B-it-qat-w4a16"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-26b-a4b-it","org":"unsloth","rank":71,"root":"google/gemma-4-26B-A4B-it","score":76.9,"variant":"gemma-4-26B-A4B-it-qat-q4_0-unquantized"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-12b","org":"Mungert","rank":72,"root":"google/gemma-4-12B","score":76.9,"variant":"gemma-4-12B-it-GGUF"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-e4b-it-qat-q4-0-unquantized","org":"unsloth","rank":73,"root":"google/gemma-4-E4B-it-qat-q4_0-unquantized","score":76.9,"variant":"gemma-4-E4B-it-qat-w4a16"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-26b-a4b-it","org":"unsloth","rank":74,"root":"google/gemma-4-26B-A4B-it","score":76.9,"variant":"gemma-4-26b-a4b-it-UD-MLX-4bit"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-26b-a4b","org":"unsloth","rank":75,"root":"google/gemma-4-26B-A4B","score":76.9,"variant":"gemma-4-26B-A4B"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-31b-it","org":"unsloth","rank":76,"root":"google/gemma-4-31B-it","score":76.9,"variant":"gemma-4-31B-it-qat-q4_0-unquantized"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-e2b-it-qat-q4-0-unquantized","org":"unsloth","rank":77,"root":"google/gemma-4-E2B-it-qat-q4_0-unquantized","score":76.9,"variant":"gemma-4-E2B-it-qat-w4a16"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-e4b","org":"unsloth","rank":78,"root":"google/gemma-4-E4B","score":76.9,"variant":"gemma-4-E4B"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-e4b-it","org":"unsloth","rank":79,"root":"google/gemma-4-E4B-it","score":76.9,"variant":"gemma-4-E4B-it-qat-q4_0-unquantized"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-31b","org":"unsloth","rank":80,"root":"google/gemma-4-31B","score":76.9,"variant":"gemma-4-31B"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-e4b-it","org":"unsloth","rank":81,"root":"google/gemma-4-E4B-it","score":76.9,"variant":"gemma-4-E4B-it-UD-MLX-4bit"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-e2b","org":"unsloth","rank":82,"root":"google/gemma-4-E2B-it","score":76.9,"variant":"gemma-4-E2B-it-UD-MLX-4bit"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-31b-it","org":"unsloth","rank":83,"root":"google/gemma-4-31B-it","score":76.9,"variant":"gemma-4-31b-it-MLX-8bit"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-26b-a4b-it","org":"unsloth","rank":84,"root":"google/gemma-4-26B-A4B-it","score":76.9,"variant":"gemma-4-26b-a4b-it-MLX-8bit"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-31b","org":"Mungert","rank":85,"root":"google/gemma-4-31B","score":76.9,"variant":"gemma-4-31B-it-GGUF"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-e2b-it-qat-q4-0-unquantized","org":"unsloth","rank":86,"root":"google/gemma-4-E2B-it-qat-q4_0-unquantized","score":76.9,"variant":"gemma-4-E2B-it-qat-mobile"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-e4b-it","org":"unsloth","rank":87,"root":"google/gemma-4-E4B-it","score":76.9,"variant":"gemma-4-E4B-it-MLX-8bit"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-31b-it","org":"unsloth","rank":88,"root":"google/gemma-4-31B-it","score":76.9,"variant":"gemma-4-31b-it-UD-MLX-4bit"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-e4b","org":"Mungert","rank":89,"root":"google/gemma-4-E4B","score":76.9,"variant":"gemma-4-E4B-it-GGUF"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-e2b","org":"Mungert","rank":90,"root":"google/gemma-4-E2B","score":76.9,"variant":"gemma-4-E2B-it-GGUF"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-e2b","org":"unsloth","rank":91,"root":"google/gemma-4-E2B-it","score":76.9,"variant":"gemma-4-E2B-it-MLX-8bit"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-26b-a4b-it","org":"unsloth","rank":92,"root":"google/gemma-4-26B-A4B-it","score":76.9,"variant":"gemma-4-26b-a4b-it-UD-MLX-3bit"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-e4b-it-qat-q4-0-unquantized","org":"unsloth","rank":93,"root":"google/gemma-4-E4B-it-qat-q4_0-unquantized","score":76.9,"variant":"gemma-4-E4B-it-qat-mobile"},{"conf":"medium","context":"MMMU Pro","model_id":"gemma-4-31b-it","org":"unsloth","rank":94,"root":"google/gemma-4-31B-it","score":76.9,"variant":"gemma-4-31b-it-UD-MLX-3bit"},{"conf":"high","context":null,"model_id":"qwen3-5-122b-a10b","org":"nvidia","rank":95,"root":"Qwen/Qwen3.5-122B-A10B","score":75.55,"variant":"Qwen3.5-122B-A10B-NVFP4"},{"conf":"high","context":null,"model_id":"minimax-m3","org":"nvidia","rank":96,"root":"MiniMaxAI/MiniMax-M3","score":71.01,"variant":"MiniMax-M3-NVFP4"},{"conf":"medium","context":"MMMU<sub>val</sub>","model_id":"qwen2-5-vl-72b-instruct","org":"Qwen","rank":97,"root":"Qwen/Qwen2.5-VL-72B-Instruct","score":70.3,"variant":"Qwen2.5-VL-72B-Instruct"},{"conf":"medium","context":"MMMU<sub>val</sub>","model_id":"qwen2-5-vl-72b-instruct","org":"unsloth","rank":98,"root":"Qwen/Qwen2.5-VL-72B-Instruct","score":70.3,"variant":"Qwen2.5-VL-72B-Instruct-GGUF"},{"conf":"medium","context":"MMMU(val)","model_id":null,"org":"Qwen","rank":99,"root":"Qwen/Qwen2-VL-72B","score":70.3,"variant":"QVQ-72B-Preview"},{"conf":"medium","context":"MMMU<sub>val</sub>","model_id":"qwen2-5-vl-72b-instruct","org":"Mungert","rank":100,"root":"Qwen/Qwen2.5-VL-72B-Instruct","score":70.3,"variant":"Qwen2.5-VL-72B-Instruct-GGUF"},{"conf":"medium","context":"MMMU<sub>val</sub>","model_id":"qwen2-5-vl-72b-instruct","org":"unsloth","rank":101,"root":"Qwen/Qwen2.5-VL-72B-Instruct","score":70.3,"variant":"Qwen2.5-VL-72B-Instruct-bnb-4bit"},{"conf":"medium","context":"MMMU<sub>val</sub>","model_id":"qwen2-5-vl-72b-instruct","org":"unsloth","rank":102,"root":"Qwen/Qwen2.5-VL-72B-Instruct","score":70.3,"variant":"Qwen2.5-VL-72B-Instruct-unsloth-bnb-4bit"},{"conf":"medium","context":"MMMU","model_id":"qwen2-5-vl-32b-instruct","org":"Qwen","rank":103,"root":"Qwen/Qwen2.5-VL-32B-Instruct","score":70.2,"variant":"Qwen2.5-VL-32B-Instruct"},{"conf":"medium","context":"MMMU","model_id":"qwen2-5-vl-32b-instruct","org":"unsloth","rank":104,"root":"Qwen/Qwen2.5-VL-32B-Instruct","score":70.2,"variant":"Qwen2.5-VL-32B-Instruct-GGUF"},{"conf":"medium","context":"MMMU","model_id":"qwen2-5-vl-32b-instruct","org":"unsloth","rank":105,"root":"Qwen/Qwen2.5-VL-32B-Instruct","score":70.2,"variant":"Qwen2.5-VL-32B-Instruct-bnb-4bit"},{"conf":"medium","context":"MMMU","model_id":"qwen2-5-vl-32b-instruct","org":"Mungert","rank":106,"root":"Qwen/Qwen2.5-VL-32B-Instruct","score":70.2,"variant":"Qwen2.5-VL-32B-Instruct-GGUF"},{"conf":"medium","context":"MMMU","model_id":"qwen2-5-vl-32b-instruct","org":"unsloth","rank":107,"root":"Qwen/Qwen2.5-VL-32B-Instruct","score":70.2,"variant":"Qwen2.5-VL-32B-Instruct-unsloth-bnb-4bit"},{"conf":"high","context":null,"model_id":"qwen2-5-vl-32b-instruct","org":"Qwen","rank":108,"root":"Qwen/Qwen2.5-VL-32B-Instruct","score":70,"variant":"Qwen2.5-VL-32B-Instruct-AWQ"},{"conf":"low","context":null,"model_id":null,"org":"zai-org","rank":109,"root":"zai-org/glm-4v-9b","score":69.2,"variant":"glm-4v-9b"},{"conf":"medium","context":"MMMU (val) (Pass@1)","model_id":"kimi-vl-a3b-instruct","org":"moonshotai","rank":110,"root":"moonshotai/Kimi-VL-A3B-Instruct","score":69.1,"variant":"Kimi-VL-A3B-Thinking"},{"conf":"medium","context":"MMMU<sub>val</sub>","model_id":"qwen2-5-7b-instruct","org":"nvidia","rank":111,"root":"Qwen/Qwen2.5-7B-Instruct","score":69.1,"variant":"Eagle2.5-8B"},{"conf":"medium","context":"MMMU (val, Pass@1)","model_id":"kimi-vl-a3b-instruct","org":"moonshotai","rank":112,"root":"moonshotai/Kimi-VL-A3B-Instruct","score":69.1,"variant":"Kimi-VL-A3B-Thinking-2506"},{"conf":"medium","context":"MMMU*","model_id":null,"org":"nvidia","rank":113,"root":"nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16","score":68,"variant":"NVIDIA-Nemotron-Nano-12B-v2-VL-BF16"},{"conf":"high","context":null,"model_id":null,"org":"inclusionAI","rank":114,"root":"inclusionAI/Ming-Lite-Uni","score":67.9,"variant":"Ming-Lite-Uni"},{"conf":"high","context":null,"model_id":"qwen2-vl-72b-instruct","org":"Qwen","rank":115,"root":"Qwen/Qwen2-VL-72B-Instruct","score":65.44,"variant":"Qwen2-VL-72B-Instruct-AWQ"},{"conf":"high","context":null,"model_id":"qwen2-vl-72b-instruct","org":"Qwen","rank":116,"root":"Qwen/Qwen2-VL-72B-Instruct","score":65.44,"variant":"Qwen2-VL-72B-Instruct-GPTQ-Int4"},{"conf":"high","context":null,"model_id":"qwen2-vl-72b-instruct","org":"Qwen","rank":117,"root":"Qwen/Qwen2-VL-72B-Instruct","score":65.44,"variant":"Qwen2-VL-72B-Instruct-GPTQ-Int8"},{"conf":"low","context":null,"model_id":"mistral-small-3-1-24b-base-2503","org":"mistralai","rank":118,"root":"mistralai/Mistral-Small-3.1-24B-Base-2503","score":64,"variant":"Mistral-Small-3.1-24B-Instruct-2503"},{"conf":"low","context":null,"model_id":"mistral-small-3-1-24b-base-2503","org":"mistralai","rank":119,"root":"mistralai/Mistral-Small-3.1-24B-Base-2503","score":64,"variant":"Mistral-Small-3.2-24B-Instruct-2506"},{"conf":"low","context":null,"model_id":"mistral-small-3-1-24b-base-2503","org":"unsloth","rank":120,"root":"mistralai/Mistral-Small-3.2-24B-Instruct-2506","score":64,"variant":"Mistral-Small-3.2-24B-Instruct-2506-GGUF"},{"conf":"low","context":null,"model_id":"mistral-small-3-1-24b-base-2503","org":"unsloth","rank":121,"root":"mistralai/Mistral-Small-3.2-24B-Instruct-2506","score":64,"variant":"Mistral-Small-3.2-24B-Instruct-2506-bnb-4bit"},{"conf":"low","context":null,"model_id":"mistral-small-3-1-24b-base-2503","org":"unsloth","rank":122,"root":"mistralai/Mistral-Small-3.2-24B-Instruct-2506","score":64,"variant":"Mistral-Small-3.2-24B-Instruct-2506"},{"conf":"low","context":null,"model_id":"mistral-small-3-1-24b-instruct-2503","org":"unsloth","rank":123,"root":"mistralai/Mistral-Small-3.1-24B-Instruct-2503","score":64,"variant":"Mistral-Small-3.1-24B-Instruct-2503-GGUF"},{"conf":"low","context":null,"model_id":"mistral-small-3-1-24b-base-2503","org":"unsloth","rank":124,"root":"mistralai/Mistral-Small-3.2-24B-Instruct-2506","score":64,"variant":"Mistral-Small-3.2-24B-Instruct-2506-unsloth-bnb-4bit"},{"conf":"low","context":null,"model_id":"mistral-small-3-1-24b-instruct-2503","org":"unsloth","rank":125,"root":"mistralai/Mistral-Small-3.1-24B-Instruct-2503","score":64,"variant":"Mistral-Small-3.1-24B-Instruct-2503-bnb-4bit"},{"conf":"low","context":null,"model_id":"mistral-small-3-1-24b-instruct-2503","org":"unsloth","rank":126,"root":"mistralai/Mistral-Small-3.1-24B-Instruct-2503","score":64,"variant":"Mistral-Small-3.1-24B-Instruct-2503-unsloth-bnb-4bit"},{"conf":"low","context":null,"model_id":"mistral-small-3-1-24b-instruct-2503","org":"unsloth","rank":127,"root":"mistralai/Mistral-Small-3.1-24B-Instruct-2503","score":64,"variant":"Mistral-Small-3.1-24B-Instruct-2503"},{"conf":"low","context":null,"model_id":"mistral-small-3-1-24b-base-2503","org":"Mungert","rank":128,"root":"mistralai/Mistral-Small-3.1-24B-Base-2503","score":64,"variant":"Mistral-Small-3.1-24B-Instruct-2503-GGUF"},{"conf":"low","context":null,"model_id":"mistral-small-3-1-24b-base-2503","org":"Mungert","rank":129,"root":"mistralai/Mistral-Small-3.1-24B-Base-2503","score":64,"variant":"Mistral-Small-3.2-24B-Instruct-2506-GGUF"},{"conf":"low","context":null,"model_id":"mistral-small-3-1-24b-base-2503","org":"unsloth","rank":130,"root":"mistralai/Mistral-Small-3.2-24B-Instruct-2506","score":64,"variant":"Mistral-Small-3.2-24B-Instruct-2506-FP8"},{"conf":"low","context":"MathVista (CoT) | MMMU (CoT) | ChartQA (CoT)","model_id":null,"org":"mistralai","rank":131,"root":"mistralai/Pixtral-Large-Instruct-2411","score":64,"variant":"Pixtral-Large-Instruct-2411"},{"conf":"medium","context":"MMMU<sup>*</sup>","model_id":"minimax-vl-01","org":"MiniMaxAI","rank":132,"root":"MiniMaxAI/MiniMax-VL-01","score":63.5,"variant":"MiniMax-VL-01"},{"conf":"high","context":null,"model_id":"mistral-medium-3-5-128b","org":"nvidia","rank":133,"root":"mistralai/Mistral-Medium-3.5-128B","score":62.79,"variant":"Mistral-Medium-3.5-128B-NVFP4"},{"conf":"low","context":"MMLU Pro (5-shot CoT) | GPQA Main (5-shot CoT)","model_id":"mistral-small-3-1-24b-base-2503","org":"mistralai","rank":134,"root":"mistralai/Mistral-Small-3.1-24B-Base-2503","score":59.27,"variant":"Mistral-Small-3.1-24B-Instruct-2503"},{"conf":"low","context":"MMLU Pro (5-shot CoT) | GPQA Main (5-shot CoT)","model_id":"mistral-small-3-1-24b-instruct-2503","org":"unsloth","rank":135,"root":"mistralai/Mistral-Small-3.1-24B-Instruct-2503","score":59.27,"variant":"Mistral-Small-3.1-24B-Instruct-2503-GGUF"},{"conf":"low","context":"MMLU Pro (5-shot CoT) | GPQA Main (5-shot CoT)","model_id":"mistral-small-3-1-24b-base-2503","org":"mistralai","rank":136,"root":"mistralai/Mistral-Small-3.1-24B-Base-2503","score":59.27,"variant":"Mistral-Small-3.1-24B-Base-2503"},{"conf":"low","context":"MMLU Pro (5-shot CoT) | GPQA Main (5-shot CoT)","model_id":"mistral-small-3-1-24b-instruct-2503","org":"unsloth","rank":137,"root":"mistralai/Mistral-Small-3.1-24B-Instruct-2503","score":59.27,"variant":"Mistral-Small-3.1-24B-Instruct-2503-bnb-4bit"},{"conf":"low","context":"MMLU Pro (5-shot CoT) | GPQA Main (5-shot CoT)","model_id":"mistral-small-3-1-24b-instruct-2503","org":"unsloth","rank":138,"root":"mistralai/Mistral-Small-3.1-24B-Instruct-2503","score":59.27,"variant":"Mistral-Small-3.1-24B-Instruct-2503-unsloth-bnb-4bit"},{"conf":"low","context":"MMLU Pro (5-shot CoT) | GPQA Main (5-shot CoT)","model_id":"mistral-small-3-1-24b-instruct-2503","org":"unsloth","rank":139,"root":"mistralai/Mistral-Small-3.1-24B-Instruct-2503","score":59.27,"variant":"Mistral-Small-3.1-24B-Instruct-2503"},{"conf":"low","context":"MMLU Pro (5-shot CoT) | GPQA Main (5-shot CoT)","model_id":"mistral-small-3-1-24b-base-2503","org":"Mungert","rank":140,"root":"mistralai/Mistral-Small-3.1-24B-Base-2503","score":59.27,"variant":"Mistral-Small-3.1-24B-Instruct-2503-GGUF"},{"conf":"medium","context":"MMMU<sub>val</sub>","model_id":"qwen2-5-omni-3b","org":"Qwen","rank":141,"root":"Qwen/Qwen2.5-Omni-3B","score":59.2,"variant":"Qwen2.5-Omni-3B"},{"conf":"medium","context":"MMMU<sub>val</sub>","model_id":"qwen2-5-omni-7b","org":"Qwen","rank":142,"root":"Qwen/Qwen2.5-Omni-7B","score":59.2,"variant":"Qwen2.5-Omni-7B"},{"conf":"medium","context":"MMMU<sub>val</sub>","model_id":"qwen2-5-omni-7b","org":"unsloth","rank":143,"root":"Qwen/Qwen2.5-Omni-7B","score":59.2,"variant":"Qwen2.5-Omni-7B-GGUF"},{"conf":"medium","context":"MMMU<sub>val</sub>","model_id":"qwen2-5-omni-3b","org":"unsloth","rank":144,"root":"Qwen/Qwen2.5-Omni-3B","score":59.2,"variant":"Qwen2.5-Omni-3B-GGUF"},{"conf":"medium","context":"MMMU<sub>val</sub>","model_id":"qwen2-5-omni-3b","org":"Mungert","rank":145,"root":"Qwen/Qwen2.5-Omni-3B","score":59.2,"variant":"Qwen2.5-Omni-3B-GGUF"},{"conf":"medium","context":"MMMU<sub>val</sub>","model_id":"qwen2-5-omni-7b","org":"unsloth","rank":146,"root":"Qwen/Qwen2.5-Omni-7B","score":59.2,"variant":"Qwen2.5-Omni-7B"},{"conf":"medium","context":"MMMU<sub>val</sub>","model_id":"qwen2-5-omni-7b","org":"Mungert","rank":147,"root":"Qwen/Qwen2.5-Omni-7B","score":59.2,"variant":"Qwen2.5-Omni-7B-GGUF"},{"conf":"medium","context":"MMMU<sub>val</sub>","model_id":"qwen2-5-omni-3b","org":"unsloth","rank":148,"root":"Qwen/Qwen2.5-Omni-3B","score":59.2,"variant":"Qwen2.5-Omni-3B"},{"conf":"medium","context":"MMMU","model_id":"ling-lite","org":"inclusionAI","rank":149,"root":"inclusionAI/Ling-lite","score":56.3,"variant":"Ming-Lite-Omni"},{"conf":"medium","context":"MMMU<sub>val</sub>","model_id":"qwen2-5-vl-7b-instruct","org":"Qwen","rank":150,"root":"Qwen/Qwen2.5-VL-7B-Instruct","score":56,"variant":"Qwen2.5-VL-7B-Instruct"},{"conf":"medium","context":"MMMU<sub>val</sub>","model_id":"qwen2-5-vl-7b-instruct","org":"unsloth","rank":151,"root":"Qwen/Qwen2.5-VL-7B-Instruct","score":56,"variant":"Qwen2.5-VL-7B-Instruct-GGUF"},{"conf":"medium","context":"MMMU<sub>val</sub>","model_id":"qwen2-5-vl-7b-instruct","org":"unsloth","rank":152,"root":"Qwen/Qwen2.5-VL-7B-Instruct","score":56,"variant":"Qwen2.5-VL-7B-Instruct"},{"conf":"medium","context":"MMMU<sub>val</sub>","model_id":"qwen2-5-vl-7b-instruct","org":"unsloth","rank":153,"root":"Qwen/Qwen2.5-VL-7B-Instruct","score":56,"variant":"Qwen2.5-VL-7B-Instruct-unsloth-bnb-4bit"},{"conf":"medium","context":"MMMU<sub>val</sub>","model_id":"qwen2-5-vl-7b-instruct","org":"unsloth","rank":154,"root":"Qwen/Qwen2.5-VL-7B-Instruct","score":56,"variant":"Qwen2.5-VL-7B-Instruct-bnb-4bit"},{"conf":"medium","context":"MMMU<sub>val</sub>","model_id":"qwen2-5-vl-7b-instruct","org":"Mungert","rank":155,"root":"Qwen/Qwen2.5-VL-7B-Instruct","score":56,"variant":"Qwen2.5-VL-7B-Instruct-GGUF"},{"conf":"medium","context":"MMMU","model_id":"phi-4-multimodal-instruct","org":"microsoft","rank":156,"root":"microsoft/Phi-4-multimodal-instruct","score":55.1,"variant":"Phi-4-multimodal-instruct"},{"conf":"medium","context":"MMMU","model_id":"phi-4-multimodal-instruct","org":"Lexius","rank":157,"root":"microsoft/Phi-4-multimodal-instruct","score":55.1,"variant":"Phi-4-multimodal-instruct"},{"conf":"medium","context":"MMMU-Pro<sup>*</sup>","model_id":"minimax-vl-01","org":"MiniMaxAI","rank":158,"root":"MiniMaxAI/MiniMax-VL-01","score":54.5,"variant":"MiniMax-VL-01"},{"conf":"medium","context":"MMMU Pro","model_id":"diffusiongemma-26b-a4b-it","org":"google","rank":159,"root":"google/diffusiongemma-26B-A4B-it","score":54.3,"variant":"diffusiongemma-26B-A4B-it"},{"conf":"medium","context":"MMMU Pro","model_id":"diffusiongemma-26b-a4b-it","org":"unsloth","rank":160,"root":"google/diffusiongemma-26B-A4B-it","score":54.3,"variant":"diffusiongemma-26B-A4B-it-GGUF"},{"conf":"medium","context":"MMMU Pro","model_id":"diffusiongemma-26b-a4b-it","org":"unsloth","rank":161,"root":"google/diffusiongemma-26B-A4B-it","score":54.3,"variant":"diffusiongemma-26B-A4B-it"},{"conf":"high","context":null,"model_id":"qwen2-vl-7b-instruct","org":"Qwen","rank":162,"root":"Qwen/Qwen2-VL-7B-Instruct","score":53.77,"variant":"Qwen2-VL-7B-Instruct-AWQ"},{"conf":"high","context":null,"model_id":"qwen2-vl-7b-instruct","org":"Qwen","rank":163,"root":"Qwen/Qwen2-VL-7B-Instruct","score":53.77,"variant":"Qwen2-VL-7B-Instruct-GPTQ-Int8"},{"conf":"high","context":null,"model_id":"qwen2-vl-7b-instruct","org":"Qwen","rank":164,"root":"Qwen/Qwen2-VL-7B-Instruct","score":53.77,"variant":"Qwen2-VL-7B-Instruct-GPTQ-Int4"},{"conf":"medium","context":"MMMU<sub>val</sub>","model_id":"qwen2-5-vl-3b-instruct","org":"Qwen","rank":165,"root":"Qwen/Qwen2.5-VL-3B-Instruct","score":52.3,"variant":"Qwen2.5-VL-3B-Instruct"},{"conf":"medium","context":"MMMU<sub>val</sub>","model_id":"qwen2-5-vl-3b-instruct","org":"unsloth","rank":166,"root":"Qwen/Qwen2.5-VL-3B-Instruct","score":52.3,"variant":"Qwen2.5-VL-3B-Instruct-unsloth-bnb-4bit"},{"conf":"medium","context":"MMMU<sub>val</sub>","model_id":"qwen2-5-vl-3b-instruct","org":"unsloth","rank":167,"root":"Qwen/Qwen2.5-VL-3B-Instruct","score":52.3,"variant":"Qwen2.5-VL-3B-Instruct-GGUF"},{"conf":"medium","context":"MMMU<sub>val</sub>","model_id":"qwen2-5-vl-3b-instruct","org":"Mungert","rank":168,"root":"Qwen/Qwen2.5-VL-3B-Instruct","score":52.3,"variant":"Qwen2.5-VL-3B-Instruct-GGUF"},{"conf":"medium","context":"MMMU<sub>val</sub>","model_id":"qwen2-5-vl-3b-instruct","org":"unsloth","rank":169,"root":"Qwen/Qwen2.5-VL-3B-Instruct","score":52.3,"variant":"Qwen2.5-VL-3B-Instruct-bnb-4bit"},{"conf":"medium","context":"MMMU<sub>val</sub>","model_id":"qwen2-5-vl-3b-instruct","org":"unsloth","rank":170,"root":"Qwen/Qwen2.5-VL-3B-Instruct","score":52.3,"variant":"Qwen2.5-VL-3B-Instruct"},{"conf":"medium","context":"MMMU<sub>val</sub>","model_id":"qwen2-vl-7b","org":"Qwen","rank":171,"root":"Qwen/Qwen2-VL-7B","score":51.8,"variant":"Qwen2-VL-7B-Instruct"},{"conf":"medium","context":"MMMU<sub>val</sub>","model_id":"qwen2-vl-7b-instruct","org":"unsloth","rank":172,"root":"Qwen/Qwen2-VL-7B-Instruct","score":51.8,"variant":"Qwen2-VL-7B-Instruct-unsloth-bnb-4bit"},{"conf":"medium","context":"MMMU<sub>val</sub>","model_id":"qwen2-vl-7b-instruct","org":"unsloth","rank":173,"root":"Qwen/Qwen2-VL-7B-Instruct","score":51.8,"variant":"Qwen2-VL-7B-Instruct-bnb-4bit"},{"conf":"medium","context":"MMMU<sub>val</sub>","model_id":"qwen2-vl-7b-instruct","org":"unsloth","rank":174,"root":"Qwen/Qwen2-VL-7B-Instruct","score":51.8,"variant":"Qwen2-VL-7B-Instruct"},{"conf":"medium","context":"MMMU-Pro (Pass@1)","model_id":"kimi-vl-a3b-instruct","org":"moonshotai","rank":175,"root":"moonshotai/Kimi-VL-A3B-Instruct","score":51.7,"variant":"Kimi-VL-A3B-Thinking-2506"},{"conf":"medium","context":"MMMU Pro","model_id":"qwen2-5-vl-32b-instruct","org":"Qwen","rank":176,"root":"Qwen/Qwen2.5-VL-32B-Instruct","score":51.1,"variant":"Qwen2.5-VL-32B-Instruct"},{"conf":"medium","context":"MMMU Pro","model_id":"qwen2-5-vl-32b-instruct","org":"unsloth","rank":177,"root":"Qwen/Qwen2.5-VL-32B-Instruct","score":51.1,"variant":"Qwen2.5-VL-32B-Instruct-GGUF"},{"conf":"medium","context":"MMMU Pro","model_id":"qwen2-5-vl-32b-instruct","org":"unsloth","rank":178,"root":"Qwen/Qwen2.5-VL-32B-Instruct","score":51.1,"variant":"Qwen2.5-VL-32B-Instruct-bnb-4bit"},{"conf":"medium","context":"MMMU Pro","model_id":"qwen2-5-vl-32b-instruct","org":"Mungert","rank":179,"root":"Qwen/Qwen2.5-VL-32B-Instruct","score":51.1,"variant":"Qwen2.5-VL-32B-Instruct-GGUF"},{"conf":"medium","context":"MMMU Pro","model_id":"qwen2-5-vl-32b-instruct","org":"unsloth","rank":180,"root":"Qwen/Qwen2.5-VL-32B-Instruct","score":51.1,"variant":"Qwen2.5-VL-32B-Instruct-unsloth-bnb-4bit"},{"conf":"medium","context":"[MMMU][] (val)","model_id":"gemma-3-270m","org":"google","rank":181,"root":"google/gemma-3-270m","score":48.8,"variant":"gemma-3-270m"},{"conf":"medium","context":"[MMMU][] (val)","model_id":"gemma-3-270m","org":"google","rank":182,"root":"google/gemma-3-270m","score":48.8,"variant":"gemma-3-270m-it"},{"conf":"medium","context":"[MMMU][] (val)","model_id":"gemma-3-270m-it","org":"unsloth","rank":183,"root":"google/gemma-3-270m-it","score":48.8,"variant":"gemma-3-270m-it-GGUF"},{"conf":"medium","context":"[MMMU][] (val)","model_id":"gemma-3-270m-it","org":"unsloth","rank":184,"root":"google/gemma-3-270m-it","score":48.8,"variant":"gemma-3-270m-it"},{"conf":"medium","context":"[MMMU][] (val)","model_id":"gemma-3-270m-it","org":"unsloth","rank":185,"root":"google/gemma-3-270m-it","score":48.8,"variant":"gemma-3-270m-it-unsloth-bnb-4bit"},{"conf":"medium","context":"[MMMU][] (val)","model_id":"gemma-3-270m-it-qat-q4-0-unquantized","org":"unsloth","rank":186,"root":"google/gemma-3-270m-it-qat-q4_0-unquantized","score":48.8,"variant":"gemma-3-270m-it-qat-GGUF"},{"conf":"medium","context":"[MMMU][] (val)","model_id":"gemma-3-270m","org":"unsloth","rank":187,"root":"google/gemma-3-270m","score":48.8,"variant":"gemma-3-270m"},{"conf":"medium","context":"[MMMU][] (val)","model_id":"gemma-3-270m-it","org":"unsloth","rank":188,"root":"google/gemma-3-270m-it","score":48.8,"variant":"gemma-3-270m-unsloth-bnb-4bit"},{"conf":"medium","context":"[MMMU][] (val)","model_id":"gemma-3-270m-it","org":"google","rank":189,"root":"google/gemma-3-270m-it","score":48.8,"variant":"gemma-3-270m-it-qat-q4_0-unquantized"},{"conf":"medium","context":"[MMMU][] (val)","model_id":"gemma-3-270m-it","org":"unsloth","rank":190,"root":"google/gemma-3-270m-it","score":48.8,"variant":"gemma-3-270m-it-bnb-4bit"},{"conf":"medium","context":"[MMMU][] (val)","model_id":"gemma-3-270m-it-qat-q4-0-unquantized","org":"unsloth","rank":191,"root":"google/gemma-3-270m-it-qat-q4_0-unquantized","score":48.8,"variant":"gemma-3-270m-it-qat"},{"conf":"medium","context":"[MMMU][] (val)","model_id":"gemma-3-270m","org":"unsloth","rank":192,"root":"google/gemma-3-270m","score":48.8,"variant":"gemma-3-270m-bnb-4bit"},{"conf":"medium","context":"[MMMU][] (val)","model_id":"gemma-3-270m","org":"google","rank":193,"root":"google/gemma-3-270m","score":48.8,"variant":"gemma-3-270m-qat-q4_0-unquantized"},{"conf":"medium","context":"[MMMU][] (val)","model_id":"gemma-3-270m","org":"Mungert","rank":194,"root":"google/gemma-3-270m","score":48.8,"variant":"gemma-3-270m-it-GGUF"},{"conf":"medium","context":"[MMMU][] (val)","model_id":"gemma-3-270m-it","org":"unsloth","rank":195,"root":"google/gemma-3-270m-it","score":48.8,"variant":"gemma-3-270m-it-torchao-FP8"},{"conf":"medium","context":"MMMU Val with chatGPT as a judge","model_id":null,"org":"nvidia","rank":196,"root":"nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1","score":48.2,"variant":"Llama-3.1-Nemotron-Nano-VL-8B-V1"},{"conf":"medium","context":"**MMMU** *(CoT)*","model_id":null,"org":"mistralai","rank":197,"root":"mistralai/Pixtral-12B-Base-2409","score":47.6,"variant":"Pixtral-12B-2409"},{"conf":"medium","context":"Video-MMMU","model_id":"ling-lite","org":"inclusionAI","rank":198,"root":"inclusionAI/Ling-lite","score":46.3,"variant":"Ming-Lite-Omni"},{"conf":"low","context":null,"model_id":null,"org":"unsloth","rank":199,"root":"LiquidAI/LFM2.5-VL-1.6B","score":40.56,"variant":"LFM2.5-VL-1.6B-GGUF"},{"conf":"medium","context":"MMMU","model_id":null,"org":"microsoft","rank":200,"root":"microsoft/Phi-3-vision-128k-instruct","score":40.4,"variant":"Phi-3-vision-128k-instruct"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-1b-pt","org":"google","rank":201,"root":"google/gemma-3-1b-pt","score":39.2,"variant":"gemma-3-1b-it"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-4b-pt","org":"google","rank":202,"root":"google/gemma-3-4b-pt","score":39.2,"variant":"gemma-3-4b-it"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-270m","org":"google","rank":203,"root":"google/gemma-3-270m","score":39.2,"variant":"gemma-3-270m"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-12b-pt","org":"google","rank":204,"root":"google/gemma-3-12b-pt","score":39.2,"variant":"gemma-3-12b-it"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-27b-pt","org":"google","rank":205,"root":"google/gemma-3-27b-pt","score":39.2,"variant":"gemma-3-27b-it"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-270m","org":"google","rank":206,"root":"google/gemma-3-270m","score":39.2,"variant":"gemma-3-270m-it"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-4b-pt","org":"unsloth","rank":207,"root":"google/gemma-3-4b-it","score":39.2,"variant":"gemma-3-4b-it"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-1b-pt","org":"ggml-org","rank":208,"root":"google/gemma-3-1b-it","score":39.2,"variant":"gemma-3-1b-it-GGUF"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-12b-pt","org":"google","rank":209,"root":"google/gemma-3-12b-it","score":39.2,"variant":"gemma-3-12b-it-qat-q4_0-unquantized"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-1b-pt","org":"unsloth","rank":210,"root":"google/gemma-3-1b-it","score":39.2,"variant":"gemma-3-1b-it"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-270m-it","org":"unsloth","rank":211,"root":"google/gemma-3-270m-it","score":39.2,"variant":"gemma-3-270m-it-GGUF"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-12b-pt","org":"unsloth","rank":212,"root":"google/gemma-3-12b-it","score":39.2,"variant":"gemma-3-12b-it-GGUF"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-4b-pt","org":"google","rank":213,"root":"google/gemma-3-4b-pt","score":39.2,"variant":"gemma-3-4b-pt"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-12b-it-qat-q4-0-unquantized","org":"unsloth","rank":214,"root":"google/gemma-3-12b-it-qat-q4_0-unquantized","score":39.2,"variant":"gemma-3-12b-it-qat-GGUF"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-27b-it","org":"unsloth","rank":215,"root":"google/gemma-3-27b-it","score":39.2,"variant":"gemma-3-27b-it"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-1b-pt","org":"google","rank":216,"root":"google/gemma-3-1b-pt","score":39.2,"variant":"gemma-3-1b-pt"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-4b-pt","org":"ggml-org","rank":217,"root":"google/gemma-3-4b-it","score":39.2,"variant":"gemma-3-4b-it-GGUF"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-4b-pt","org":"unsloth","rank":218,"root":"google/gemma-3-4b-it","score":39.2,"variant":"gemma-3-4b-it-GGUF"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-1b-pt","org":"unsloth","rank":219,"root":"google/gemma-3-1b-it","score":39.2,"variant":"gemma-3-1b-it-GGUF"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":null,"org":"google","rank":220,"root":"google/gemma-3-12b","score":39.2,"variant":"gemma-3-12b-pt"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-12b-pt","org":"unsloth","rank":221,"root":"google/gemma-3-12b-it","score":39.2,"variant":"gemma-3-12b-it-unsloth-bnb-4bit"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-4b-pt","org":"unsloth","rank":222,"root":"google/gemma-3-4b-it","score":39.2,"variant":"gemma-3-4b-it-unsloth-bnb-4bit"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-12b-pt","org":"unsloth","rank":223,"root":"google/gemma-3-12b-pt","score":39.2,"variant":"gemma-3-12b-it"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-270m-it","org":"unsloth","rank":224,"root":"google/gemma-3-270m-it","score":39.2,"variant":"gemma-3-270m-it"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-27b-it","org":"unsloth","rank":225,"root":"google/gemma-3-27b-it","score":39.2,"variant":"gemma-3-27b-it-GGUF"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-1b-pt","org":"unsloth","rank":226,"root":"google/gemma-3-1b-it","score":39.2,"variant":"gemma-3-1b-it-unsloth-bnb-4bit"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-4b-pt","org":"unsloth","rank":227,"root":"google/gemma-3-4b-it","score":39.2,"variant":"gemma-3-4b-it-bnb-4bit"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-4b-pt","org":"unsloth","rank":228,"root":"google/gemma-3-4b-pt","score":39.2,"variant":"gemma-3-4b-pt"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-27b-pt","org":"google","rank":229,"root":"google/gemma-3-27b-pt","score":39.2,"variant":"gemma-3-27b-pt"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-12b-pt","org":"unsloth","rank":230,"root":"google/gemma-3-12b-it","score":39.2,"variant":"gemma-3-12b-it-bnb-4bit"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-270m-it","org":"unsloth","rank":231,"root":"google/gemma-3-270m-it","score":39.2,"variant":"gemma-3-270m-it-unsloth-bnb-4bit"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-27b-it","org":"ggml-org","rank":232,"root":"google/gemma-3-27b-it","score":39.2,"variant":"gemma-3-27b-it-GGUF"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-12b-pt","org":"unsloth","rank":233,"root":"google/gemma-3-12b-pt","score":39.2,"variant":"gemma-3-12b-pt"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-27b-it","org":"unsloth","rank":234,"root":"google/gemma-3-27b-it","score":39.2,"variant":"gemma-3-27b-it-bnb-4bit"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-27b-it","org":"unsloth","rank":235,"root":"google/gemma-3-27b-it","score":39.2,"variant":"gemma-3-27b-it-unsloth-bnb-4bit"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-1b-pt","org":"unsloth","rank":236,"root":"google/gemma-3-1b-pt","score":39.2,"variant":"gemma-3-1b-pt"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-12b-pt","org":"ggml-org","rank":237,"root":"google/gemma-3-12b-it","score":39.2,"variant":"gemma-3-12b-it-GGUF"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-4b-it-qat-q4-0-unquantized","org":"unsloth","rank":238,"root":"google/gemma-3-4b-it-qat-q4_0-unquantized","score":39.2,"variant":"gemma-3-4b-it-qat-GGUF"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-270m-it-qat-q4-0-unquantized","org":"unsloth","rank":239,"root":"google/gemma-3-270m-it-qat-q4_0-unquantized","score":39.2,"variant":"gemma-3-270m-it-qat-GGUF"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-12b-it-qat-q4-0-unquantized","org":"unsloth","rank":240,"root":"google/gemma-3-12b-it-qat-q4_0-unquantized","score":39.2,"variant":"gemma-3-12b-it-qat-bnb-4bit"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-270m","org":"unsloth","rank":241,"root":"google/gemma-3-270m","score":39.2,"variant":"gemma-3-270m"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-12b-it-qat-int4-unquantized","org":"unsloth","rank":242,"root":"google/gemma-3-12b-it-qat-int4-unquantized","score":39.2,"variant":"gemma-3-12b-it-qat-int4-GGUF"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-4b-pt","org":"google","rank":243,"root":"google/gemma-3-4b-it","score":39.2,"variant":"gemma-3-4b-it-qat-q4_0-gguf"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-4b-pt","org":"google","rank":244,"root":"google/gemma-3-4b-it","score":39.2,"variant":"gemma-3-4b-it-qat-q4_0-unquantized"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-4b-pt","org":"unsloth","rank":245,"root":"google/gemma-3-4b-pt","score":39.2,"variant":"gemma-3-4b-pt-bnb-4bit"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-27b-it-qat-q4-0-unquantized","org":"unsloth","rank":246,"root":"google/gemma-3-27b-it-qat-q4_0-unquantized","score":39.2,"variant":"gemma-3-27b-it-qat-GGUF"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-12b-pt","org":"google","rank":247,"root":"google/gemma-3-12b-it","score":39.2,"variant":"gemma-3-12b-it-qat-q4_0-gguf"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-1b-pt","org":"unsloth","rank":248,"root":"google/gemma-3-1b-it","score":39.2,"variant":"gemma-3-1b-it-bnb-4bit"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-1b-pt","org":"google","rank":249,"root":"google/gemma-3-1b-it","score":39.2,"variant":"gemma-3-1b-it-qat-q4_0-gguf"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-4b-pt","org":"unsloth","rank":250,"root":"google/gemma-3-4b-pt","score":39.2,"variant":"gemma-3-4b-pt-unsloth-bnb-4bit"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-270m-it","org":"unsloth","rank":251,"root":"google/gemma-3-270m-it","score":39.2,"variant":"gemma-3-270m-unsloth-bnb-4bit"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-12b-it-qat-q4-0-unquantized","org":"unsloth","rank":252,"root":"google/gemma-3-12b-it-qat-q4_0-unquantized","score":39.2,"variant":"gemma-3-12b-it-qat"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-1b-pt","org":"unsloth","rank":253,"root":"google/gemma-3-1b-pt","score":39.2,"variant":"gemma-3-1b-pt-unsloth-bnb-4bit"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-4b-pt","org":"unsloth","rank":254,"root":"google/gemma-3-4b-it","score":39.2,"variant":"gemma-3-4b-it-FP8-Dynamic"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-270m-it","org":"google","rank":255,"root":"google/gemma-3-270m-it","score":39.2,"variant":"gemma-3-270m-it-qat-q4_0-unquantized"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-12b-pt","org":"unsloth","rank":256,"root":"google/gemma-3-12b-pt","score":39.2,"variant":"gemma-3-12b-pt-bnb-4bit"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-12b-pt","org":"google","rank":257,"root":"google/gemma-3-12b-it","score":39.2,"variant":"gemma-3-12b-it-qat-int4-unquantized"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-27b-pt","org":"Mungert","rank":258,"root":"google/gemma-3-27b-pt","score":39.2,"variant":"gemma-3-27b-it-GGUF"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-27b-it","org":"google","rank":259,"root":"google/gemma-3-27b-it","score":39.2,"variant":"gemma-3-27b-it-qat-q4_0-unquantized"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-12b-pt","org":"unsloth","rank":260,"root":"google/gemma-3-12b-pt","score":39.2,"variant":"gemma-3-12b-pt-unsloth-bnb-4bit"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-270m-it","org":"unsloth","rank":261,"root":"google/gemma-3-270m-it","score":39.2,"variant":"gemma-3-270m-it-bnb-4bit"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-27b-it","org":"google","rank":262,"root":"google/gemma-3-27b-it","score":39.2,"variant":"gemma-3-27b-it-qat-q4_0-gguf"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-27b-it","org":"Mungert","rank":263,"root":"google/gemma-3-27b-it","score":39.2,"variant":"gemma-3-27b-it-qat-q4_0-GGUF"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-270m-it-qat-q4-0-unquantized","org":"unsloth","rank":264,"root":"google/gemma-3-270m-it-qat-q4_0-unquantized","score":39.2,"variant":"gemma-3-270m-it-qat"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-4b-it-qat-q4-0-unquantized","org":"unsloth","rank":265,"root":"google/gemma-3-4b-it-qat-q4_0-unquantized","score":39.2,"variant":"gemma-3-4b-it-qat-bnb-4bit"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-1b-pt","org":"unsloth","rank":266,"root":"google/gemma-3-1b-pt","score":39.2,"variant":"gemma-3-1b-pt-bnb-4bit"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-27b-pt","org":"unsloth","rank":267,"root":"google/gemma-3-27b-pt","score":39.2,"variant":"gemma-3-27b-pt"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-12b-pt","org":"unsloth","rank":268,"root":"google/gemma-3-12b-pt","score":39.2,"variant":"gemma-3-12b-it-FP8-Dynamic"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-270m","org":"unsloth","rank":269,"root":"google/gemma-3-270m","score":39.2,"variant":"gemma-3-270m-bnb-4bit"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-1b-pt","org":"google","rank":270,"root":"google/gemma-3-1b-it","score":39.2,"variant":"gemma-3-1b-it-qat-int4-unquantized"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-27b-pt","org":"unsloth","rank":271,"root":"google/gemma-3-27b-pt","score":39.2,"variant":"gemma-3-27b-pt-bnb-4bit"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-12b-it-qat-q4-0-unquantized","org":"unsloth","rank":272,"root":"google/gemma-3-12b-it-qat-q4_0-unquantized","score":39.2,"variant":"gemma-3-12b-it-qat-unsloth-bnb-4bit"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-270m","org":"google","rank":273,"root":"google/gemma-3-270m","score":39.2,"variant":"gemma-3-270m-qat-q4_0-unquantized"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-12b-pt","org":"google","rank":274,"root":"google/gemma-3-12b-pt","score":39.2,"variant":"gemma-3-12b-pt-qat-q4_0-gguf"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-270m","org":"Mungert","rank":275,"root":"google/gemma-3-270m","score":39.2,"variant":"gemma-3-270m-it-GGUF"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-4b-it-qat-q4-0-unquantized","org":"unsloth","rank":276,"root":"google/gemma-3-4b-it-qat-q4_0-unquantized","score":39.2,"variant":"gemma-3-4b-it-qat"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-270m-it","org":"unsloth","rank":277,"root":"google/gemma-3-270m-it","score":39.2,"variant":"gemma-3-270m-it-torchao-FP8"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-1b-pt","org":"google","rank":278,"root":"google/gemma-3-1b-it","score":39.2,"variant":"gemma-3-1b-it-qat-q4_0-unquantized"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-1b-it-qat-q4-0-unquantized","org":"unsloth","rank":279,"root":"google/gemma-3-1b-it-qat-q4_0-unquantized","score":39.2,"variant":"gemma-3-1b-it-qat"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-27b-it-qat-q4-0-unquantized","org":"unsloth","rank":280,"root":"google/gemma-3-27b-it-qat-q4_0-unquantized","score":39.2,"variant":"gemma-3-27b-it-qat"},{"conf":"medium","context":"[MMMU][] (pt)","model_id":"gemma-3-4b-pt","org":"google","rank":281,"root":"google/gemma-3-4b-pt","score":39.2,"variant":"gemma-3-4b-pt-qat-q4_0-gguf"},{"conf":"high","context":null,"model_id":null,"org":"Mungert","rank":282,"root":"HuggingFaceTB/SmolLM2-1.7B-Instruct","score":38.8,"variant":"SmolVLM-Instruct-GGUF"},{"conf":"medium","context":"MMMU-Pro (std/vision)","model_id":"phi-4-multimodal-instruct","org":"microsoft","rank":283,"root":"microsoft/Phi-4-multimodal-instruct","score":38.5,"variant":"Phi-4-multimodal-instruct"},{"conf":"medium","context":"MMMU-Pro (std/vision)","model_id":"phi-4-multimodal-instruct","org":"Lexius","rank":284,"root":"microsoft/Phi-4-multimodal-instruct","score":38.5,"variant":"Phi-4-multimodal-instruct"},{"conf":"low","context":"Open Source","model_id":"cogvlm2-llama3-chat-19b","org":"zai-org","rank":285,"root":"zai-org/cogvlm2-llama3-chat-19B","score":37.3,"variant":"cogvlm2-llama3-chat-19B"},{"conf":"low","context":"Open Source","model_id":null,"org":"zai-org","rank":286,"root":"zai-org/cogvlm2-llama3-chinese-chat-19B","score":37.3,"variant":"cogvlm2-llama3-chinese-chat-19B"},{"conf":"low","context":"Open Source","model_id":"cogvlm2-llama3-chat-19b","org":"zai-org","rank":287,"root":"zai-org/cogvlm2-llama3-chat-19B","score":37.3,"variant":"cogvlm2-llama3-chat-19B-int4"},{"conf":"medium","context":"MMMU-Pro<sub>overall</sub>","model_id":"qwen2-5-omni-3b","org":"Qwen","rank":288,"root":"Qwen/Qwen2.5-Omni-3B","score":36.6,"variant":"Qwen2.5-Omni-3B"},{"conf":"medium","context":"MMMU-Pro<sub>overall</sub>","model_id":"qwen2-5-omni-7b","org":"Qwen","rank":289,"root":"Qwen/Qwen2.5-Omni-7B","score":36.6,"variant":"Qwen2.5-Omni-7B"},{"conf":"medium","context":"MMMU-Pro<sub>overall</sub>","model_id":"qwen2-5-omni-7b","org":"unsloth","rank":290,"root":"Qwen/Qwen2.5-Omni-7B","score":36.6,"variant":"Qwen2.5-Omni-7B-GGUF"},{"conf":"medium","context":"MMMU-Pro<sub>overall</sub>","model_id":"qwen2-5-omni-3b","org":"unsloth","rank":291,"root":"Qwen/Qwen2.5-Omni-3B","score":36.6,"variant":"Qwen2.5-Omni-3B-GGUF"},{"conf":"medium","context":"MMMU-Pro<sub>overall</sub>","model_id":"qwen2-5-omni-3b","org":"Mungert","rank":292,"root":"Qwen/Qwen2.5-Omni-3B","score":36.6,"variant":"Qwen2.5-Omni-3B-GGUF"},{"conf":"medium","context":"MMMU-Pro<sub>overall</sub>","model_id":"qwen2-5-omni-7b","org":"unsloth","rank":293,"root":"Qwen/Qwen2.5-Omni-7B","score":36.6,"variant":"Qwen2.5-Omni-7B"},{"conf":"medium","context":"MMMU-Pro<sub>overall</sub>","model_id":"qwen2-5-omni-7b","org":"Mungert","rank":294,"root":"Qwen/Qwen2.5-Omni-7B","score":36.6,"variant":"Qwen2.5-Omni-7B-GGUF"},{"conf":"medium","context":"MMMU-Pro<sub>overall</sub>","model_id":"qwen2-5-omni-3b","org":"unsloth","rank":295,"root":"Qwen/Qwen2.5-Omni-3B","score":36.6,"variant":"Qwen2.5-Omni-3B"},{"conf":"medium","context":"MMMU<sub>val</sub>","model_id":null,"org":"Qwen","rank":296,"root":"Qwen/Qwen2-VL-2B","score":36.3,"variant":"Qwen2-VL-2B-Instruct"},{"conf":"medium","context":"MMMU<sub>val</sub>","model_id":"qwen2-vl-2b-instruct","org":"unsloth","rank":297,"root":"Qwen/Qwen2-VL-2B-Instruct","score":36.3,"variant":"Qwen2-VL-2B-Instruct-unsloth-bnb-4bit"},{"conf":"medium","context":"MMMU<sub>val</sub>","model_id":"qwen2-vl-2b-instruct","org":"unsloth","rank":298,"root":"Qwen/Qwen2-VL-2B-Instruct","score":36.3,"variant":"Qwen2-VL-2B-Instruct-bnb-4bit"},{"conf":"medium","context":"MMMU<sub>val</sub>","model_id":"qwen2-vl-2b-instruct","org":"unsloth","rank":299,"root":"Qwen/Qwen2-VL-2B-Instruct","score":36.3,"variant":"Qwen2-VL-2B-Instruct"},{"conf":"medium","context":"MMMU<sub>val</sub>","model_id":"qwen2-vl-72b-instruct","org":"unsloth","rank":300,"root":"Qwen/Qwen2-VL-72B-Instruct","score":36.3,"variant":"Qwen2-VL-72B-Instruct-bnb-4bit"},{"conf":"medium","context":"MMMU<sub>val</sub>","model_id":"qwen2-vl-72b-instruct","org":"unsloth","rank":301,"root":"Qwen/Qwen2-VL-72B-Instruct","score":36.3,"variant":"Qwen2-VL-72B-Instruct"},{"conf":"low","context":null,"model_id":null,"org":"facebook","rank":302,"root":"facebook/Perception-LM-1B","score":34.8,"variant":"Perception-LM-1B"},{"conf":"low","context":null,"model_id":null,"org":"facebook","rank":303,"root":"facebook/Perception-LM-8B","score":34.8,"variant":"Perception-LM-8B"},{"conf":"medium","context":"MMMU-Pro<sub>val</sub>","model_id":"qwen2-5-vl-7b-instruct","org":"Qwen","rank":304,"root":"Qwen/Qwen2.5-VL-7B-Instruct","score":34.3,"variant":"Qwen2.5-VL-7B-Instruct"},{"conf":"medium","context":"MMMU-Pro<sub>val</sub>","model_id":"qwen2-5-vl-7b-instruct","org":"unsloth","rank":305,"root":"Qwen/Qwen2.5-VL-7B-Instruct","score":34.3,"variant":"Qwen2.5-VL-7B-Instruct-GGUF"},{"conf":"medium","context":"MMMU-Pro<sub>val</sub>","model_id":"qwen2-5-vl-7b-instruct","org":"unsloth","rank":306,"root":"Qwen/Qwen2.5-VL-7B-Instruct","score":34.3,"variant":"Qwen2.5-VL-7B-Instruct"},{"conf":"medium","context":"MMMU-Pro<sub>val</sub>","model_id":"qwen2-5-vl-7b-instruct","org":"unsloth","rank":307,"root":"Qwen/Qwen2.5-VL-7B-Instruct","score":34.3,"variant":"Qwen2.5-VL-7B-Instruct-unsloth-bnb-4bit"},{"conf":"medium","context":"MMMU-Pro<sub>val</sub>","model_id":"qwen2-5-vl-7b-instruct","org":"unsloth","rank":308,"root":"Qwen/Qwen2.5-VL-7B-Instruct","score":34.3,"variant":"Qwen2.5-VL-7B-Instruct-bnb-4bit"},{"conf":"medium","context":"MMMU-Pro<sub>val</sub>","model_id":"qwen2-5-vl-7b-instruct","org":"Mungert","rank":309,"root":"Qwen/Qwen2.5-VL-7B-Instruct","score":34.3,"variant":"Qwen2.5-VL-7B-Instruct-GGUF"},{"conf":"medium","context":"MMMU-Pro<sub>val</sub>","model_id":"qwen2-5-vl-3b-instruct","org":"Qwen","rank":310,"root":"Qwen/Qwen2.5-VL-3B-Instruct","score":32.7,"variant":"Qwen2.5-VL-3B-Instruct"},{"conf":"medium","context":"MMMU-Pro<sub>val</sub>","model_id":"qwen2-5-vl-3b-instruct","org":"unsloth","rank":311,"root":"Qwen/Qwen2.5-VL-3B-Instruct","score":32.7,"variant":"Qwen2.5-VL-3B-Instruct-unsloth-bnb-4bit"},{"conf":"medium","context":"MMMU-Pro<sub>val</sub>","model_id":"qwen2-5-vl-3b-instruct","org":"unsloth","rank":312,"root":"Qwen/Qwen2.5-VL-3B-Instruct","score":32.7,"variant":"Qwen2.5-VL-3B-Instruct-GGUF"},{"conf":"medium","context":"MMMU-Pro<sub>val</sub>","model_id":"qwen2-5-vl-3b-instruct","org":"Mungert","rank":313,"root":"Qwen/Qwen2.5-VL-3B-Instruct","score":32.7,"variant":"Qwen2.5-VL-3B-Instruct-GGUF"},{"conf":"medium","context":"MMMU-Pro<sub>val</sub>","model_id":"qwen2-5-vl-3b-instruct","org":"unsloth","rank":314,"root":"Qwen/Qwen2.5-VL-3B-Instruct","score":32.7,"variant":"Qwen2.5-VL-3B-Instruct-bnb-4bit"},{"conf":"medium","context":"MMMU-Pro<sub>val</sub>","model_id":"qwen2-5-vl-3b-instruct","org":"unsloth","rank":315,"root":"Qwen/Qwen2.5-VL-3B-Instruct","score":32.7,"variant":"Qwen2.5-VL-3B-Instruct"},{"conf":"medium","context":"MMMU","model_id":null,"org":"unsloth","rank":316,"root":"ibm-granite/granite-vision-3.2-2b","score":32,"variant":"granite-vision-3.2-2b"},{"conf":"medium","context":"MMMU<sub>val</sub>","model_id":"paligemma-3b-mix-448","org":"nvidia","rank":317,"root":"google/paligemma-3b-mix-448","score":31.4,"variant":"Eagle2-1B"},{"conf":"medium","context":"MMMU<sub>val</sub>","model_id":"paligemma-3b-mix-448","org":"Mungert","rank":318,"root":"google/paligemma-3b-mix-448","score":31.4,"variant":"Eagle2-1B-GGUF"},{"conf":"high","context":null,"model_id":null,"org":"Mungert","rank":319,"root":"HuggingFaceTB/SmolLM2-135M-Instruct","score":28.3,"variant":"SmolVLM-256M-Instruct-GGUF"},{"conf":"medium","context":"[MMMU] (pt)","model_id":"t5gemma-2-270m-270m","org":"google","rank":320,"root":"google/t5gemma-2-270m-270m","score":22.7,"variant":"t5gemma-2-270m-270m"},{"conf":"medium","context":"[MMMU] (pt)","model_id":"t5gemma-2-4b-4b","org":"google","rank":321,"root":"google/t5gemma-2-4b-4b","score":22.7,"variant":"t5gemma-2-4b-4b"},{"conf":"medium","context":"[MMMU] (pt)","model_id":"t5gemma-2-1b-1b","org":"google","rank":322,"root":"google/t5gemma-2-1b-1b","score":22.7,"variant":"t5gemma-2-1b-1b"},{"conf":"medium","context":"MMMU","model_id":null,"org":"nvidia","rank":323,"root":"nvidia/Nemotron-3.5-Content-Safety","score":3,"variant":"Nemotron-3.5-Content-Safety"},{"conf":"medium","context":"MMMU","model_id":"gemma-3-4b-pt","org":"nvidia","rank":324,"root":"google/gemma-3-4b-it","score":2.3,"variant":"Nemotron-3-Content-Safety"}],"schema_version":"local-ai-registry/v1","source":{"kind":"leaderboard-scrape","paths":["benchmarks/mmmu.html"],"url":"https://huggingface.co/spaces/mmmu"}},"meta":{"source":"registry"}}