{"data":{"category":"knowledge","id":"openbookqa","name":"OpenBookQA","rows":[{"conf":"medium","context":"OpenBookQA","model_id":"grin-moe","org":"microsoft","rank":1,"root":"microsoft/GRIN-MoE","score":89.8,"variant":"GRIN-MoE"},{"conf":"medium","context":"OpenBookQA<br>10-shot","model_id":"phi-3-small-8k-instruct","org":"microsoft","rank":2,"root":"microsoft/Phi-3-small-8k-instruct","score":88,"variant":"Phi-3-small-8k-instruct"},{"conf":"medium","context":"OpenBookQA<br>10-shot","model_id":"phi-3-small-128k-instruct","org":"microsoft","rank":3,"root":"microsoft/Phi-3-small-128k-instruct","score":87.8,"variant":"Phi-3-small-128k-instruct"},{"conf":"medium","context":"OpenBookQA<br>10-shot","model_id":"phi-3-medium-4k-instruct","org":"microsoft","rank":4,"root":"microsoft/Phi-3-medium-4k-instruct","score":87.4,"variant":"Phi-3-medium-4k-instruct"},{"conf":"medium","context":"OpenBookQA<br>10-shot","model_id":"phi-3-medium-128k-instruct","org":"microsoft","rank":5,"root":"microsoft/Phi-3-medium-128k-instruct","score":87.2,"variant":"Phi-3-medium-128k-instruct"},{"conf":"medium","context":"OpenBookQA (10-shot)","model_id":"phi-4-mini-instruct","org":"microsoft","rank":6,"root":"microsoft/Phi-4-mini-instruct","score":79.2,"variant":"Phi-4-mini-instruct"},{"conf":"medium","context":"OpenBookQA (10-shot)","model_id":"phi-4-mini-instruct","org":"unsloth","rank":7,"root":"microsoft/Phi-4-mini-instruct","score":79.2,"variant":"Phi-4-mini-instruct-GGUF"},{"conf":"medium","context":"OpenBookQA (10-shot)","model_id":"phi-4-mini-instruct","org":"unsloth","rank":8,"root":"microsoft/Phi-4-mini-instruct","score":79.2,"variant":"Phi-4-mini-instruct-unsloth-bnb-4bit"},{"conf":"medium","context":"OpenBookQA (10-shot)","model_id":"phi-4-mini-instruct","org":"unsloth","rank":9,"root":"microsoft/Phi-4-mini-instruct","score":79.2,"variant":"Phi-4-mini-instruct-bnb-4bit"},{"conf":"medium","context":"OpenBookQA (10-shot)","model_id":"phi-4-mini-instruct","org":"unsloth","rank":10,"root":"microsoft/Phi-4-mini-instruct","score":79.2,"variant":"Phi-4-mini-instruct"},{"conf":"medium","context":"OpenBookQA (10-shot)","model_id":"phi-4-mini-instruct","org":"Mungert","rank":11,"root":"microsoft/Phi-4-mini-instruct","score":79.2,"variant":"Phi-4-mini-instruct-GGUF"},{"conf":"low","context":"Average(Open-ended QA)","model_id":"ming-lite-omni","org":"inclusionAI","rank":12,"root":"inclusionAI/Ming-Lite-Omni","score":65.93,"variant":"Ming-Lite-Omni-1.5"},{"conf":"high","context":null,"model_id":"ling-lite","org":"inclusionAI","rank":13,"root":"inclusionAI/Ling-lite","score":61.98,"variant":"Ming-Lite-Omni"},{"conf":"medium","context":"OpenBookQA (0-shot)","model_id":"mistral-nemo-base-2407","org":"mistralai","rank":14,"root":"mistralai/Mistral-Nemo-Base-2407","score":60.6,"variant":"Mistral-Nemo-Instruct-2407"},{"conf":"medium","context":"OpenBookQA (0-shot)","model_id":"mistral-nemo-base-2407","org":"mistralai","rank":15,"root":"mistralai/Mistral-Nemo-Base-2407","score":60.6,"variant":"Mistral-Nemo-Base-2407"},{"conf":"medium","context":"OpenBookQA (0-shot)","model_id":"mistral-nemo-base-2407","org":"mistralai","rank":16,"root":"mistralai/Mistral-Nemo-Base-2407","score":60.6,"variant":"Mistral-Nemo-Instruct-FP8-2407"},{"conf":"medium","context":null,"model_id":"olmo-1b-0724-hf","org":"allenai","rank":17,"root":"allenai/OLMo-1B-0724-hf","score":51.2,"variant":"OLMo-1B-0724-hf"},{"conf":"medium","context":null,"model_id":"olmo-7b-0424-hf","org":"allenai","rank":18,"root":"allenai/OLMo-7B-0424-hf","score":51.2,"variant":"OLMo-7B-0424-hf"},{"conf":"medium","context":null,"model_id":"olmo-7b-0724-hf","org":"allenai","rank":19,"root":"allenai/OLMo-7B-0724-hf","score":51.2,"variant":"OLMo-7B-0724-hf"},{"conf":"medium","context":null,"model_id":null,"org":"allenai","rank":20,"root":"allenai/OLMo-7B-0424","score":51.2,"variant":"OLMo-7B-0424"},{"conf":"medium","context":"OpenBookQA","model_id":"nvidia-nemotron-3-super-120b-a12b-base-bf16","org":"nvidia","rank":21,"root":"nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-Base-BF16","score":50.2,"variant":"NVIDIA-Nemotron-3-Super-120B-A12B-Base-BF16"},{"conf":"medium","context":null,"model_id":"olmo-1b-hf","org":"allenai","rank":22,"root":"allenai/OLMo-1B-hf","score":49.8,"variant":"OLMo-1B-hf"},{"conf":"medium","context":null,"model_id":"olmo-7b","org":"allenai","rank":23,"root":"allenai/OLMo-7B","score":49.8,"variant":"OLMo-7B"},{"conf":"medium","context":null,"model_id":"olmo-7b-hf","org":"allenai","rank":24,"root":"allenai/OLMo-7B-hf","score":49.8,"variant":"OLMo-7B-hf"},{"conf":"medium","context":null,"model_id":"olmo-1b","org":"allenai","rank":25,"root":"allenai/OLMo-1B","score":49.8,"variant":"OLMo-1B"},{"conf":"medium","context":null,"model_id":"olmo-7b-twin-2t-hf","org":"allenai","rank":26,"root":"allenai/OLMo-7B-Twin-2T-hf","score":49.8,"variant":"OLMo-7B-Twin-2T-hf"},{"conf":"low","context":null,"model_id":"bagel-dpo-7b-v0-1","org":"TheBloke","rank":27,"root":"jondurbin/bagel-dpo-7b-v0.1","score":49.8,"variant":"bagel-dpo-7B-v0.1-GGUF"},{"conf":"medium","context":null,"model_id":"olmo-7b-twin-2t","org":"allenai","rank":28,"root":"allenai/OLMo-7B-Twin-2T","score":49.8,"variant":"OLMo-7B-Twin-2T"},{"conf":"medium","context":"OpenBookQA","model_id":"nvidia-nemotron-3-ultra-550b-a55b-base-bf16","org":"nvidia","rank":29,"root":"nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-Base-BF16","score":48.6,"variant":"NVIDIA-Nemotron-3-Ultra-550B-A55B-Base-BF16"},{"conf":"medium","context":"[OpenBookQA](https://arxiv.org/abs/1809.02789)","model_id":"gemma-1-1-2b-it","org":"google","rank":30,"root":"google/gemma-1.1-2b-it","score":47.8,"variant":"gemma-1.1-2b-it"},{"conf":"medium","context":"[OpenBookQA](https://arxiv.org/abs/1809.02789)","model_id":null,"org":"google","rank":31,"root":"google/gemma-2b-it-pytorch","score":47.8,"variant":"gemma-2b"},{"conf":"medium","context":"[OpenBookQA](https://arxiv.org/abs/1809.02789)","model_id":null,"org":"google","rank":32,"root":"google/gemma-2b-it-pytorch","score":47.8,"variant":"gemma-2b-it"},{"conf":"medium","context":"[OpenBookQA](https://arxiv.org/abs/1809.02789)","model_id":null,"org":"google","rank":33,"root":"google/gemma-7b-it-quant-pytorch","score":47.8,"variant":"gemma-7b"},{"conf":"medium","context":"[OpenBookQA](https://arxiv.org/abs/1809.02789)","model_id":"gemma-7b","org":"google","rank":34,"root":"google/gemma-7b","score":47.8,"variant":"gemma-7b-it"},{"conf":"medium","context":"[OpenBookQA](https://arxiv.org/abs/1809.02789)","model_id":"gemma-1-1-7b-it","org":"google","rank":35,"root":"google/gemma-1.1-7b-it","score":47.8,"variant":"gemma-1.1-7b-it"},{"conf":"medium","context":"[OpenBookQA](https://arxiv.org/abs/1809.02789)","model_id":null,"org":"google","rank":36,"root":"google/gemma-2b-it-pytorch","score":47.8,"variant":"gemma-2b-AWQ"},{"conf":"medium","context":"[OpenBookQA](https://arxiv.org/abs/1809.02789)","model_id":null,"org":"alpindale","rank":37,"root":"google/gemma-7b-it-quant-pytorch","score":47.8,"variant":"gemma-7b"},{"conf":"medium","context":"[OpenBookQA](https://arxiv.org/abs/1809.02789)","model_id":null,"org":"alpindale","rank":38,"root":"google/gemma-2b-it-pytorch","score":47.8,"variant":"gemma-2b"},{"conf":"medium","context":"[OpenBookQA](https://arxiv.org/abs/1809.02789)","model_id":null,"org":"google","rank":39,"root":"google/gemma-7b-it-quant-pytorch","score":47.8,"variant":"gemma-7b-AWQ"},{"conf":"medium","context":"[OpenBookQA](https://arxiv.org/abs/1809.02789)","model_id":null,"org":"alpindale","rank":40,"root":"google/gemma-7b-it-quant-pytorch","score":47.8,"variant":"gemma-7b-it"},{"conf":"medium","context":"[OpenBookQA](https://arxiv.org/abs/1809.02789)","model_id":null,"org":"google","rank":41,"root":"google/gemma-7b-it-quant-pytorch","score":47.8,"variant":"gemma-7b-GGUF"},{"conf":"medium","context":"OpenBookQA","model_id":"nvidia-nemotron-nano-9b-v2-base","org":"nvidia","rank":42,"root":"nvidia/NVIDIA-Nemotron-Nano-9B-v2-Base","score":46,"variant":"NVIDIA-Nemotron-Nano-9B-v2-Base"},{"conf":"medium","context":"OpenBookQA","model_id":"nvidia-nemotron-nano-12b-v2-base","org":"nvidia","rank":43,"root":"nvidia/NVIDIA-Nemotron-Nano-12B-v2-Base","score":46,"variant":"NVIDIA-Nemotron-Nano-12B-v2-Base"},{"conf":"medium","context":"OpenBookQA (0-shot, acc_norm)","model_id":"nvidia-nemotron-3-nano-30b-a3b-base-bf16","org":"nvidia","rank":44,"root":"nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16","score":44.8,"variant":"NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16"},{"conf":"medium","context":"OpenBookQA","model_id":"smollm2-1-7b","org":"Mungert","rank":45,"root":"HuggingFaceTB/SmolLM2-1.7B","score":42.2,"variant":"SmolLM2-1.7B-Instruct-GGUF"},{"conf":"medium","context":null,"model_id":null,"org":"TheBloke","rank":46,"root":"NousResearch/Nous-Hermes-Llama2-70b","score":38.8,"variant":"Nous-Hermes-Llama2-70B-GGUF"},{"conf":"medium","context":null,"model_id":"beyonder-4x7b-v2","org":"TheBloke","rank":47,"root":"mlabonne/Beyonder-4x7B-v2","score":37.8,"variant":"Beyonder-4x7B-v2-GGUF"},{"conf":"medium","context":null,"model_id":"marcoro14-7b-slerp","org":"mlabonne","rank":48,"root":"mlabonne/Marcoro14-7B-slerp","score":37.8,"variant":"Beyonder-4x7B-v2"},{"conf":"medium","context":"OpenBookQA","model_id":"smollm2-360m","org":"Mungert","rank":49,"root":"HuggingFaceTB/SmolLM2-360M","score":37.4,"variant":"SmolLM2-360M-Instruct-GGUF"},{"conf":"medium","context":null,"model_id":null,"org":"mlabonne","rank":50,"root":"AIDC-ai-business/Marcoroni-7B-v3","score":36.4,"variant":"Marcoro14-7B-slerp"},{"conf":"medium","context":null,"model_id":null,"org":"TheBloke","rank":51,"root":"NousResearch/Nous-Hermes-13b","score":35.6,"variant":"Nous-Hermes-13B-GGUF"},{"conf":"medium","context":null,"model_id":null,"org":"TheBloke","rank":52,"root":"HyperbeeAI/Tulpar-7b-v0","score":35.6,"variant":"Tulpar-7B-v0-GGUF"},{"conf":"medium","context":null,"model_id":null,"org":"TheBloke","rank":53,"root":"NousResearch/Redmond-Puffin-13B","score":35.6,"variant":"Redmond-Puffin-13B-GGUF"},{"conf":"medium","context":null,"model_id":null,"org":"TheBloke","rank":54,"root":"—","score":35.6,"variant":"Nous-Hermes-13B-SuperHOT-8K-fp16"},{"conf":"medium","context":null,"model_id":null,"org":"TheBloke","rank":55,"root":"NousResearch/Redmond-Puffin-13B","score":35.6,"variant":"Redmond-Puffin-13B-GPTQ"},{"conf":"medium","context":null,"model_id":null,"org":"TheBloke","rank":56,"root":"—","score":35.6,"variant":"Nous-Hermes-13B-SuperHOT-8K-GPTQ"},{"conf":"medium","context":null,"model_id":"nous-hermes-2-yi-34b","org":"TheBloke","rank":57,"root":"NousResearch/Nous-Hermes-2-Yi-34B","score":35.2,"variant":"Nous-Hermes-2-Yi-34B-GGUF"},{"conf":"medium","context":null,"model_id":"nous-hermes-2-yi-34b","org":"TheBloke","rank":58,"root":"NousResearch/Nous-Hermes-2-Yi-34B","score":35.2,"variant":"Nous-Hermes-2-Yi-34B-AWQ"},{"conf":"medium","context":null,"model_id":"laser-dolphin-mixtral-2x7b-dpo","org":"TheBloke","rank":59,"root":"macadeliccc/laser-dolphin-mixtral-2x7b-dpo","score":35,"variant":"laser-dolphin-mixtral-2x7b-dpo-GGUF"},{"conf":"medium","context":"OpenbookQA","model_id":"bitnet-b1-58-2b-4t-bf16","org":"microsoft","rank":60,"root":"microsoft/bitnet-b1.58-2B-4T-bf16","score":34.8,"variant":"bitnet-b1.58-2B-4T-gguf"},{"conf":"medium","context":"OpenbookQA","model_id":"bitnet-b1-58-2b-4t","org":"microsoft","rank":61,"root":"microsoft/bitnet-b1.58-2B-4T","score":34.8,"variant":"bitnet-b1.58-2B-4T"},{"conf":"medium","context":"OpenbookQA","model_id":"bitnet-b1-58-2b-4t-bf16","org":"microsoft","rank":62,"root":"microsoft/bitnet-b1.58-2B-4T-bf16","score":34.8,"variant":"bitnet-b1.58-2B-4T-bf16"},{"conf":"medium","context":null,"model_id":null,"org":"TheBloke","rank":63,"root":"teknium/OpenHermes-2-Mistral-7B","score":34.8,"variant":"OpenHermes-2-Mistral-7B-GGUF"},{"conf":"low","context":null,"model_id":null,"org":"microsoft","rank":64,"root":"microsoft/dolly-v2-7b-olive-optimized","score":34.8,"variant":"dolly-v2-7b-olive-optimized"},{"conf":"medium","context":null,"model_id":null,"org":"TheBloke","rank":65,"root":"teknium/OpenHermes-2-Mistral-7B","score":34.8,"variant":"OpenHermes-2-Mistral-7B-AWQ"},{"conf":"medium","context":null,"model_id":"openhermes-2-5-mistral-7b","org":"TheBloke","rank":66,"root":"teknium/OpenHermes-2.5-Mistral-7B","score":34.6,"variant":"OpenHermes-2.5-Mistral-7B-AWQ"},{"conf":"medium","context":null,"model_id":"openhermes-2-5-mistral-7b","org":"TheBloke","rank":67,"root":"teknium/OpenHermes-2.5-Mistral-7B","score":34.6,"variant":"OpenHermes-2.5-Mistral-7B-GGUF"},{"conf":"medium","context":null,"model_id":null,"org":"TheBloke","rank":68,"root":"NurtureAI/OpenHermes-2.5-Mistral-7B-16k","score":34.6,"variant":"OpenHermes-2.5-Mistral-7B-16k-GGUF"},{"conf":"medium","context":null,"model_id":"openhermes-2-5-mistral-7b","org":"TheBloke","rank":69,"root":"teknium/OpenHermes-2.5-Mistral-7B","score":34.6,"variant":"OpenHermes-2.5-Mistral-7B-GPTQ"},{"conf":"medium","context":"OpenBookQA","model_id":"smollm2-135m","org":"Mungert","rank":70,"root":"HuggingFaceTB/SmolLM2-135M","score":34.6,"variant":"SmolLM2-135M-Instruct-GGUF"},{"conf":"medium","context":null,"model_id":"nous-hermes-2-mixtral-8x7b-dpo","org":"TheBloke","rank":71,"root":"NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO","score":34.4,"variant":"Nous-Hermes-2-Mixtral-8x7B-DPO-GGUF"},{"conf":"medium","context":null,"model_id":"dpopenhermes-7b","org":"TheBloke","rank":72,"root":"openaccess-ai-collective/DPOpenHermes-7B","score":34.4,"variant":"DPOpenHermes-7B-GGUF"},{"conf":"medium","context":null,"model_id":"openhermes-2-5-mistral-7b","org":"mlabonne","rank":73,"root":"teknium/OpenHermes-2.5-Mistral-7B","score":34.4,"variant":"NeuralHermes-2.5-Mistral-7B-laser"},{"conf":"medium","context":null,"model_id":"nous-hermes-2-mixtral-8x7b-dpo","org":"TheBloke","rank":74,"root":"NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO","score":34.4,"variant":"Nous-Hermes-2-Mixtral-8x7B-DPO-GPTQ"},{"conf":"medium","context":null,"model_id":"nous-hermes-2-solar-10-7b","org":"TheBloke","rank":75,"root":"NousResearch/Nous-Hermes-2-SOLAR-10.7B","score":34.2,"variant":"Nous-Hermes-2-SOLAR-10.7B-GGUF"},{"conf":"medium","context":null,"model_id":null,"org":"TheBloke","rank":76,"root":"openaccess-ai-collective/minotaur-13b-fixed","score":34.2,"variant":"minotaur-13B-fixed-GGUF"},{"conf":"medium","context":null,"model_id":"nous-hermes-2-mixtral-8x7b-sft","org":"TheBloke","rank":77,"root":"NousResearch/Nous-Hermes-2-Mixtral-8x7B-SFT","score":34,"variant":"Nous-Hermes-2-Mixtral-8x7B-SFT-GGUF"},{"conf":"medium","context":null,"model_id":null,"org":"TheBloke","rank":78,"root":"teknium/CollectiveCognition-v1-Mistral-7B","score":33.8,"variant":"CollectiveCognition-v1-Mistral-7B-GGUF"},{"conf":"low","context":null,"model_id":null,"org":"microsoft","rank":79,"root":"microsoft/Llama2-7b-WhoIsHarryPotter","score":33.8,"variant":"Llama2-7b-WhoIsHarryPotter"},{"conf":"medium","context":null,"model_id":null,"org":"TheBloke","rank":80,"root":"teknium/CollectiveCognition-v1.1-Mistral-7B","score":32.4,"variant":"CollectiveCognition-v1.1-Mistral-7B-GGUF"},{"conf":"medium","context":null,"model_id":null,"org":"TheBloke","rank":81,"root":"—","score":19.6,"variant":"Redmond-Hermes-Coder-GPTQ"}],"schema_version":"local-ai-registry/v1","source":{"kind":"leaderboard-scrape","paths":["benchmarks/openbookqa.html"],"url":"https://huggingface.co/spaces/openbookqa"}},"meta":{"source":"registry"}}