{
 "edition": "2026.10",
 "date": "2026-10-14",
 "ollaya": "0.13.0",
 "score": "Mean over 14 tasks (13 public subsets and typed-decisions, equal weight) of the chance-corrected accuracy 100 * (accuracy - chance) / (1 - chance); chance is the mean of 1/K over the task's questions. A model trained on a task's train split is scored on the other tasks",
 "devices": [
  {
   "id": "rtx-5090",
   "name": "RTX 5090",
   "kind": "NVIDIA GPU, CUDA, 32 GB",
   "machine": "rtx-5090",
   "device": "cuda"
  },
  {
   "id": "threadripper-3960x",
   "name": "Threadripper 3960X",
   "kind": "CPU, 24 cores",
   "machine": "rtx-5090",
   "device": "cpu"
  }
 ],
 "models": [
  {
   "model": "decima:small",
   "url": "https://bench.ollaya.dev/models/decima:small",
   "badge": "https://bench.ollaya.dev/badge/decima/small.svg",
   "publisher": "A. M. Madani",
   "params": "122M",
   "runtime": "ONNX Runtime",
   "license": "Apache-2.0",
   "digest": "d0cdad8529bb48df9511afb3f16e97254bceb40d9ec38a6dd4064930e2352db3",
   "score": 17.22,
   "score_ci": [
    15.22,
    19.43
   ],
   "rank": 30,
   "trained_on": [],
   "public_macro_accuracy": 0.4622,
   "public_pooled_ece": 0.0646,
   "typed_accuracy": 0.4145,
   "typed_ece": 0.1138,
   "speed": {
    "rtx-5090": {
     "p50_ms": 13.1,
     "p90_ms": 14.7,
     "load_ms": 4142.7,
     "memory_mb": 1270,
     "precision": "F32"
    },
    "threadripper-3960x": {
     "p50_ms": 625,
     "p90_ms": 645.1,
     "load_ms": 1812.6,
     "memory_mb": 589,
     "precision": "F32"
    }
   }
  },
  {
   "model": "decima:base",
   "url": "https://bench.ollaya.dev/models/decima:base",
   "badge": "https://bench.ollaya.dev/badge/decima/base.svg",
   "publisher": "A. M. Madani",
   "params": "321M",
   "runtime": "ONNX Runtime",
   "license": "Apache-2.0",
   "digest": "56ad25422fccba646359e24655a82fc836a07b760cdc121aced5a56f222c25f1",
   "score": 31.04,
   "score_ci": [
    28.88,
    33.22
   ],
   "rank": 25,
   "trained_on": [],
   "public_macro_accuracy": 0.5503,
   "public_pooled_ece": 0.0525,
   "typed_accuracy": 0.4835,
   "typed_ece": 0.0758,
   "speed": {
    "rtx-5090": {
     "p50_ms": 25.9,
     "p90_ms": 26.3,
     "load_ms": 4931.3,
     "memory_mb": 2166,
     "precision": "F32"
    },
    "threadripper-3960x": {
     "p50_ms": 1210.6,
     "p90_ms": 1257.1,
     "load_ms": 4143.3,
     "memory_mb": 1125,
     "precision": "F32"
    }
   }
  },
  {
   "model": "decima:agent",
   "url": "https://bench.ollaya.dev/models/decima:agent",
   "badge": "https://bench.ollaya.dev/badge/decima/agent.svg",
   "publisher": "A. M. Madani",
   "params": "321M",
   "runtime": "ONNX Runtime",
   "license": "Apache-2.0",
   "digest": "6b5f4883a460936fd5dd3b427fcd57b6711b668495bbdc5c960f5fbb62532c83",
   "score": 27.87,
   "score_ci": [
    25.69,
    29.87
   ],
   "rank": 25,
   "trained_on": [],
   "public_macro_accuracy": 0.5326,
   "public_pooled_ece": 0.2107,
   "typed_accuracy": 0.4605,
   "typed_ece": 0.193,
   "speed": {
    "rtx-5090": {
     "p50_ms": 27.1,
     "p90_ms": 27.6,
     "load_ms": 5934.9,
     "memory_mb": 2806,
     "precision": "F32"
    },
    "threadripper-3960x": {
     "p50_ms": 1230.2,
     "p90_ms": 1267.6,
     "load_ms": 4158.2,
     "memory_mb": 1131,
     "precision": "F32"
    }
   }
  },
  {
   "model": "laya:en",
   "url": "https://bench.ollaya.dev/models/laya:en",
   "badge": "https://bench.ollaya.dev/badge/laya/en.svg",
   "publisher": "Convai Innovations",
   "params": "421M",
   "runtime": "ONNX Runtime",
   "license": "Apache-2.0",
   "digest": "c305a9276531a47000bf93559d2c94f1ed6cbb67055c9151084682dad7655e9d",
   "score": 30.12,
   "score_ci": [
    28.16,
    31.99
   ],
   "rank": 25,
   "trained_on": [],
   "public_macro_accuracy": 0.5321,
   "public_pooled_ece": 0.1063,
   "typed_accuracy": 0.3575,
   "typed_ece": 0.174,
   "speed": {
    "rtx-5090": {
     "p50_ms": 18.3,
     "p90_ms": 19.2,
     "load_ms": 3833.4,
     "memory_mb": 1654,
     "precision": "F16"
    },
    "threadripper-3960x": {
     "p50_ms": 945.2,
     "p90_ms": 950.2,
     "load_ms": 6337.7,
     "memory_mb": 3207,
     "precision": "F32"
    }
   }
  },
  {
   "model": "laya:multilingual",
   "url": "https://bench.ollaya.dev/models/laya:multilingual",
   "badge": "https://bench.ollaya.dev/badge/laya/multilingual.svg",
   "publisher": "Convai Innovations",
   "params": "322M",
   "runtime": "ONNX Runtime",
   "license": "Apache-2.0",
   "digest": "2840506e1f978aeb696a6f84af43ecc7fca95754cb63a67bafe6132532d384eb",
   "score": 36.74,
   "score_ci": [
    34.66,
    38.94
   ],
   "rank": 21,
   "trained_on": [],
   "public_macro_accuracy": 0.5787,
   "public_pooled_ece": 0.1548,
   "typed_accuracy": 0.349,
   "typed_ece": 0.3164,
   "speed": {
    "rtx-5090": {
     "p50_ms": 14.3,
     "p90_ms": 14.6,
     "load_ms": 3619.3,
     "memory_mb": 1398,
     "precision": "F16"
    },
    "threadripper-3960x": {
     "p50_ms": 515.5,
     "p90_ms": 517.2,
     "load_ms": 4780.1,
     "memory_mb": 1941,
     "precision": "F32"
    }
   }
  },
  {
   "model": "laya:typed-decisions",
   "url": "https://bench.ollaya.dev/models/laya:typed-decisions",
   "badge": "https://bench.ollaya.dev/badge/laya/typed-decisions.svg",
   "publisher": "Convai Innovations",
   "params": "421M",
   "runtime": "ONNX Runtime",
   "license": "Apache-2.0",
   "digest": "6d17e5fbcbb8215a74f2efd0dcc0ffdbd472e9b25ccb07867f9f146a791673ba",
   "score": 40.26,
   "score_ci": [
    37.78,
    42.63
   ],
   "rank": 21,
   "trained_on": [
    "typed-decisions"
   ],
   "public_macro_accuracy": 0.5935,
   "public_pooled_ece": 0.0219,
   "typed_accuracy": 0.767,
   "typed_ece": 0.2142,
   "speed": {
    "rtx-5090": {
     "p50_ms": 20.4,
     "p90_ms": 21.1,
     "load_ms": 3862.5,
     "memory_mb": 1654,
     "precision": "F16"
    },
    "threadripper-3960x": {
     "p50_ms": 944.1,
     "p90_ms": 952,
     "load_ms": 6842.3,
     "memory_mb": 3215,
     "precision": "F32"
    }
   }
  },
  {
   "model": "laya",
   "url": "https://bench.ollaya.dev/models/laya",
   "badge": "https://bench.ollaya.dev/badge/laya.svg",
   "publisher": "Convai Innovations",
   "params": "",
   "runtime": "router",
   "license": "Apache-2.0",
   "digest": "89aa53ce17e105dc801b899b4c1b6d4fcbee7ea2710a303e149a5129476bc2c6",
   "score": 30.41,
   "score_ci": [
    28.35,
    32.41
   ],
   "rank": 25,
   "trained_on": [],
   "public_macro_accuracy": 0.5348,
   "public_pooled_ece": 0.1032,
   "typed_accuracy": 0.3575,
   "typed_ece": 0.174,
   "speed": {
    "rtx-5090": null,
    "threadripper-3960x": null
   }
  },
  {
   "model": "nli:deberta-v3-large",
   "url": "https://bench.ollaya.dev/models/nli:deberta-v3-large",
   "badge": "https://bench.ollaya.dev/badge/nli/deberta-v3-large.svg",
   "publisher": "Moritz Laurer",
   "params": "435M",
   "runtime": "ONNX Runtime",
   "license": "MIT",
   "digest": "b33142f79a0338ab5398c7be66c18f58483c07aaec8262dc1f314aa02cbee205",
   "score": 20.98,
   "score_ci": [
    19.01,
    23.07
   ],
   "rank": 29,
   "trained_on": [],
   "public_macro_accuracy": 0.4583,
   "public_pooled_ece": 0.1956,
   "typed_accuracy": 0.536,
   "typed_ece": 0.0959,
   "speed": {
    "rtx-5090": {
     "p50_ms": 17.7,
     "p90_ms": 18,
     "load_ms": 8222.5,
     "memory_mb": 2682,
     "precision": "F32"
    },
    "threadripper-3960x": {
     "p50_ms": 2318.2,
     "p90_ms": 2438.6,
     "load_ms": 7709.9,
     "memory_mb": 3848,
     "precision": "F32"
    }
   }
  },
  {
   "model": "nli:modernbert-large",
   "url": "https://bench.ollaya.dev/models/nli:modernbert-large",
   "badge": "https://bench.ollaya.dev/badge/nli/modernbert-large.svg",
   "publisher": "Moritz Laurer",
   "params": "396M",
   "runtime": "ONNX Runtime",
   "license": "Apache-2.0",
   "digest": "0c984ae95685d511953341fd75e5cf49e31bae5110574dba78999c5f2267cd7f",
   "score": 17.07,
   "score_ci": [
    14.87,
    19.15
   ],
   "rank": 30,
   "trained_on": [],
   "public_macro_accuracy": 0.4232,
   "public_pooled_ece": 0.1111,
   "typed_accuracy": 0.5155,
   "typed_ece": 0.1481,
   "speed": {
    "rtx-5090": {
     "p50_ms": 16.8,
     "p90_ms": 17.2,
     "load_ms": 6018.1,
     "memory_mb": 2678,
     "precision": "F32"
    },
    "threadripper-3960x": {
     "p50_ms": 2101.4,
     "p90_ms": 2141.9,
     "load_ms": 6018,
     "memory_mb": 2991,
     "precision": "F32"
    }
   }
  },
  {
   "model": "gliclass:large",
   "url": "https://bench.ollaya.dev/models/gliclass:large",
   "badge": "https://bench.ollaya.dev/badge/gliclass/large.svg",
   "publisher": "Knowledgator",
   "params": "439M",
   "runtime": "ONNX Runtime",
   "license": "Apache-2.0",
   "digest": "8c4c3acece90b83f44da3694f0aedcb90503de37aff702995e18dc317a63e5a6",
   "score": 15.34,
   "score_ci": [
    13.14,
    17.56
   ],
   "rank": 31,
   "trained_on": [],
   "public_macro_accuracy": 0.4336,
   "public_pooled_ece": 0.2838,
   "typed_accuracy": 0.477,
   "typed_ece": 0.2189,
   "speed": {
    "rtx-5090": {
     "p50_ms": 14.1,
     "p90_ms": 14.5,
     "load_ms": 7149.2,
     "memory_mb": 2702,
     "precision": "F32"
    },
    "threadripper-3960x": {
     "p50_ms": 2133.2,
     "p90_ms": 2185.3,
     "load_ms": 6786.5,
     "memory_mb": 2415,
     "precision": "F32"
    }
   }
  },
  {
   "model": "von:1.1",
   "url": "https://bench.ollaya.dev/models/von:1.1",
   "badge": "https://bench.ollaya.dev/badge/von/1.1.svg",
   "publisher": "Victor Hugo Panisa",
   "params": "395M",
   "runtime": "ONNX Runtime",
   "license": "Apache-2.0",
   "digest": "25606bfd644593e63b4c63b8219d4fd0085d11634d6fa4701268d77cf804f689",
   "score": 23.7,
   "score_ci": [
    21.52,
    26.01
   ],
   "rank": 28,
   "trained_on": [],
   "public_macro_accuracy": 0.4853,
   "public_pooled_ece": 0.0979,
   "typed_accuracy": 0.4465,
   "typed_ece": 0.0521,
   "speed": {
    "rtx-5090": {
     "p50_ms": 19.3,
     "p90_ms": 19.8,
     "load_ms": 5703.1,
     "memory_mb": 2686,
     "precision": "F32"
    },
    "threadripper-3960x": {
     "p50_ms": 2590.6,
     "p90_ms": 2602.9,
     "load_ms": 5948.1,
     "memory_mb": 1747,
     "precision": "F32"
    }
   }
  },
  {
   "model": "qwen3guard:0.6b",
   "url": "https://bench.ollaya.dev/models/qwen3guard:0.6b",
   "badge": null,
   "publisher": "Qwen",
   "params": "0.6B",
   "runtime": "ONNX Runtime",
   "license": "Apache-2.0",
   "digest": "396892330ef0cb0c3057575dee6a9c46a8c8bfe49afafb738ed1addf2c466a63",
   "score": null,
   "score_ci": null,
   "rank": null,
   "trained_on": [],
   "public_macro_accuracy": null,
   "public_pooled_ece": null,
   "typed_accuracy": null,
   "typed_ece": null,
   "speed": {
    "rtx-5090": {
     "p50_ms": 30.2,
     "p90_ms": 32.3,
     "load_ms": 10289.2,
     "memory_mb": 3701,
     "precision": "F32"
    },
    "threadripper-3960x": {
     "p50_ms": 1493.7,
     "p90_ms": 1505.5,
     "load_ms": 9624.7,
     "memory_mb": 5152,
     "precision": "F32"
    }
   }
  },
  {
   "model": "snap:2b",
   "url": "https://bench.ollaya.dev/models/snap:2b",
   "badge": "https://bench.ollaya.dev/badge/snap/2b.svg",
   "publisher": "logitlab",
   "params": "2B",
   "runtime": "llama.cpp",
   "license": "Apache-2.0",
   "digest": "f356b13c94415f1237ab94d752b0de1705ed98f38ee389549264dfa53532f171",
   "score": 55.75,
   "score_ci": [
    53.79,
    57.66
   ],
   "rank": 16,
   "trained_on": [],
   "public_macro_accuracy": 0.7017,
   "public_pooled_ece": 0.049,
   "typed_accuracy": 0.6475,
   "typed_ece": 0.051,
   "speed": {
    "rtx-5090": {
     "p50_ms": 82.3,
     "p90_ms": 84.6,
     "load_ms": 6249.7,
     "memory_mb": 3450,
     "precision": "Q8_0"
    },
    "threadripper-3960x": {
     "p50_ms": 2649.8,
     "p90_ms": 2729.4,
     "load_ms": 6428.3,
     "memory_mb": 2958,
     "precision": "Q8_0"
    }
   }
  },
  {
   "model": "snap:2b-q4_k_m",
   "url": "https://bench.ollaya.dev/models/snap:2b-q4_k_m",
   "badge": "https://bench.ollaya.dev/badge/snap/2b-q4_k_m.svg",
   "publisher": "logitlab",
   "params": "2B",
   "runtime": "llama.cpp",
   "license": "Apache-2.0",
   "digest": "a19db7952873b56351a1a1e820fa571c65a8f50b4e9862addbf0dda063661036",
   "score": 54.35,
   "score_ci": [
    52.28,
    56.41
   ],
   "rank": 16,
   "trained_on": [],
   "public_macro_accuracy": 0.6907,
   "public_pooled_ece": 0.0435,
   "typed_accuracy": 0.6555,
   "typed_ece": 0.0427,
   "speed": {
    "rtx-5090": {
     "p50_ms": 78.2,
     "p90_ms": 78.7,
     "load_ms": 4096.6,
     "memory_mb": 2514,
     "precision": "Q4_K - MEDIUM"
    },
    "threadripper-3960x": {
     "p50_ms": 1734.7,
     "p90_ms": 1765,
     "load_ms": 5514.5,
     "memory_mb": 2802,
     "precision": "Q4_K - MEDIUM"
    }
   }
  },
  {
   "model": "d1:3b",
   "url": "https://bench.ollaya.dev/models/d1:3b",
   "badge": "https://bench.ollaya.dev/badge/d1/3b.svg",
   "publisher": "LiquidAI",
   "params": "3B",
   "runtime": "llama.cpp",
   "license": "LFM Open License v1.0",
   "digest": "d92cc117d0d9ef8624b4408ef4823b4b8efef403fee9f7d54629cb037a2b58da",
   "score": 63.13,
   "score_ci": [
    61.43,
    64.97
   ],
   "rank": 2,
   "trained_on": [],
   "public_macro_accuracy": 0.7507,
   "public_pooled_ece": 0.0625,
   "typed_accuracy": 0.65,
   "typed_ece": 0.0353,
   "speed": {
    "rtx-5090": {
     "p50_ms": 50.8,
     "p90_ms": 52.5,
     "load_ms": 6859.9,
     "memory_mb": 4082,
     "precision": "Q8_0"
    },
    "threadripper-3960x": {
     "p50_ms": 1757.4,
     "p90_ms": 1944.1,
     "load_ms": 6607.3,
     "memory_mb": 3349,
     "precision": "Q8_0"
    }
   }
  },
  {
   "model": "winnow:e4b",
   "url": "https://bench.ollaya.dev/models/winnow:e4b",
   "badge": "https://bench.ollaya.dev/badge/winnow/e4b.svg",
   "publisher": "EldanRing",
   "params": "7.5B",
   "runtime": "llama.cpp",
   "license": "Apache-2.0",
   "digest": "dd4bf88aa50bebb02e7a26fbebed14682befd037e3e89789ee22b9793f82d226",
   "score": 59.93,
   "score_ci": [
    57.89,
    61.94
   ],
   "rank": 6,
   "trained_on": [],
   "public_macro_accuracy": 0.7343,
   "public_pooled_ece": 0.0714,
   "typed_accuracy": 0.724,
   "typed_ece": 0.0199,
   "speed": {
    "rtx-5090": {
     "p50_ms": 112.6,
     "p90_ms": 119.4,
     "load_ms": 17524.7,
     "memory_mb": 6356,
     "precision": "Q8_0"
    },
    "threadripper-3960x": {
     "p50_ms": 4095.3,
     "p90_ms": 4149.7,
     "load_ms": 18974.6,
     "memory_mb": 8265,
     "precision": "Q8_0"
    }
   }
  },
  {
   "model": "credence:e4b",
   "url": "https://bench.ollaya.dev/models/credence:e4b",
   "badge": "https://bench.ollaya.dev/badge/credence/e4b.svg",
   "publisher": "Txoka",
   "params": "7.5B",
   "runtime": "llama.cpp",
   "license": "Apache-2.0",
   "digest": "ab2f84b8484681a88d465b7771f43804886e7bf4d529a731d4b8ce24bea57b69",
   "score": 60.99,
   "score_ci": [
    58.97,
    62.79
   ],
   "rank": 4,
   "trained_on": [],
   "public_macro_accuracy": 0.7414,
   "public_pooled_ece": 0.0545,
   "typed_accuracy": 0.725,
   "typed_ece": 0.0445,
   "speed": {
    "rtx-5090": {
     "p50_ms": 120.7,
     "p90_ms": 122.5,
     "load_ms": 17863,
     "memory_mb": 6356,
     "precision": "Q8_0"
    },
    "threadripper-3960x": {
     "p50_ms": 4038.9,
     "p90_ms": 4113.5,
     "load_ms": 19870.5,
     "memory_mb": 8946,
     "precision": "Q8_0"
    }
   }
  },
  {
   "model": "credence:e4b-calibrated",
   "url": "https://bench.ollaya.dev/models/credence:e4b-calibrated",
   "badge": "https://bench.ollaya.dev/badge/credence/e4b-calibrated.svg",
   "publisher": "Txoka",
   "params": "7.5B",
   "runtime": "llama.cpp",
   "license": "Apache-2.0",
   "digest": "e9d941a002f0bba8438d09f8c8fe9bdad8a830dab0ea8cba13cd758e1fafb565",
   "score": 60.46,
   "score_ci": [
    58.59,
    62.41
   ],
   "rank": 6,
   "trained_on": [],
   "public_macro_accuracy": 0.7368,
   "public_pooled_ece": 0.0378,
   "typed_accuracy": 0.7235,
   "typed_ece": 0.0436,
   "speed": {
    "rtx-5090": {
     "p50_ms": 114.3,
     "p90_ms": 115.4,
     "load_ms": 17368.1,
     "memory_mb": 6356,
     "precision": "Q8_0"
    },
    "threadripper-3960x": {
     "p50_ms": 4097.1,
     "p90_ms": 4183.9,
     "load_ms": 19453.5,
     "memory_mb": 8945,
     "precision": "Q8_0"
    }
   }
  },
  {
   "model": "jevk5:4b",
   "url": "https://bench.ollaya.dev/models/jevk5:4b",
   "badge": "https://bench.ollaya.dev/badge/jevk5/4b.svg",
   "publisher": "alibiserikbay",
   "params": "4B",
   "runtime": "llama.cpp",
   "license": "Apache-2.0",
   "digest": "58b5d6516410070d5f165d77357ed7bfefe5ffedde63e6ae75ab5101d4fe0221",
   "score": 48.6,
   "score_ci": [
    46.6,
    50.69
   ],
   "rank": 19,
   "trained_on": [],
   "public_macro_accuracy": 0.6205,
   "public_pooled_ece": 0.04,
   "typed_accuracy": 0.629,
   "typed_ece": 0.0888,
   "speed": {
    "rtx-5090": {
     "p50_ms": 95.3,
     "p90_ms": 97.5,
     "load_ms": 10392.8,
     "memory_mb": 5910,
     "precision": "Q8_0"
    },
    "threadripper-3960x": {
     "p50_ms": 6979.7,
     "p90_ms": 7267.8,
     "load_ms": 11265.7,
     "memory_mb": 4997,
     "precision": "Q8_0"
    }
   }
  },
  {
   "model": "decider:0.8b",
   "url": "https://bench.ollaya.dev/models/decider:0.8b",
   "badge": "https://bench.ollaya.dev/badge/decider/0.8b.svg",
   "publisher": "Mapika",
   "params": "0.75B",
   "runtime": "ONNX Runtime",
   "license": "Apache-2.0",
   "digest": "97a22ac9c6c42eebb762ddb5eb47f0542b5e0788cf1cea31cee6205d5bd1064a",
   "score": 47.82,
   "score_ci": [
    45.72,
    49.98
   ],
   "rank": 19,
   "trained_on": [],
   "public_macro_accuracy": 0.6687,
   "public_pooled_ece": 0.0612,
   "typed_accuracy": 0.5055,
   "typed_ece": 0.1715,
   "speed": {
    "rtx-5090": {
     "p50_ms": 189,
     "p90_ms": 190.8,
     "load_ms": 17435.7,
     "memory_mb": 5116,
     "precision": "F32"
    },
    "threadripper-3960x": {
     "p50_ms": 3105.5,
     "p90_ms": 3108.8,
     "load_ms": 15122.1,
     "memory_mb": 6740,
     "precision": "F32"
    }
   }
  },
  {
   "model": "decider:2b",
   "url": "https://bench.ollaya.dev/models/decider:2b",
   "badge": "https://bench.ollaya.dev/badge/decider/2b.svg",
   "publisher": "Mapika",
   "params": "1.9B",
   "runtime": "ONNX Runtime",
   "license": "Apache-2.0",
   "digest": "5507d6f80e9a2c82d7dd4c0eb6cffc03916669a4ce5f0fe506ab5e434cf9a62a",
   "score": 54.34,
   "score_ci": [
    52.15,
    56.2
   ],
   "rank": 16,
   "trained_on": [],
   "public_macro_accuracy": 0.7033,
   "public_pooled_ece": 0.0872,
   "typed_accuracy": 0.591,
   "typed_ece": 0.0881,
   "speed": {
    "rtx-5090": {
     "p50_ms": 192.7,
     "p90_ms": 194.1,
     "load_ms": 32189.4,
     "memory_mb": 12176,
     "precision": "F32"
    },
    "threadripper-3960x": {
     "p50_ms": 4540.2,
     "p90_ms": 4560.1,
     "load_ms": 36578.8,
     "memory_mb": 14924,
     "precision": "F32"
    }
   }
  },
  {
   "model": "decider:4b",
   "url": "https://bench.ollaya.dev/models/decider:4b",
   "badge": "https://bench.ollaya.dev/badge/decider/4b.svg",
   "publisher": "Mapika",
   "params": "4.2B",
   "runtime": "ONNX Runtime",
   "license": "Apache-2.0",
   "digest": "920dea4d1851680da7129db890b97542e824a3709f2ccb09b7f35ce6b4ff4669",
   "score": 62.87,
   "score_ci": [
    61.01,
    64.86
   ],
   "rank": 2,
   "trained_on": [],
   "public_macro_accuracy": 0.7565,
   "public_pooled_ece": 0.0427,
   "typed_accuracy": 0.68,
   "typed_ece": 0.0198,
   "speed": {
    "rtx-5090": {
     "p50_ms": 393,
     "p90_ms": 393.9,
     "load_ms": 22062.3,
     "memory_mb": 11278,
     "precision": "F32"
    },
    "threadripper-3960x": {
     "p50_ms": 11875.9,
     "p90_ms": 11918.5,
     "load_ms": 20199,
     "memory_mb": 9524,
     "precision": "F32"
    }
   }
  },
  {
   "model": "decision:eos",
   "url": "https://bench.ollaya.dev/models/decision:eos",
   "badge": "https://bench.ollaya.dev/badge/decision/eos.svg",
   "publisher": "vLLM Semantic Router",
   "params": "0.75B",
   "runtime": "ONNX Runtime",
   "license": "Apache-2.0",
   "digest": "fc119d7067ecd029a5316442cf42a394236fff8758ab296f3d25a1285a34e047",
   "score": 38.08,
   "score_ci": [
    35.82,
    40.21
   ],
   "rank": 21,
   "trained_on": [],
   "public_macro_accuracy": 0.5905,
   "public_pooled_ece": 0.087,
   "typed_accuracy": 0.5125,
   "typed_ece": 0.1212,
   "speed": {
    "rtx-5090": {
     "p50_ms": 209.2,
     "p90_ms": 210.2,
     "load_ms": 13277,
     "memory_mb": 5910,
     "precision": "F32"
    },
    "threadripper-3960x": {
     "p50_ms": 3505.7,
     "p90_ms": 4296.9,
     "load_ms": 14264.6,
     "memory_mb": 6746,
     "precision": "F32"
    }
   }
  },
  {
   "model": "kev:0.8b",
   "url": "https://bench.ollaya.dev/models/kev:0.8b",
   "badge": "https://bench.ollaya.dev/badge/kev/0.8b.svg",
   "publisher": "Jared Palmer",
   "params": "0.76B",
   "runtime": "ONNX Runtime",
   "license": "Apache-2.0",
   "digest": "8a0d4094df504ca7329bc999dabbb89c15ad5221f55728acde0ff61c2b3909f2",
   "score": 40.75,
   "score_ci": [
    38.68,
    42.94
   ],
   "rank": 21,
   "trained_on": [],
   "public_macro_accuracy": 0.6106,
   "public_pooled_ece": 0.0345,
   "typed_accuracy": 0.4595,
   "typed_ece": 0.0459,
   "speed": {
    "rtx-5090": {
     "p50_ms": 141.8,
     "p90_ms": 142.7,
     "load_ms": 11888.6,
     "memory_mb": 4650,
     "precision": "F32"
    },
    "threadripper-3960x": {
     "p50_ms": 2907.7,
     "p90_ms": 2911.3,
     "load_ms": 12524.8,
     "memory_mb": 5086,
     "precision": "F32"
    }
   }
  },
  {
   "model": "kev:4b",
   "url": "https://bench.ollaya.dev/models/kev:4b",
   "badge": "https://bench.ollaya.dev/badge/kev/4b.svg",
   "publisher": "Jared Palmer",
   "params": "4.2B",
   "runtime": "ONNX Runtime",
   "license": "Apache-2.0",
   "digest": "c0e732655dc9d799dd21ba7f2f9a445fbd50ee83578e96ccfbf691c1be9dde53",
   "score": 61.21,
   "score_ci": [
    59.19,
    63.21
   ],
   "rank": 4,
   "trained_on": [],
   "public_macro_accuracy": 0.7448,
   "public_pooled_ece": 0.062,
   "typed_accuracy": 0.669,
   "typed_ece": 0.094,
   "speed": {
    "rtx-5090": {
     "p50_ms": 318.7,
     "p90_ms": 320.1,
     "load_ms": 21586,
     "memory_mb": 11200,
     "precision": "F32"
    },
    "threadripper-3960x": {
     "p50_ms": 8513.1,
     "p90_ms": 8519.9,
     "load_ms": 22526.9,
     "memory_mb": 9146,
     "precision": "F32"
    }
   }
  },
  {
   "model": "kev:9b",
   "url": "https://bench.ollaya.dev/models/kev:9b",
   "badge": "https://bench.ollaya.dev/badge/kev/9b.svg",
   "publisher": "Jared Palmer",
   "params": "7.9B",
   "runtime": "ONNX Runtime",
   "license": "Apache-2.0",
   "digest": "c9130c1a1e629c13497c833300395b56c7e347ecd3558596a2c2e1ce41ce7eda",
   "score": 62.56,
   "score_ci": [
    60.63,
    64.55
   ],
   "rank": 2,
   "trained_on": [],
   "public_macro_accuracy": 0.7526,
   "public_pooled_ece": 0.0583,
   "typed_accuracy": 0.7215,
   "typed_ece": 0.0945,
   "speed": {
    "rtx-5090": {
     "p50_ms": 374.8,
     "p90_ms": 375.3,
     "load_ms": 36583.3,
     "memory_mb": 18438,
     "precision": "F32"
    },
    "threadripper-3960x": {
     "p50_ms": 12497.5,
     "p90_ms": 12524.1,
     "load_ms": 39177.7,
     "memory_mb": 16262,
     "precision": "F32"
    }
   }
  },
  {
   "model": "arbiter:4b",
   "url": "https://bench.ollaya.dev/models/arbiter:4b",
   "badge": "https://bench.ollaya.dev/badge/arbiter/4b.svg",
   "publisher": "Codekins · Zyot Lab",
   "params": "4.3B",
   "runtime": "ONNX Runtime",
   "license": "Apache-2.0 (LoRA adapter and head) and the Gemma Terms of Use (Gemma 3 base model)",
   "digest": "2896b8aecfca38e37f22e75e78dca0a6955eba6cfd8dc5b24b637eb124b8cff8",
   "score": 15.21,
   "score_ci": [
    13.23,
    17.07
   ],
   "rank": 31,
   "trained_on": [],
   "public_macro_accuracy": 0.4198,
   "public_pooled_ece": 0.1372,
   "typed_accuracy": 0,
   "typed_ece": null,
   "speed": {
    "rtx-5090": {
     "p50_ms": 134.5,
     "p90_ms": 134.7,
     "load_ms": 6789.5,
     "memory_mb": 8449,
     "precision": "F32"
    },
    "threadripper-3960x": {
     "p50_ms": 6110.1,
     "p90_ms": 6130.7,
     "load_ms": 6571.7,
     "memory_mb": 7387,
     "precision": "F32"
    }
   }
  },
  {
   "model": "winnow:12b",
   "url": "https://bench.ollaya.dev/models/winnow:12b",
   "badge": "https://bench.ollaya.dev/badge/winnow/12b.svg",
   "publisher": "EldanRing",
   "params": "12B",
   "runtime": "llama.cpp",
   "license": "Apache-2.0",
   "digest": "8d72fee0e5f92a625cdb85ae4f6abdc024c9e6f124a0ca67e64522c3f24c47ce",
   "score": 65.67,
   "score_ci": [
    63.69,
    67.55
   ],
   "rank": 1,
   "trained_on": [],
   "public_macro_accuracy": 0.7726,
   "public_pooled_ece": 0.1415,
   "typed_accuracy": 0.702,
   "typed_ece": 0.1546,
   "speed": {
    "rtx-5090": {
     "p50_ms": 137.5,
     "p90_ms": 149.1,
     "load_ms": 27609.2,
     "memory_mb": 15872,
     "precision": "Q8_0"
    },
    "threadripper-3960x": {
     "p50_ms": 11242.9,
     "p90_ms": 11338.9,
     "load_ms": 32353.6,
     "memory_mb": 14965,
     "precision": "Q8_0"
    }
   }
  },
  {
   "model": "cygnet:12b",
   "url": "https://bench.ollaya.dev/models/cygnet:12b",
   "badge": "https://bench.ollaya.dev/badge/cygnet/12b.svg",
   "publisher": "blockbrain-ai",
   "params": "12B",
   "runtime": "llama.cpp",
   "license": "Apache-2.0",
   "digest": "81160d85e13acfda209f78ba38a3e1d7ef0c80727820b022bbd9c878476e93cc",
   "score": 64.56,
   "score_ci": [
    62.64,
    66.47
   ],
   "rank": 1,
   "trained_on": [],
   "public_macro_accuracy": 0.7676,
   "public_pooled_ece": 0.1034,
   "typed_accuracy": 0.684,
   "typed_ece": 0.1459,
   "speed": {
    "rtx-5090": {
     "p50_ms": 152.2,
     "p90_ms": 153.1,
     "load_ms": 28013.8,
     "memory_mb": 18560,
     "precision": "Q8_0"
    },
    "threadripper-3960x": {
     "p50_ms": 20664.2,
     "p90_ms": 20975.7,
     "load_ms": 41616.9,
     "memory_mb": 17648,
     "precision": "Q8_0"
    }
   }
  },
  {
   "model": "clm:8b",
   "url": "https://bench.ollaya.dev/models/clm:8b",
   "badge": "https://bench.ollaya.dev/badge/clm/8b.svg",
   "publisher": "Contrastive-LM",
   "params": "8.2B",
   "runtime": "ONNX Runtime",
   "license": "Apache-2.0",
   "digest": "3f89a66972f1398210d2f41b94667665132aa51c2d313237c3fc70a49c275cb5",
   "score": 9.28,
   "score_ci": [
    7.09,
    11.31
   ],
   "rank": 35,
   "trained_on": [],
   "public_macro_accuracy": 0.3767,
   "public_pooled_ece": 0.3725,
   "typed_accuracy": 0.357,
   "typed_ece": 0.4851,
   "speed": {
    "rtx-5090": {
     "p50_ms": 94.5,
     "p90_ms": 95.1,
     "load_ms": 31936.4,
     "memory_mb": 15678,
     "precision": "F32"
    },
    "threadripper-3960x": {
     "p50_ms": 5230.2,
     "p90_ms": 5548.5,
     "load_ms": 36123.7,
     "memory_mb": 14808,
     "precision": "F32"
    }
   }
  },
  {
   "model": "jeb:4b",
   "url": "https://bench.ollaya.dev/models/jeb:4b",
   "badge": "https://bench.ollaya.dev/badge/jeb/4b.svg",
   "publisher": "frontier-infra",
   "params": "4B",
   "runtime": "llama.cpp",
   "license": "Apache-2.0",
   "digest": "4aba27707f6f4ba6f116710a0d05743220e5a5819b8826a9bd32385d0d9df2ff",
   "score": 63.86,
   "score_ci": [
    61.56,
    65.9
   ],
   "rank": 1,
   "trained_on": [
    "typed-decisions"
   ],
   "public_macro_accuracy": 0.7589,
   "public_pooled_ece": 0.0481,
   "typed_accuracy": 0.7995,
   "typed_ece": 0.1363,
   "speed": {
    "rtx-5090": {
     "p50_ms": 90.7,
     "p90_ms": 92.4,
     "load_ms": 10487.9,
     "memory_mb": 5530,
     "precision": "Q8_0"
    },
    "threadripper-3960x": {
     "p50_ms": 5333.8,
     "p90_ms": 5435.4,
     "load_ms": 11129.7,
     "memory_mb": 4595,
     "precision": "Q8_0"
    }
   }
  },
  {
   "model": "jeb:9b",
   "url": "https://bench.ollaya.dev/models/jeb:9b",
   "badge": "https://bench.ollaya.dev/badge/jeb/9b.svg",
   "publisher": "frontier-infra",
   "params": "9B",
   "runtime": "llama.cpp",
   "license": "Apache-2.0",
   "digest": "dfe0411a8d060f1fa64ff2539b1e4b44a061c11cce4d5d96a82a7077b01d8b9e",
   "score": 64.9,
   "score_ci": [
    62.62,
    66.98
   ],
   "rank": 1,
   "trained_on": [
    "typed-decisions"
   ],
   "public_macro_accuracy": 0.7663,
   "public_pooled_ece": 0.0389,
   "typed_accuracy": 0.7865,
   "typed_ece": 0.1269,
   "speed": {
    "rtx-5090": {
     "p50_ms": 106.9,
     "p90_ms": 107.7,
     "load_ms": 20756.8,
     "memory_mb": 9316,
     "precision": "Q8_0"
    },
    "threadripper-3960x": {
     "p50_ms": 9417.9,
     "p90_ms": 9711,
     "load_ms": 21571.4,
     "memory_mb": 9434,
     "precision": "Q8_0"
    }
   }
  },
  {
   "model": "clef:flash",
   "url": "https://bench.ollaya.dev/models/clef:flash",
   "badge": "https://bench.ollaya.dev/badge/clef/flash.svg",
   "publisher": "Cloudflare",
   "params": "9B",
   "runtime": "ONNX Runtime",
   "license": "Apache-2.0",
   "digest": "acda6b87d74dc55e04916220fcfc643ca2d409ca22ec5e9d18bf53b91604845d",
   "score": 65.14,
   "score_ci": [
    63.24,
    67
   ],
   "rank": 1,
   "trained_on": [],
   "public_macro_accuracy": 0.7718,
   "public_pooled_ece": 0.0249,
   "typed_accuracy": 0.703,
   "typed_ece": 0.0139,
   "speed": {
    "rtx-5090": {
     "p50_ms": 406.4,
     "p90_ms": 407.2,
     "load_ms": 38224.8,
     "memory_mb": 20244,
     "precision": "F32"
    },
    "threadripper-3960x": {
     "p50_ms": 12477.2,
     "p90_ms": 12698.5,
     "load_ms": 35898.7,
     "memory_mb": 16138,
     "precision": "F32"
    }
   }
  },
  {
   "model": "jeeves:9b",
   "url": "https://bench.ollaya.dev/models/jeeves:9b",
   "badge": "https://bench.ollaya.dev/badge/jeeves/9b.svg",
   "publisher": "PostHog",
   "params": "9B",
   "runtime": "ONNX Runtime",
   "license": "Apache-2.0",
   "digest": "0f4a1ff8396c0718c432603d96c0a8eb975a0d3bab83bd374bd35e36dfc5512d",
   "score": 61.11,
   "score_ci": [
    59.09,
    62.96
   ],
   "rank": 4,
   "trained_on": [],
   "public_macro_accuracy": 0.7441,
   "public_pooled_ece": 0.0488,
   "typed_accuracy": 0.6805,
   "typed_ece": 0.0284,
   "speed": {
    "rtx-5090": {
     "p50_ms": 641.1,
     "p90_ms": 643,
     "load_ms": 35354.7,
     "memory_mb": 18528,
     "precision": "F32"
    },
    "threadripper-3960x": {
     "p50_ms": 20474.3,
     "p90_ms": 20558,
     "load_ms": 34872.7,
     "memory_mb": 16912,
     "precision": "F32"
    }
   }
  },
  {
   "model": "nimble:9b",
   "url": "https://bench.ollaya.dev/models/nimble:9b",
   "badge": "https://bench.ollaya.dev/badge/nimble/9b.svg",
   "publisher": "Bespoke Labs",
   "params": "9B",
   "runtime": "ONNX Runtime",
   "license": "Apache-2.0",
   "digest": "fb7e7f742e34636e853f91c467b9463e3155dbb08cabc5c49906d52a5750487d",
   "score": 61.86,
   "score_ci": [
    59.91,
    63.81
   ],
   "rank": 2,
   "trained_on": [],
   "public_macro_accuracy": 0.7476,
   "public_pooled_ece": 0.0225,
   "typed_accuracy": 0.665,
   "typed_ece": 0.0829,
   "speed": {
    "rtx-5090": {
     "p50_ms": 1688.1,
     "p90_ms": 1694.1,
     "load_ms": 39745.3,
     "memory_mb": 20782,
     "precision": "F32"
    },
    "threadripper-3960x": {
     "p50_ms": 62005.5,
     "p90_ms": 62148.5,
     "load_ms": 41091.2,
     "memory_mb": 19041,
     "precision": "F32"
    }
   }
  },
  {
   "model": "jeb:27b",
   "url": "https://bench.ollaya.dev/models/jeb:27b",
   "badge": "https://bench.ollaya.dev/badge/jeb/27b.svg",
   "publisher": "frontier-infra",
   "params": "27B",
   "runtime": "llama.cpp",
   "license": "Apache-2.0",
   "digest": "da59bd8b0f90123b7ca0de07f2a325aa860b8ab97814ba36226f11463186f561",
   "score": 67.85,
   "score_ci": [
    65.76,
    69.85
   ],
   "rank": 1,
   "trained_on": [
    "typed-decisions"
   ],
   "public_macro_accuracy": 0.7883,
   "public_pooled_ece": 0.0235,
   "typed_accuracy": 0.8025,
   "typed_ece": 0.1328,
   "speed": {
    "rtx-5090": {
     "p50_ms": 252.6,
     "p90_ms": 254.4,
     "load_ms": 36285.8,
     "memory_mb": 16592,
     "precision": "Q4_K - MEDIUM"
    },
    "threadripper-3960x": {
     "p50_ms": 20990.3,
     "p90_ms": 21377.5,
     "load_ms": 55205.6,
     "memory_mb": 27189,
     "precision": "Q4_K - MEDIUM"
    }
   }
  }
 ]
}