| { | |
| "dataset_revision": "7d571f92784cd94a019292a1f45445077d0ef634", | |
| "mteb_dataset_name": "MassiveScenarioClassification", | |
| "mteb_version": "1.1.1", | |
| "test": { | |
| "af": { | |
| "accuracy": 0.36028917283120376, | |
| "accuracy_stderr": 0.020886490567978682, | |
| "f1": 0.3137050291209066, | |
| "f1_stderr": 0.022537688020998856, | |
| "main_score": 0.36028917283120376 | |
| }, | |
| "am": { | |
| "accuracy": 0.1933086751849361, | |
| "accuracy_stderr": 0.02283239223864356, | |
| "f1": 0.1590055477686407, | |
| "f1_stderr": 0.013935127381527872, | |
| "main_score": 0.1933086751849361 | |
| }, | |
| "ar": { | |
| "accuracy": 0.23880295897780765, | |
| "accuracy_stderr": 0.021550709196053692, | |
| "f1": 0.2169558689754628, | |
| "f1_stderr": 0.0204399607001084, | |
| "main_score": 0.23880295897780765 | |
| }, | |
| "az": { | |
| "accuracy": 0.28853396099529255, | |
| "accuracy_stderr": 0.03707651080095728, | |
| "f1": 0.26709933198206504, | |
| "f1_stderr": 0.03130524907329002, | |
| "main_score": 0.28853396099529255 | |
| }, | |
| "bn": { | |
| "accuracy": 0.18382649630127773, | |
| "accuracy_stderr": 0.01723677514691048, | |
| "f1": 0.16772353498005438, | |
| "f1_stderr": 0.01481328757620055, | |
| "main_score": 0.18382649630127773 | |
| }, | |
| "cy": { | |
| "accuracy": 0.3540685944855414, | |
| "accuracy_stderr": 0.013893980159096653, | |
| "f1": 0.31488014251203256, | |
| "f1_stderr": 0.009843904769997955, | |
| "main_score": 0.3540685944855414 | |
| }, | |
| "da": { | |
| "accuracy": 0.34475453934095496, | |
| "accuracy_stderr": 0.01657331722800284, | |
| "f1": 0.31226537880607297, | |
| "f1_stderr": 0.013690885206849407, | |
| "main_score": 0.34475453934095496 | |
| }, | |
| "de": { | |
| "accuracy": 0.3, | |
| "accuracy_stderr": 0.026370378633190784, | |
| "f1": 0.2808153376359271, | |
| "f1_stderr": 0.02199698204026743, | |
| "main_score": 0.3 | |
| }, | |
| "el": { | |
| "accuracy": 0.2687962340282448, | |
| "accuracy_stderr": 0.020183365160538702, | |
| "f1": 0.24773267596830473, | |
| "f1_stderr": 0.016260145830922133, | |
| "main_score": 0.2687962340282448 | |
| }, | |
| "en": { | |
| "accuracy": 0.4377605917955615, | |
| "accuracy_stderr": 0.030195866016467707, | |
| "f1": 0.4160519309254586, | |
| "f1_stderr": 0.0285031791741231, | |
| "main_score": 0.4377605917955615 | |
| }, | |
| "es": { | |
| "accuracy": 0.3280766644250169, | |
| "accuracy_stderr": 0.021019663333640054, | |
| "f1": 0.3025035008255447, | |
| "f1_stderr": 0.018385344062269507, | |
| "main_score": 0.3280766644250169 | |
| }, | |
| "evaluation_time": 81.05, | |
| "fa": { | |
| "accuracy": 0.24515803631472766, | |
| "accuracy_stderr": 0.027835300630002634, | |
| "f1": 0.21726717236406928, | |
| "f1_stderr": 0.021536607731664367, | |
| "main_score": 0.24515803631472766 | |
| }, | |
| "fi": { | |
| "accuracy": 0.3331540013449899, | |
| "accuracy_stderr": 0.018238628577628934, | |
| "f1": 0.29353644700383935, | |
| "f1_stderr": 0.01170716493423995, | |
| "main_score": 0.3331540013449899 | |
| }, | |
| "fr": { | |
| "accuracy": 0.3242434431741762, | |
| "accuracy_stderr": 0.03480154901247634, | |
| "f1": 0.2987668403274445, | |
| "f1_stderr": 0.029511724262027892, | |
| "main_score": 0.3242434431741762 | |
| }, | |
| "he": { | |
| "accuracy": 0.24401479488903838, | |
| "accuracy_stderr": 0.025245916568530903, | |
| "f1": 0.21429656257015556, | |
| "f1_stderr": 0.018644802102533106, | |
| "main_score": 0.24401479488903838 | |
| }, | |
| "hi": { | |
| "accuracy": 0.2143577673167451, | |
| "accuracy_stderr": 0.02410369120931473, | |
| "f1": 0.18344740043506685, | |
| "f1_stderr": 0.015750165983010513, | |
| "main_score": 0.2143577673167451 | |
| }, | |
| "hu": { | |
| "accuracy": 0.3153328850033625, | |
| "accuracy_stderr": 0.020250251381445077, | |
| "f1": 0.29185785763469296, | |
| "f1_stderr": 0.018877820656201503, | |
| "main_score": 0.3153328850033625 | |
| }, | |
| "hy": { | |
| "accuracy": 0.19983187626092802, | |
| "accuracy_stderr": 0.023493594694878057, | |
| "f1": 0.17088318438817676, | |
| "f1_stderr": 0.015627620315522263, | |
| "main_score": 0.19983187626092802 | |
| }, | |
| "id": { | |
| "accuracy": 0.32733691997310016, | |
| "accuracy_stderr": 0.02881122933709865, | |
| "f1": 0.3137876294000571, | |
| "f1_stderr": 0.023998064464835975, | |
| "main_score": 0.32733691997310016 | |
| }, | |
| "is": { | |
| "accuracy": 0.31503026227303293, | |
| "accuracy_stderr": 0.02055429430786041, | |
| "f1": 0.28944584739759727, | |
| "f1_stderr": 0.01067996912064206, | |
| "main_score": 0.31503026227303293 | |
| }, | |
| "it": { | |
| "accuracy": 0.31116341627437794, | |
| "accuracy_stderr": 0.024149124398237676, | |
| "f1": 0.29410419968584234, | |
| "f1_stderr": 0.02122587264236695, | |
| "main_score": 0.31116341627437794 | |
| }, | |
| "ja": { | |
| "accuracy": 0.31136516476126425, | |
| "accuracy_stderr": 0.021111950678360036, | |
| "f1": 0.296719090248999, | |
| "f1_stderr": 0.01710265227320577, | |
| "main_score": 0.31136516476126425 | |
| }, | |
| "jv": { | |
| "accuracy": 0.343813046402152, | |
| "accuracy_stderr": 0.029728251673394785, | |
| "f1": 0.3148137403543144, | |
| "f1_stderr": 0.024921519074675536, | |
| "main_score": 0.343813046402152 | |
| }, | |
| "ka": { | |
| "accuracy": 0.183591123066577, | |
| "accuracy_stderr": 0.016855358634194025, | |
| "f1": 0.16158113168004945, | |
| "f1_stderr": 0.016067468774995444, | |
| "main_score": 0.183591123066577 | |
| }, | |
| "km": { | |
| "accuracy": 0.19250168123739073, | |
| "accuracy_stderr": 0.01647286656468353, | |
| "f1": 0.16728590019294526, | |
| "f1_stderr": 0.016752508710513803, | |
| "main_score": 0.19250168123739073 | |
| }, | |
| "kn": { | |
| "accuracy": 0.1604236718224613, | |
| "accuracy_stderr": 0.024421592069461574, | |
| "f1": 0.14304518042661318, | |
| "f1_stderr": 0.018877079513601795, | |
| "main_score": 0.1604236718224613 | |
| }, | |
| "ko": { | |
| "accuracy": 0.28954270342972427, | |
| "accuracy_stderr": 0.019105515429040258, | |
| "f1": 0.2614194694048272, | |
| "f1_stderr": 0.016750617590205023, | |
| "main_score": 0.28954270342972427 | |
| }, | |
| "lv": { | |
| "accuracy": 0.3228648285137861, | |
| "accuracy_stderr": 0.019874359393773237, | |
| "f1": 0.2942600336423284, | |
| "f1_stderr": 0.01601582290721286, | |
| "main_score": 0.3228648285137861 | |
| }, | |
| "ml": { | |
| "accuracy": 0.18762609280430395, | |
| "accuracy_stderr": 0.020738593066723977, | |
| "f1": 0.15764539982254006, | |
| "f1_stderr": 0.014771977910111181, | |
| "main_score": 0.18762609280430395 | |
| }, | |
| "mn": { | |
| "accuracy": 0.21570275722932078, | |
| "accuracy_stderr": 0.027219124319134883, | |
| "f1": 0.1898517402487539, | |
| "f1_stderr": 0.02640138202202358, | |
| "main_score": 0.21570275722932078 | |
| }, | |
| "ms": { | |
| "accuracy": 0.3595158036314728, | |
| "accuracy_stderr": 0.026062021234739854, | |
| "f1": 0.33458533277257285, | |
| "f1_stderr": 0.021026748542514302, | |
| "main_score": 0.3595158036314728 | |
| }, | |
| "my": { | |
| "accuracy": 0.2003026227303295, | |
| "accuracy_stderr": 0.01798219950339644, | |
| "f1": 0.16864192363299615, | |
| "f1_stderr": 0.01594740452626996, | |
| "main_score": 0.2003026227303295 | |
| }, | |
| "nb": { | |
| "accuracy": 0.3289172831203766, | |
| "accuracy_stderr": 0.021176010855405224, | |
| "f1": 0.2990554132334421, | |
| "f1_stderr": 0.01836052491494201, | |
| "main_score": 0.3289172831203766 | |
| }, | |
| "nl": { | |
| "accuracy": 0.3236381977135171, | |
| "accuracy_stderr": 0.028009334197584248, | |
| "f1": 0.2957501043505274, | |
| "f1_stderr": 0.012846209069565424, | |
| "main_score": 0.3236381977135171 | |
| }, | |
| "pl": { | |
| "accuracy": 0.31856086079354407, | |
| "accuracy_stderr": 0.02512612514658827, | |
| "f1": 0.3024275881944355, | |
| "f1_stderr": 0.023934697043335035, | |
| "main_score": 0.31856086079354407 | |
| }, | |
| "pt": { | |
| "accuracy": 0.31694687289845325, | |
| "accuracy_stderr": 0.024157082217185412, | |
| "f1": 0.29495870419371684, | |
| "f1_stderr": 0.01707790080358615, | |
| "main_score": 0.31694687289845325 | |
| }, | |
| "ro": { | |
| "accuracy": 0.2701412239408204, | |
| "accuracy_stderr": 0.01663946049973496, | |
| "f1": 0.2644782204477556, | |
| "f1_stderr": 0.01698244767624393, | |
| "main_score": 0.2701412239408204 | |
| }, | |
| "ru": { | |
| "accuracy": 0.3055480833893746, | |
| "accuracy_stderr": 0.031261270003082424, | |
| "f1": 0.2793696283146029, | |
| "f1_stderr": 0.026155944419436417, | |
| "main_score": 0.3055480833893746 | |
| }, | |
| "sl": { | |
| "accuracy": 0.35813718897108276, | |
| "accuracy_stderr": 0.03372159891683901, | |
| "f1": 0.33021495683147284, | |
| "f1_stderr": 0.022847569620226906, | |
| "main_score": 0.35813718897108276 | |
| }, | |
| "sq": { | |
| "accuracy": 0.22992602555480834, | |
| "accuracy_stderr": 0.02556091315605778, | |
| "f1": 0.21524928515996447, | |
| "f1_stderr": 0.022419827659728348, | |
| "main_score": 0.22992602555480834 | |
| }, | |
| "sv": { | |
| "accuracy": 0.347074646940148, | |
| "accuracy_stderr": 0.022102894910012768, | |
| "f1": 0.3154759971873104, | |
| "f1_stderr": 0.015570917225314567, | |
| "main_score": 0.347074646940148 | |
| }, | |
| "sw": { | |
| "accuracy": 0.3624075319435104, | |
| "accuracy_stderr": 0.01976082688360606, | |
| "f1": 0.33343978818160824, | |
| "f1_stderr": 0.01885062497033148, | |
| "main_score": 0.3624075319435104 | |
| }, | |
| "ta": { | |
| "accuracy": 0.19741089441829185, | |
| "accuracy_stderr": 0.022245483621730178, | |
| "f1": 0.16129268723975768, | |
| "f1_stderr": 0.01796286205867304, | |
| "main_score": 0.19741089441829185 | |
| }, | |
| "te": { | |
| "accuracy": 0.1754203093476799, | |
| "accuracy_stderr": 0.024895964012172724, | |
| "f1": 0.1553738138389406, | |
| "f1_stderr": 0.017103954736130998, | |
| "main_score": 0.1754203093476799 | |
| }, | |
| "th": { | |
| "accuracy": 0.22817753866846, | |
| "accuracy_stderr": 0.02705871016607919, | |
| "f1": 0.20722454859904277, | |
| "f1_stderr": 0.017308265065482564, | |
| "main_score": 0.22817753866846 | |
| }, | |
| "tl": { | |
| "accuracy": 0.2858439811701412, | |
| "accuracy_stderr": 0.013371027993939882, | |
| "f1": 0.2688190194852028, | |
| "f1_stderr": 0.010297065422356548, | |
| "main_score": 0.2858439811701412 | |
| }, | |
| "tr": { | |
| "accuracy": 0.2860457296570275, | |
| "accuracy_stderr": 0.018553458217986867, | |
| "f1": 0.2698989368733863, | |
| "f1_stderr": 0.01840904720687564, | |
| "main_score": 0.2860457296570275 | |
| }, | |
| "ur": { | |
| "accuracy": 0.23345662407531945, | |
| "accuracy_stderr": 0.033218477023455165, | |
| "f1": 0.1975032390408514, | |
| "f1_stderr": 0.02110974460048037, | |
| "main_score": 0.23345662407531945 | |
| }, | |
| "vi": { | |
| "accuracy": 0.2971082716879624, | |
| "accuracy_stderr": 0.010212406641330548, | |
| "f1": 0.2667592046024078, | |
| "f1_stderr": 0.008746481574257298, | |
| "main_score": 0.2971082716879624 | |
| }, | |
| "zh-CN": { | |
| "accuracy": 0.4409549428379288, | |
| "accuracy_stderr": 0.028089527474795855, | |
| "f1": 0.41275350430825675, | |
| "f1_stderr": 0.021552994416472702, | |
| "main_score": 0.4409549428379288 | |
| }, | |
| "zh-TW": { | |
| "accuracy": 0.3724277067921991, | |
| "accuracy_stderr": 0.02357437759675858, | |
| "f1": 0.3565629114113254, | |
| "f1_stderr": 0.018330511829508813, | |
| "main_score": 0.3724277067921991 | |
| } | |
| } | |
| } |