Total 713/713 errors found.
Error encountered 271 time(s): RuntimeError: CUDA error: CUBLAS_STATUS_EXECUTION_FAILED	{'m_mmlu_eu': ['FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base', 'facebook__xlm-roberta-xl'], 'm_mmlu_ml': ['facebook__xlm-roberta-xl', 'FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base'], 'm_mmlu_vi': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl', 'cis-lmu__glot500-base'], 'm_mmlu_ne': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl', 'cis-lmu__glot500-base'], 'm_mmlu_es': ['cis-lmu__glot500-base', 'FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl'], 'm_mmlu_sr': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl', 'cis-lmu__glot500-base'], 'm_mmlu_fr': ['FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base', 'facebook__xlm-roberta-xl'], 'm_mmlu_pt': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl', 'cis-lmu__glot500-base'], 'm_mmlu_ta': ['cis-lmu__glot500-base', 'facebook__xlm-roberta-xl', 'FacebookAI__xlm-roberta-large'], 'm_mmlu_sk': ['facebook__xlm-roberta-xl', 'FacebookAI__xlm-roberta-large'], 'm_mmlu_hy': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl', 'cis-lmu__glot500-base'], 'm_mmlu_it': ['cis-lmu__glot500-base', 'facebook__xlm-roberta-xl', 'FacebookAI__xlm-roberta-large'], 'm_mmlu_sv': ['cis-lmu__glot500-base', 'facebook__xlm-roberta-xl', 'FacebookAI__xlm-roberta-large'], 'm_mmlu_hr': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl', 'cis-lmu__glot500-base'], 'm_mmlu_hu': ['cis-lmu__glot500-base', 'FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl'], 'm_mmlu_nb': ['cis-lmu__glot500-base', 'FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl'], 'm_mmlu_uk': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl', 'cis-lmu__glot500-base'], 'm_mmlu_hi': ['cis-lmu__glot500-base', 'FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl'], 'm_mmlu_is': ['FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base', 'facebook__xlm-roberta-xl'], 'm_mmlu_ru': ['facebook__xlm-roberta-xl', 'cis-lmu__glot500-base', 'FacebookAI__xlm-roberta-large'], 'm_mmlu_te': ['FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base', 'facebook__xlm-roberta-xl'], 'm_mmlu_en': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl', 'cis-lmu__glot500-base'], 'm_mmlu_ca': ['FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base', 'facebook__xlm-roberta-xl'], 'm_mmlu_gu': ['facebook__xlm-roberta-xl', 'FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base'], 'm_mmlu_bn': ['facebook__xlm-roberta-xl', 'cis-lmu__glot500-base', 'FacebookAI__xlm-roberta-large'], 'm_mmlu_id': ['facebook__xlm-roberta-xl', 'FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base'], 'm_mmlu_kn': ['facebook__xlm-roberta-xl', 'cis-lmu__glot500-base', 'FacebookAI__xlm-roberta-large'], 'm_mmlu_da': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl', 'cis-lmu__glot500-base'], 'm_mmlu_mr': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl', 'cis-lmu__glot500-base'], 'm_mmlu_de': ['cis-lmu__glot500-base', 'facebook__xlm-roberta-xl', 'FacebookAI__xlm-roberta-large'], 'm_mmlu_zh': ['facebook__xlm-roberta-xl', 'FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base'], 'm_mmlu_ar': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl', 'cis-lmu__glot500-base'], 'm_mmlu_nl': ['FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base', 'facebook__xlm-roberta-xl'], 'm_mmlu_ro': ['cis-lmu__glot500-base', 'facebook__xlm-roberta-xl', 'FacebookAI__xlm-roberta-large'], 'belebele_fra_Latn': ['facebook__xlm-roberta-xl', 'FacebookAI__xlm-roberta-large'], 'belebele_shn_Mymr': ['facebook__xlm-roberta-xl', 'cis-lmu__glot500-base', 'FacebookAI__xlm-roberta-large'], 'belebele_pbt_Arab': ['facebook__xlm-roberta-xl', 'cis-lmu__glot500-base', 'FacebookAI__xlm-roberta-large'], 'belebele_tir_Ethi': ['cis-lmu__glot500-base', 'FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl'], 'belebele_arb_Latn': ['FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base', 'facebook__xlm-roberta-xl'], 'belebele_ceb_Latn': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl'], 'belebele_snd_Arab': ['facebook__xlm-roberta-xl', 'cis-lmu__glot500-base', 'FacebookAI__xlm-roberta-large'], 'belebele_sin_Latn': ['FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base', 'facebook__xlm-roberta-xl'], 'belebele_nya_Latn': ['facebook__xlm-roberta-xl', 'FacebookAI__xlm-roberta-large'], 'belebele_luo_Latn': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl'], 'belebele_npi_Latn': ['facebook__xlm-roberta-xl', 'FacebookAI__xlm-roberta-large'], 'belebele_som_Latn': ['FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base', 'facebook__xlm-roberta-xl'], 'belebele_grn_Latn': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl'], 'belebele_bam_Latn': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl'], 'belebele_kan_Knda': ['cis-lmu__glot500-base', 'facebook__xlm-roberta-xl', 'FacebookAI__xlm-roberta-large'], 'belebele_war_Latn': ['facebook__xlm-roberta-xl', 'cis-lmu__glot500-base', 'FacebookAI__xlm-roberta-large'], 'belebele_lao_Laoo': ['FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base', 'facebook__xlm-roberta-xl'], 'belebele_plt_Latn': ['cis-lmu__glot500-base', 'facebook__xlm-roberta-xl', 'FacebookAI__xlm-roberta-large'], 'belebele_guj_Gujr': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl'], 'belebele_yor_Latn': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl', 'cis-lmu__glot500-base'], 'belebele_hin_Latn': ['FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base', 'facebook__xlm-roberta-xl'], 'belebele_kac_Latn': ['cis-lmu__glot500-base', 'FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl'], 'belebele_tel_Telu': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl'], 'belebele_lin_Latn': ['facebook__xlm-roberta-xl', 'FacebookAI__xlm-roberta-large'], 'belebele_kin_Latn': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl', 'cis-lmu__glot500-base'], 'belebele_mri_Latn': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl', 'cis-lmu__glot500-base'], 'belebele_sot_Latn': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl'], 'belebele_kat_Geor': ['cis-lmu__glot500-base', 'FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl'], 'belebele_sin_Sinh': ['FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base', 'facebook__xlm-roberta-xl'], 'belebele_ell_Grek': ['facebook__xlm-roberta-xl', 'cis-lmu__glot500-base', 'FacebookAI__xlm-roberta-large'], 'belebele_ukr_Cyrl': ['facebook__xlm-roberta-xl', 'FacebookAI__xlm-roberta-large'], 'belebele_urd_Latn': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl', 'cis-lmu__glot500-base'], 'belebele_pan_Guru': ['facebook__xlm-roberta-xl', 'FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base'], 'belebele_khm_Khmr': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl', 'cis-lmu__glot500-base'], 'belebele_uzn_Latn': ['cis-lmu__glot500-base', 'FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl'], 'belebele_tgk_Cyrl': ['facebook__xlm-roberta-xl', 'FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base'], 'belebele_ben_Latn': ['facebook__xlm-roberta-xl', 'cis-lmu__glot500-base', 'FacebookAI__xlm-roberta-large'], 'belebele_tso_Latn': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl', 'cis-lmu__glot500-base'], 'belebele_asm_Beng': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl', 'cis-lmu__glot500-base'], 'belebele_ckb_Arab': ['facebook__xlm-roberta-xl', 'FacebookAI__xlm-roberta-large'], 'belebele_wol_Latn': ['facebook__xlm-roberta-xl', 'FacebookAI__xlm-roberta-large'], 'belebele_mlt_Latn': ['cis-lmu__glot500-base', 'facebook__xlm-roberta-xl', 'FacebookAI__xlm-roberta-large'], 'belebele_mar_Deva': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl'], 'belebele_mya_Mymr': ['cis-lmu__glot500-base', 'FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl'], 'belebele_tsn_Latn': ['FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base', 'facebook__xlm-roberta-xl'], 'belebele_hye_Armn': ['facebook__xlm-roberta-xl', 'FacebookAI__xlm-roberta-large'], 'belebele_lug_Latn': ['facebook__xlm-roberta-xl', 'FacebookAI__xlm-roberta-large'], 'belebele_tgl_Latn': ['facebook__xlm-roberta-xl', 'FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base'], 'belebele_amh_Ethi': ['facebook__xlm-roberta-xl', 'FacebookAI__xlm-roberta-large'], 'belebele_ssw_Latn': ['cis-lmu__glot500-base', 'facebook__xlm-roberta-xl', 'FacebookAI__xlm-roberta-large'], 'belebele_zul_Latn': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl'], 'belebele_fuv_Latn': ['facebook__xlm-roberta-xl', 'FacebookAI__xlm-roberta-large'], 'belebele_hau_Latn': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl'], 'belebele_tam_Taml': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl'], 'belebele_ilo_Latn': ['cis-lmu__glot500-base', 'facebook__xlm-roberta-xl', 'FacebookAI__xlm-roberta-large'], 'belebele_mal_Mlym': ['cis-lmu__glot500-base', 'facebook__xlm-roberta-xl', 'FacebookAI__xlm-roberta-large'], 'belebele_ibo_Latn': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl'], 'belebele_ben_Beng': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl'], 'belebele_als_Latn': ['cis-lmu__glot500-base', 'FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl'], 'belebele_kea_Latn': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl'], 'belebele_hat_Latn': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl'], 'belebele_xho_Latn': ['facebook__xlm-roberta-xl'], 'belebele_gaz_Latn': ['cis-lmu__glot500-base', 'facebook__xlm-roberta-xl', 'FacebookAI__xlm-roberta-large'], 'belebele_ory_Orya': ['FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base', 'facebook__xlm-roberta-xl'], 'belebele_nso_Latn': ['FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base', 'facebook__xlm-roberta-xl'], 'belebele_sna_Latn': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl'], 'belebele_bod_Tibt': ['cis-lmu__glot500-base']}
Error encountered 326 time(s): RuntimeError: CUDA error: device-side assert triggered	{'m_mmlu_vi': ['facebook__mbart-large-50'], 'arc_es': ['facebook__mbart-large-50'], 'hellaswag_fr': ['facebook__mbart-large-50'], 'arc_nl': ['facebook__mbart-large-50'], 'mgsm_native_cot_en': ['facebook__mbart-large-50'], 'xwinograd_fr': ['facebook__mbart-large-50'], 'arc_hu': ['facebook__mbart-large-50'], 'hellaswag_ta': ['facebook__mbart-large-50'], 'm_mmlu_sv': ['facebook__mbart-large-50'], 'lambada_openai_mt_stablelm_en': ['facebook__mbart-large-50'], 'lambada_openai_mt_stablelm_it': ['facebook__mbart-large-50'], 'hellaswag_ro': ['facebook__mbart-large-50'], 'm_mmlu_sk': ['facebook__mbart-large-50', 'cis-lmu__glot500-base'], 'lambada_openai_mt_stablelm_fr': ['facebook__mbart-large-50'], 'xstorycloze_te': ['facebook__mbart-large-50'], 'truthfulqa_fr_mc1': ['facebook__mbart-large-50'], 'mgsm_native_cot_de': ['facebook__mbart-large-50'], 'arc_hy': ['facebook__mbart-large-50'], 'mgsm_direct_th': ['facebook__mbart-large-50'], 'arc_mr': ['facebook__mbart-large-50'], 'arc_ta': ['facebook__mbart-large-50'], 'hellaswag_uk': ['facebook__mbart-large-50'], 'truthfulqa_hi_mc1': ['facebook__mbart-large-50'], 'hellaswag_bn': ['facebook__mbart-large-50'], 'arc_ne': ['facebook__mbart-large-50'], 'arc_gu': ['facebook__mbart-large-50'], 'arc_ca': ['facebook__mbart-large-50'], 'xwinograd_zh': ['facebook__mbart-large-50'], 'm_mmlu_fr': ['facebook__mbart-large-50'], 'truthfulqa_de_mc1': ['facebook__mbart-large-50'], 'xstorycloze_zh': ['facebook__mbart-large-50'], 'm_mmlu_ar': ['facebook__mbart-large-50'], 'truthfulqa_gu_mc1': ['facebook__mbart-large-50'], 'paws_de': ['facebook__mbart-large-50'], 'xcopa_et': ['facebook__mbart-large-50'], 'm_mmlu_te': ['facebook__mbart-large-50'], 'truthfulqa_uk_mc1': ['facebook__mbart-large-50'], 'mgsm_direct_sw': ['facebook__mbart-large-50'], 'm_mmlu_hy': ['facebook__mbart-large-50'], 'arc_id': ['facebook__mbart-large-50'], 'hellaswag_ru': ['facebook__mbart-large-50'], 'truthfulqa_ca_mc1': ['facebook__mbart-large-50'], 'mgsm_direct_zh': ['facebook__mbart-large-50'], 'm_mmlu_es': ['facebook__mbart-large-50'], 'hellaswag_mr': ['facebook__mbart-large-50'], 'xcopa_it': ['facebook__mbart-large-50'], 'm_mmlu_ne': ['facebook__mbart-large-50'], 'm_mmlu_ro': ['facebook__mbart-large-50'], 'hellaswag_hu': ['facebook__mbart-large-50'], 'hellaswag_ml': ['facebook__mbart-large-50'], 'xstorycloze_es': ['facebook__mbart-large-50'], 'mgsm_direct_de': ['facebook__mbart-large-50'], 'm_mmlu_kn': ['facebook__mbart-large-50'], 'm_mmlu_pt': ['facebook__mbart-large-50'], 'xnli_bg': ['facebook__mbart-large-50'], 'm_mmlu_da': ['facebook__mbart-large-50'], 'xstorycloze_hi': ['facebook__mbart-large-50'], 'truthfulqa_hy_mc1': ['facebook__mbart-large-50'], 'truthfulqa_nl_mc1': ['facebook__mbart-large-50'], 'hellaswag_ca': ['facebook__mbart-large-50'], 'truthfulqa_zh_mc1': ['facebook__mbart-large-50'], 'truthfulqa_mr_mc1': ['facebook__mbart-large-50'], 'hellaswag_es': ['facebook__mbart-large-50'], 'm_mmlu_ca': ['facebook__mbart-large-50'], 'xcopa_ta': ['facebook__mbart-large-50'], 'xwinograd_en': ['facebook__mbart-large-50'], 'truthfulqa_ro_mc1': ['facebook__mbart-large-50'], 'mgsm_direct_ru': ['facebook__mbart-large-50'], 'xcopa_qu': ['facebook__mbart-large-50'], 'm_mmlu_ta': ['facebook__mbart-large-50'], 'mgsm_native_cot_fr': ['facebook__mbart-large-50'], 'arc_ro': ['facebook__mbart-large-50'], 'arc_fr': ['facebook__mbart-large-50'], 'paws_es': ['facebook__mbart-large-50'], 'truthfulqa_pt_mc1': ['facebook__mbart-large-50'], 'arc_bn': ['facebook__mbart-large-50'], 'mgsm_native_cot_th': ['facebook__mbart-large-50'], 'xcopa_tr': ['facebook__mbart-large-50'], 'm_mmlu_nl': ['facebook__mbart-large-50'], 'arc_ar': ['facebook__mbart-large-50'], 'hellaswag_hr': ['facebook__mbart-large-50'], 'mgsm_native_cot_ja': ['facebook__mbart-large-50'], 'mgsm_native_cot_bn': ['facebook__mbart-large-50'], 'xnli_hi': ['facebook__mbart-large-50'], 'hellaswag_pt': ['facebook__mbart-large-50'], 'hellaswag_sv': ['facebook__mbart-large-50'], 'm_mmlu_id': ['facebook__mbart-large-50'], 'hellaswag_vi': ['facebook__mbart-large-50'], 'hellaswag_kn': ['facebook__mbart-large-50'], 'hellaswag_hy': ['facebook__mbart-large-50'], 'm_mmlu_bn': ['facebook__mbart-large-50'], 'mgsm_native_cot_te': ['facebook__mbart-large-50'], 'hellaswag_te': ['facebook__mbart-large-50'], 'truthfulqa_ta_mc1': ['facebook__mbart-large-50'], 'mgsm_direct_bn': ['facebook__mbart-large-50'], 'truthfulqa_it_mc1': ['facebook__mbart-large-50'], 'arc_te': ['facebook__mbart-large-50'], 'paws_ja': ['facebook__mbart-large-50'], 'lambada_openai_mt_stablelm_de': ['facebook__mbart-large-50'], 'paws_en': ['facebook__mbart-large-50'], 'arc_de': ['facebook__mbart-large-50'], 'xcopa_id': ['facebook__mbart-large-50'], 'xnli_ar': ['facebook__mbart-large-50'], 'arc_eu': ['facebook__mbart-large-50'], 'm_mmlu_eu': ['facebook__mbart-large-50'], 'm_mmlu_nb': ['facebook__mbart-large-50'], 'xcopa_ht': ['facebook__mbart-large-50'], 'truthfulqa_vi_mc1': ['facebook__mbart-large-50'], 'hellaswag_da': ['facebook__mbart-large-50'], 'truthfulqa_ar_mc1': ['facebook__mbart-large-50'], 'm_mmlu_gu': ['facebook__mbart-large-50'], 'mgsm_direct_fr': ['facebook__mbart-large-50'], 'm_mmlu_uk': ['facebook__mbart-large-50'], 'hellaswag_de': ['facebook__mbart-large-50'], 'arc_sv': ['facebook__mbart-large-50'], 'm_mmlu_sr': ['facebook__mbart-large-50'], 'xnli_tr': ['facebook__mbart-large-50'], 'xnli_ru': ['facebook__mbart-large-50'], 'xstorycloze_ar': ['facebook__mbart-large-50'], 'xnli_zh': ['facebook__mbart-large-50'], 'arc_zh': ['facebook__mbart-large-50'], 'xwinograd_ja': ['facebook__mbart-large-50'], 'mgsm_direct_en': ['facebook__mbart-large-50'], 'truthfulqa_sr_mc1': ['facebook__mbart-large-50'], 'xnli_de': ['facebook__mbart-large-50'], 'truthfulqa_es_mc1': ['facebook__mbart-large-50'], 'hellaswag_id': ['facebook__mbart-large-50'], 'arc_hr': ['facebook__mbart-large-50'], 'xnli_el': ['facebook__mbart-large-50'], 'mgsm_direct_ja': ['facebook__mbart-large-50'], 'truthfulqa_ne_mc1': ['facebook__mbart-large-50'], 'm_mmlu_hi': ['facebook__mbart-large-50'], 'mgsm_native_cot_ru': ['facebook__mbart-large-50'], 'xnli_th': ['facebook__mbart-large-50'], 'arc_uk': ['facebook__mbart-large-50'], 'truthfulqa_eu_mc1': ['facebook__mbart-large-50'], 'hellaswag_it': ['facebook__mbart-large-50'], 'hellaswag_eu': ['facebook__mbart-large-50'], 'hellaswag_nl': ['facebook__mbart-large-50'], 'xcopa_th': ['facebook__mbart-large-50'], 'truthfulqa_te_mc1': ['facebook__mbart-large-50'], 'xstorycloze_en': ['facebook__mbart-large-50'], 'arc_it': ['facebook__mbart-large-50'], 'truthfulqa_da_mc1': ['facebook__mbart-large-50'], 'xstorycloze_my': ['facebook__mbart-large-50'], 'arc_pt': ['facebook__mbart-large-50'], 'm_mmlu_it': ['facebook__mbart-large-50'], 'truthfulqa_hu_mc1': ['facebook__mbart-large-50'], 'xwinograd_pt': ['facebook__mbart-large-50'], 'm_mmlu_ru': ['facebook__mbart-large-50'], 'm_mmlu_en': ['facebook__mbart-large-50'], 'truthfulqa_hr_mc1': ['facebook__mbart-large-50'], 'hellaswag_ar': ['facebook__mbart-large-50'], 'hellaswag_gu': ['facebook__mbart-large-50'], 'xwinograd_ru': ['facebook__mbart-large-50'], 'arc_ml': ['facebook__mbart-large-50'], 'arc_vi': ['facebook__mbart-large-50'], 'arc_hi': ['facebook__mbart-large-50'], 'arc_ru': ['facebook__mbart-large-50'], 'xnli_sw': ['facebook__mbart-large-50'], 'mgsm_native_cot_zh': ['facebook__mbart-large-50'], 'arc_kn': ['facebook__mbart-large-50'], 'xnli_en': ['facebook__mbart-large-50'], 'xnli_vi': ['facebook__mbart-large-50'], 'xnli_fr': ['facebook__mbart-large-50'], 'arc_sk': ['facebook__mbart-large-50'], 'arc_da': ['facebook__mbart-large-50'], 'truthfulqa_id_mc1': ['facebook__mbart-large-50'], 'm_mmlu_de': ['facebook__mbart-large-50'], 'm_mmlu_hr': ['facebook__mbart-large-50'], 'truthfulqa_ru_mc1': ['facebook__mbart-large-50'], 'truthfulqa_kn_mc1': ['facebook__mbart-large-50'], 'xstorycloze_sw': ['facebook__mbart-large-50'], 'paws_fr': ['facebook__mbart-large-50'], 'hellaswag_sr': ['facebook__mbart-large-50'], 'truthfulqa_ml_mc1': ['facebook__mbart-large-50'], 'arc_sr': ['facebook__mbart-large-50'], 'paws_zh': ['facebook__mbart-large-50'], 'm_mmlu_hu': ['facebook__mbart-large-50'], 'truthfulqa_bn_mc1': ['facebook__mbart-large-50'], 'xcopa_zh': ['facebook__mbart-large-50'], 'mgsm_direct_te': ['facebook__mbart-large-50'], 'xstorycloze_id': ['facebook__mbart-large-50'], 'xcopa_vi': ['facebook__mbart-large-50'], 'truthfulqa_sv_mc1': ['facebook__mbart-large-50'], 'mgsm_native_cot_es': ['facebook__mbart-large-50'], 'm_mmlu_is': ['facebook__mbart-large-50'], 'xnli_es': ['facebook__mbart-large-50'], 'm_mmlu_mr': ['facebook__mbart-large-50'], 'hellaswag_hi': ['facebook__mbart-large-50'], 'xcopa_sw': ['facebook__mbart-large-50'], 'm_mmlu_zh': ['facebook__mbart-large-50'], 'mgsm_native_cot_sw': ['facebook__mbart-large-50'], 'xstorycloze_ru': ['facebook__mbart-large-50'], 'm_mmlu_ml': ['facebook__mbart-large-50'], 'lambada_openai_mt_stablelm_es': ['facebook__mbart-large-50'], 'truthfulqa_sk_mc1': ['facebook__mbart-large-50'], 'mgsm_direct_es': ['facebook__mbart-large-50'], 'hellaswag_sk': ['facebook__mbart-large-50'], 'belebele_kan_Knda': ['facebook__mbart-large-50'], 'belebele_pol_Latn': ['facebook__mbart-large-50'], 'paws_ko': ['facebook__mbart-large-50'], 'xnli_ur': ['facebook__mbart-large-50'], 'hellaswag_ne': ['facebook__mbart-large-50'], 'xstorycloze_eu': ['facebook__mbart-large-50'], 'belebele_mri_Latn': ['facebook__mbart-large-50'], 'belebele_tgl_Latn': ['facebook__mbart-large-50'], 'belebele_tam_Taml': ['facebook__mbart-large-50'], 'belebele_ars_Arab': ['facebook__mbart-large-50'], 'belebele_kea_Latn': ['facebook__mbart-large-50'], 'belebele_ibo_Latn': ['facebook__mbart-large-50'], 'belebele_npi_Deva': ['facebook__mbart-large-50'], 'belebele_nob_Latn': ['facebook__mbart-large-50'], 'belebele_kat_Geor': ['facebook__mbart-large-50'], 'belebele_srp_Cyrl': ['facebook__mbart-large-50'], 'belebele_nld_Latn': ['facebook__mbart-large-50'], 'belebele_som_Latn': ['facebook__mbart-large-50'], 'belebele_sin_Sinh': ['facebook__mbart-large-50'], 'belebele_pan_Guru': ['facebook__mbart-large-50'], 'belebele_fra_Latn': ['facebook__mbart-large-50'], 'belebele_hat_Latn': ['facebook__mbart-large-50'], 'belebele_sun_Latn': ['facebook__mbart-large-50'], 'belebele_lvs_Latn': ['facebook__mbart-large-50'], 'belebele_vie_Latn': ['facebook__mbart-large-50'], 'belebele_swh_Latn': ['facebook__mbart-large-50'], 'belebele_kaz_Cyrl': ['facebook__mbart-large-50'], 'belebele_urd_Arab': ['facebook__mbart-large-50'], 'belebele_bam_Latn': ['facebook__mbart-large-50'], 'belebele_mkd_Cyrl': ['facebook__mbart-large-50'], 'belebele_apc_Arab': ['facebook__mbart-large-50'], 'belebele_zho_Hant': ['facebook__mbart-large-50'], 'belebele_zho_Hans': ['facebook__mbart-large-50'], 'belebele_jpn_Jpan': ['facebook__mbart-large-50'], 'belebele_ben_Latn': ['facebook__mbart-large-50'], 'belebele_tsn_Latn': ['facebook__mbart-large-50'], 'belebele_uzn_Latn': ['facebook__mbart-large-50'], 'belebele_sin_Latn': ['facebook__mbart-large-50'], 'belebele_zsm_Latn': ['facebook__mbart-large-50'], 'belebele_ron_Latn': ['facebook__mbart-large-50'], 'belebele_kir_Cyrl': ['facebook__mbart-large-50'], 'belebele_war_Latn': ['facebook__mbart-large-50'], 'belebele_yor_Latn': ['facebook__mbart-large-50'], 'belebele_isl_Latn': ['facebook__mbart-large-50'], 'belebele_swe_Latn': ['facebook__mbart-large-50'], 'belebele_arb_Latn': ['facebook__mbart-large-50'], 'belebele_azj_Latn': ['facebook__mbart-large-50'], 'belebele_fuv_Latn': ['facebook__mbart-large-50'], 'belebele_lin_Latn': ['facebook__mbart-large-50'], 'belebele_deu_Latn': ['facebook__mbart-large-50'], 'belebele_lug_Latn': ['facebook__mbart-large-50'], 'belebele_arb_Arab': ['facebook__mbart-large-50'], 'belebele_slk_Latn': ['facebook__mbart-large-50'], 'belebele_acm_Arab': ['facebook__mbart-large-50'], 'belebele_ssw_Latn': ['facebook__mbart-large-50'], 'belebele_als_Latn': ['facebook__mbart-large-50'], 'belebele_hrv_Latn': ['facebook__mbart-large-50'], 'belebele_afr_Latn': ['facebook__mbart-large-50'], 'belebele_kor_Hang': ['facebook__mbart-large-50'], 'belebele_hin_Latn': ['facebook__mbart-large-50'], 'belebele_cat_Latn': ['facebook__mbart-large-50'], 'belebele_kin_Latn': ['facebook__mbart-large-50'], 'belebele_xho_Latn': ['FacebookAI__xlm-roberta-large', 'facebook__mbart-large-50'], 'belebele_jav_Latn': ['facebook__mbart-large-50'], 'belebele_zul_Latn': ['facebook__mbart-large-50'], 'belebele_rus_Cyrl': ['facebook__mbart-large-50'], 'belebele_ben_Beng': ['facebook__mbart-large-50'], 'belebele_khk_Cyrl': ['facebook__mbart-large-50'], 'belebele_luo_Latn': ['facebook__mbart-large-50'], 'belebele_ind_Latn': ['facebook__mbart-large-50'], 'belebele_tso_Latn': ['facebook__mbart-large-50'], 'belebele_tur_Latn': ['facebook__mbart-large-50'], 'belebele_ukr_Cyrl': ['facebook__mbart-large-50'], 'belebele_wol_Latn': ['facebook__mbart-large-50'], 'belebele_ces_Latn': ['facebook__mbart-large-50'], 'belebele_eus_Latn': ['facebook__mbart-large-50'], 'belebele_tgk_Cyrl': ['facebook__mbart-large-50'], 'belebele_sna_Latn': ['facebook__mbart-large-50'], 'belebele_plt_Latn': ['facebook__mbart-large-50'], 'belebele_guj_Gujr': ['facebook__mbart-large-50'], 'belebele_ell_Grek': ['facebook__mbart-large-50'], 'belebele_ceb_Latn': ['facebook__mbart-large-50'], 'belebele_ckb_Arab': ['facebook__mbart-large-50'], 'belebele_bul_Cyrl': ['facebook__mbart-large-50'], 'belebele_tha_Thai': ['facebook__mbart-large-50'], 'belebele_hin_Deva': ['facebook__mbart-large-50'], 'belebele_lao_Laoo': ['facebook__mbart-large-50'], 'belebele_ilo_Latn': ['facebook__mbart-large-50'], 'belebele_gaz_Latn': ['facebook__mbart-large-50'], 'belebele_lit_Latn': ['facebook__mbart-large-50'], 'belebele_grn_Latn': ['facebook__mbart-large-50'], 'belebele_mal_Mlym': ['facebook__mbart-large-50'], 'belebele_bod_Tibt': ['facebook__mbart-large-50'], 'belebele_hye_Armn': ['facebook__mbart-large-50'], 'belebele_dan_Latn': ['facebook__mbart-large-50'], 'belebele_ory_Orya': ['facebook__mbart-large-50'], 'belebele_eng_Latn': ['facebook__mbart-large-50'], 'belebele_amh_Ethi': ['facebook__mbart-large-50'], 'belebele_npi_Latn': ['facebook__mbart-large-50'], 'belebele_hau_Latn': ['facebook__mbart-large-50'], 'belebele_fin_Latn': ['facebook__mbart-large-50'], 'belebele_hun_Latn': ['facebook__mbart-large-50'], 'belebele_mya_Mymr': ['facebook__mbart-large-50'], 'belebele_tel_Telu': ['facebook__mbart-large-50'], 'belebele_por_Latn': ['facebook__mbart-large-50'], 'belebele_slv_Latn': ['facebook__mbart-large-50'], 'belebele_heb_Hebr': ['facebook__mbart-large-50'], 'belebele_khm_Khmr': ['facebook__mbart-large-50'], 'belebele_snd_Arab': ['facebook__mbart-large-50'], 'belebele_pes_Arab': ['facebook__mbart-large-50'], 'belebele_kac_Latn': ['facebook__mbart-large-50'], 'belebele_urd_Latn': ['facebook__mbart-large-50'], 'belebele_arz_Arab': ['facebook__mbart-large-50'], 'belebele_sot_Latn': ['facebook__mbart-large-50'], 'belebele_asm_Beng': ['facebook__mbart-large-50'], 'belebele_nso_Latn': ['facebook__mbart-large-50'], 'belebele_ita_Latn': ['facebook__mbart-large-50'], 'belebele_est_Latn': ['facebook__mbart-large-50'], 'belebele_mlt_Latn': ['facebook__mbart-large-50'], 'belebele_tir_Ethi': ['facebook__mbart-large-50'], 'belebele_mar_Deva': ['facebook__mbart-large-50'], 'belebele_pbt_Arab': ['facebook__mbart-large-50'], 'belebele_spa_Latn': ['facebook__mbart-large-50'], 'belebele_shn_Mymr': ['facebook__mbart-large-50'], 'belebele_nya_Latn': ['facebook__mbart-large-50']}
Error encountered 80 time(s): AssertionError	{'hellaswag_te': ['cis-lmu__glot500-base', 'google__mt5-xl', 'bigscience__bloom-7b1', 'bigscience__bloom-1b1', 'facebook__m2m100_1.2B', 'FacebookAI__xlm-roberta-large', 'google__mt5-large', 'bigscience__bloom-1b7'], 'hellaswag_ta': ['bigscience__bloom-1b7', 'facebook__m2m100_1.2B', 'google__mt5-xl', 'bigscience__bloom-7b1', 'bigscience__bloom-1b1', 'google__mt5-large', 'cis-lmu__glot500-base', 'FacebookAI__xlm-roberta-large'], 'hellaswag_gu': ['facebook__m2m100_1.2B', 'bigscience__bloom-1b1', 'bigscience__bloom-1b7', 'FacebookAI__xlm-roberta-large', 'bigscience__bloom-7b1', 'google__mt5-xl', 'google__mt5-large', 'cis-lmu__glot500-base'], 'hellaswag_bn': ['bigscience__bloom-7b1', 'FacebookAI__xlm-roberta-large', 'bigscience__bloom-1b7', 'google__mt5-large', 'facebook__m2m100_1.2B', 'cis-lmu__glot500-base', 'bigscience__bloom-1b1'], 'hellaswag_nl': ['bigscience__bloom-7b1', 'facebook__m2m100_1.2B', 'FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base', 'bigscience__bloom-1b1', 'google__mt5-large', 'bigscience__bloom-1b7'], 'hellaswag_ml': ['bigscience__bloom-1b7', 'facebook__m2m100_1.2B', 'google__mt5-large', 'FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base', 'bigscience__bloom-1b1', 'bigscience__bloom-7b1'], 'hellaswag_eu': ['cis-lmu__glot500-base', 'google__mt5-xl', 'facebook__m2m100_1.2B', 'bigscience__bloom-1b7', 'FacebookAI__xlm-roberta-large', 'bigscience__bloom-7b1', 'bigscience__bloom-1b1', 'google__mt5-large'], 'hellaswag_kn': ['cis-lmu__glot500-base', 'google__mt5-xl', 'bigscience__bloom-1b1', 'facebook__m2m100_1.2B', 'FacebookAI__xlm-roberta-large', 'bigscience__bloom-1b7', 'google__mt5-large', 'bigscience__bloom-7b1'], 'hellaswag_mr': ['facebook__m2m100_1.2B', 'bigscience__bloom-1b7', 'bigscience__bloom-1b1', 'google__mt5-large', 'FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base', 'bigscience__bloom-7b1'], 'lambada_openai_mt_stablelm_fr': ['FacebookAI__xlm-roberta-large', 'google__mt5-xl', 'facebook__m2m100_1.2B', 'cis-lmu__glot500-base', 'google__mt5-large'], 'hellaswag_ne': ['FacebookAI__xlm-roberta-large', 'facebook__m2m100_1.2B', 'bigscience__bloom-1b1', 'bigscience__bloom-7b1', 'google__mt5-large', 'bigscience__bloom-1b7', 'cis-lmu__glot500-base']}
Error encountered 17 time(s): torch.OutOfMemoryError: CUDA out of memory.	{'hellaswag_sr': ['google__mt5-xl'], 'hellaswag_pt': ['google__mt5-xl'], 'hellaswag_es': ['google__mt5-xl'], 'hellaswag_ro': ['google__mt5-xl'], 'hellaswag_ne': ['google__mt5-xl'], 'xstorycloze_te': ['SeaLLMs__SeaLLM-7B-v2'], 'truthfulqa_it_mc1': ['dice-research__lola_v1'], 'hellaswag_nl': ['google__mt5-xl'], 'truthfulqa_pt_mc1': ['dice-research__lola_v1'], 'hellaswag_sv': ['google__mt5-xl'], 'hellaswag_ar': ['google__mt5-xl'], 'hellaswag_ml': ['google__mt5-xl'], 'hellaswag_bn': ['google__mt5-xl'], 'hellaswag_mr': ['google__mt5-xl'], 'hellaswag_de': ['google__mt5-xl'], 'hellaswag_ru': ['google__mt5-xl'], 'truthfulqa_zh_mc1': ['dice-research__lola_v1']}
Error encountered 19 time(s): ValueError: Tasks not found: belebele_ary_arab. Try `lm-eval --tasks {list_groups,list_subtasks,list_tags,list}` to list out all available names for task groupings; only (sub)tasks; tags; or all of the above, or pass '--verbosity DEBUG' to troubleshoot task registration issues.	{'belebele_ary_arab': ['SeaLLMs__SeaLLMs-v3-1.5B-Chat', 'FacebookAI__xlm-roberta-large', 'HuggingFaceH4__zephyr-7b-beta', 'cis-lmu__glot500-base', 'ai-forever__mGPT', 'bigscience__bloom-1b1', 'tiiuae__falcon-7b', 'Unbabel__TowerBase-7B-v0.1', 'facebook__xlm-roberta-xl', 'SeaLLMs__SeaLLM-7B-v2.5', 'SeaLLMs__SeaLLM-7B-v2', 'dice-research__lola_v1', 'google__mt5-xl', 'google__mt5-large', 'facebook__mbart-large-50', 'MediaTek-Research__Breeze-7B-Base-v1_0', 'facebook__m2m100_1.2B', 'bigscience__bloom-7b1', 'bigscience__bloom-1b7']}