Total 402/402 errors found.
Error encountered 101 time(s): RuntimeError: CUDA error: CUBLAS_STATUS_EXECUTION_FAILED	{'m_mmlu_fr': ['FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base', 'facebook__xlm-roberta-xl'], 'm_mmlu_sk': ['facebook__xlm-roberta-xl', 'FacebookAI__xlm-roberta-large'], 'm_mmlu_vi': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl', 'cis-lmu__glot500-base'], 'm_mmlu_ne': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl', 'cis-lmu__glot500-base'], 'm_mmlu_it': ['cis-lmu__glot500-base', 'facebook__xlm-roberta-xl', 'FacebookAI__xlm-roberta-large'], 'm_mmlu_es': ['cis-lmu__glot500-base', 'FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl'], 'm_mmlu_sr': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl', 'cis-lmu__glot500-base'], 'm_mmlu_ta': ['cis-lmu__glot500-base', 'facebook__xlm-roberta-xl', 'FacebookAI__xlm-roberta-large'], 'm_mmlu_ml': ['facebook__xlm-roberta-xl', 'FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base'], 'm_mmlu_eu': ['FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base', 'facebook__xlm-roberta-xl'], 'm_mmlu_pt': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl', 'cis-lmu__glot500-base'], 'm_mmlu_hy': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl', 'cis-lmu__glot500-base'], 'm_mmlu_hu': ['cis-lmu__glot500-base', 'FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl'], 'm_mmlu_sv': ['cis-lmu__glot500-base', 'facebook__xlm-roberta-xl', 'FacebookAI__xlm-roberta-large'], 'm_mmlu_hr': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl', 'cis-lmu__glot500-base'], 'm_mmlu_uk': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl', 'cis-lmu__glot500-base'], 'm_mmlu_is': ['FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base', 'facebook__xlm-roberta-xl'], 'm_mmlu_hi': ['cis-lmu__glot500-base', 'FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl'], 'm_mmlu_nb': ['cis-lmu__glot500-base', 'FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl'], 'm_mmlu_ru': ['facebook__xlm-roberta-xl', 'cis-lmu__glot500-base', 'FacebookAI__xlm-roberta-large'], 'm_mmlu_te': ['FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base', 'facebook__xlm-roberta-xl'], 'm_mmlu_en': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl', 'cis-lmu__glot500-base'], 'm_mmlu_ca': ['FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base', 'facebook__xlm-roberta-xl'], 'm_mmlu_bn': ['facebook__xlm-roberta-xl', 'cis-lmu__glot500-base', 'FacebookAI__xlm-roberta-large'], 'm_mmlu_gu': ['facebook__xlm-roberta-xl', 'FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base'], 'm_mmlu_kn': ['facebook__xlm-roberta-xl', 'cis-lmu__glot500-base', 'FacebookAI__xlm-roberta-large'], 'm_mmlu_id': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl', 'cis-lmu__glot500-base'], 'm_mmlu_da': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl', 'cis-lmu__glot500-base'], 'm_mmlu_de': ['cis-lmu__glot500-base', 'facebook__xlm-roberta-xl', 'FacebookAI__xlm-roberta-large'], 'm_mmlu_mr': ['FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base', 'facebook__xlm-roberta-xl'], 'm_mmlu_ar': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl', 'cis-lmu__glot500-base'], 'm_mmlu_zh': ['facebook__xlm-roberta-xl', 'FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base'], 'm_mmlu_nl': ['FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base', 'facebook__xlm-roberta-xl'], 'm_mmlu_ro': ['cis-lmu__glot500-base', 'FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl']}
Error encountered 80 time(s): AssertionError	{'hellaswag_bn': ['FacebookAI__xlm-roberta-large', 'bigscience__bloom-7b1', 'bigscience__bloom-1b7', 'google__mt5-large', 'facebook__m2m100_1.2B', 'cis-lmu__glot500-base', 'bigscience__bloom-1b1'], 'hellaswag_gu': ['facebook__m2m100_1.2B', 'bigscience__bloom-1b1', 'bigscience__bloom-1b7', 'google__mt5-xl', 'bigscience__bloom-7b1', 'FacebookAI__xlm-roberta-large', 'google__mt5-large', 'cis-lmu__glot500-base'], 'hellaswag_te': ['google__mt5-xl', 'cis-lmu__glot500-base', 'bigscience__bloom-7b1', 'bigscience__bloom-1b1', 'facebook__m2m100_1.2B', 'FacebookAI__xlm-roberta-large', 'google__mt5-large', 'bigscience__bloom-1b7'], 'hellaswag_eu': ['cis-lmu__glot500-base', 'google__mt5-xl', 'facebook__m2m100_1.2B', 'bigscience__bloom-1b7', 'FacebookAI__xlm-roberta-large', 'bigscience__bloom-7b1', 'bigscience__bloom-1b1', 'google__mt5-large'], 'hellaswag_ta': ['bigscience__bloom-1b7', 'google__mt5-xl', 'facebook__m2m100_1.2B', 'bigscience__bloom-7b1', 'bigscience__bloom-1b1', 'cis-lmu__glot500-base', 'google__mt5-large', 'FacebookAI__xlm-roberta-large'], 'hellaswag_ml': ['bigscience__bloom-1b7', 'facebook__m2m100_1.2B', 'google__mt5-large', 'FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base', 'bigscience__bloom-1b1', 'bigscience__bloom-7b1'], 'hellaswag_kn': ['cis-lmu__glot500-base', 'google__mt5-xl', 'bigscience__bloom-1b1', 'facebook__m2m100_1.2B', 'FacebookAI__xlm-roberta-large', 'bigscience__bloom-1b7', 'google__mt5-large', 'bigscience__bloom-7b1'], 'hellaswag_nl': ['facebook__m2m100_1.2B', 'bigscience__bloom-7b1', 'FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base', 'bigscience__bloom-1b1', 'google__mt5-large', 'bigscience__bloom-1b7'], 'hellaswag_mr': ['facebook__m2m100_1.2B', 'bigscience__bloom-1b7', 'bigscience__bloom-1b1', 'FacebookAI__xlm-roberta-large', 'google__mt5-large', 'cis-lmu__glot500-base', 'bigscience__bloom-7b1'], 'hellaswag_ne': ['FacebookAI__xlm-roberta-large', 'bigscience__bloom-1b1', 'facebook__m2m100_1.2B', 'bigscience__bloom-7b1', 'google__mt5-large', 'bigscience__bloom-1b7', 'cis-lmu__glot500-base'], 'lambada_openai_mt_stablelm_fr': ['FacebookAI__xlm-roberta-large', 'google__mt5-xl', 'facebook__m2m100_1.2B', 'cis-lmu__glot500-base', 'google__mt5-large']}
Error encountered 204 time(s): RuntimeError: CUDA error: device-side assert triggered	{'xwinograd_fr': ['facebook__mbart-large-50'], 'lambada_openai_mt_stablelm_en': ['facebook__mbart-large-50'], 'm_mmlu_vi': ['facebook__mbart-large-50'], 'arc_es': ['facebook__mbart-large-50'], 'hellaswag_ro': ['facebook__mbart-large-50'], 'xstorycloze_te': ['facebook__mbart-large-50'], 'mgsm_native_cot_de': ['facebook__mbart-large-50'], 'arc_nl': ['facebook__mbart-large-50'], 'mgsm_native_cot_en': ['facebook__mbart-large-50'], 'm_mmlu_sv': ['facebook__mbart-large-50'], 'lambada_openai_mt_stablelm_fr': ['facebook__mbart-large-50'], 'truthfulqa_fr_mc1': ['facebook__mbart-large-50'], 'm_mmlu_sk': ['facebook__mbart-large-50', 'cis-lmu__glot500-base'], 'lambada_openai_mt_stablelm_it': ['facebook__mbart-large-50'], 'paws_de': ['facebook__mbart-large-50'], 'hellaswag_ta': ['facebook__mbart-large-50'], 'arc_hu': ['facebook__mbart-large-50'], 'hellaswag_fr': ['facebook__mbart-large-50'], 'mgsm_direct_th': ['facebook__mbart-large-50'], 'arc_hy': ['facebook__mbart-large-50'], 'arc_mr': ['facebook__mbart-large-50'], 'arc_gu': ['facebook__mbart-large-50'], 'arc_ta': ['facebook__mbart-large-50'], 'm_mmlu_te': ['facebook__mbart-large-50'], 'truthfulqa_hi_mc1': ['facebook__mbart-large-50'], 'xwinograd_zh': ['facebook__mbart-large-50'], 'hellaswag_uk': ['facebook__mbart-large-50'], 'truthfulqa_de_mc1': ['facebook__mbart-large-50'], 'xstorycloze_zh': ['facebook__mbart-large-50'], 'hellaswag_bn': ['facebook__mbart-large-50'], 'arc_ne': ['facebook__mbart-large-50'], 'arc_ca': ['facebook__mbart-large-50'], 'm_mmlu_ar': ['facebook__mbart-large-50'], 'truthfulqa_gu_mc1': ['facebook__mbart-large-50'], 'm_mmlu_fr': ['facebook__mbart-large-50'], 'arc_id': ['facebook__mbart-large-50'], 'm_mmlu_hy': ['facebook__mbart-large-50'], 'm_mmlu_ro': ['facebook__mbart-large-50'], 'xcopa_et': ['facebook__mbart-large-50'], 'mgsm_direct_sw': ['facebook__mbart-large-50'], 'truthfulqa_ca_mc1': ['facebook__mbart-large-50'], 'mgsm_direct_zh': ['facebook__mbart-large-50'], 'truthfulqa_uk_mc1': ['facebook__mbart-large-50'], 'm_mmlu_ne': ['facebook__mbart-large-50'], 'hellaswag_ru': ['facebook__mbart-large-50'], 'hellaswag_ml': ['facebook__mbart-large-50'], 'm_mmlu_da': ['facebook__mbart-large-50'], 'hellaswag_mr': ['facebook__mbart-large-50'], 'xcopa_it': ['facebook__mbart-large-50'], 'mgsm_direct_de': ['facebook__mbart-large-50'], 'm_mmlu_pt': ['facebook__mbart-large-50'], 'm_mmlu_es': ['facebook__mbart-large-50'], 'xstorycloze_es': ['facebook__mbart-large-50'], 'hellaswag_hu': ['facebook__mbart-large-50'], 'xnli_bg': ['facebook__mbart-large-50'], 'm_mmlu_kn': ['facebook__mbart-large-50'], 'truthfulqa_hy_mc1': ['facebook__mbart-large-50'], 'truthfulqa_zh_mc1': ['facebook__mbart-large-50'], 'm_mmlu_ca': ['facebook__mbart-large-50'], 'xstorycloze_hi': ['facebook__mbart-large-50'], 'xcopa_ta': ['facebook__mbart-large-50'], 'hellaswag_ca': ['facebook__mbart-large-50'], 'hellaswag_es': ['facebook__mbart-large-50'], 'paws_es': ['facebook__mbart-large-50'], 'mgsm_direct_ru': ['facebook__mbart-large-50'], 'mgsm_native_cot_fr': ['facebook__mbart-large-50'], 'truthfulqa_nl_mc1': ['facebook__mbart-large-50'], 'truthfulqa_mr_mc1': ['facebook__mbart-large-50'], 'm_mmlu_ta': ['facebook__mbart-large-50'], 'xwinograd_en': ['facebook__mbart-large-50'], 'xcopa_tr': ['facebook__mbart-large-50'], 'arc_fr': ['facebook__mbart-large-50'], 'xcopa_qu': ['facebook__mbart-large-50'], 'truthfulqa_pt_mc1': ['facebook__mbart-large-50'], 'm_mmlu_id': ['facebook__mbart-large-50'], 'truthfulqa_ro_mc1': ['facebook__mbart-large-50'], 'arc_ar': ['facebook__mbart-large-50'], 'mgsm_native_cot_ja': ['facebook__mbart-large-50'], 'hellaswag_sv': ['facebook__mbart-large-50'], 'arc_bn': ['facebook__mbart-large-50'], 'arc_ro': ['facebook__mbart-large-50'], 'm_mmlu_nl': ['facebook__mbart-large-50'], 'hellaswag_pt': ['facebook__mbart-large-50'], 'hellaswag_hr': ['facebook__mbart-large-50'], 'mgsm_native_cot_bn': ['facebook__mbart-large-50'], 'mgsm_native_cot_th': ['facebook__mbart-large-50'], 'hellaswag_vi': ['facebook__mbart-large-50'], 'truthfulqa_it_mc1': ['facebook__mbart-large-50'], 'mgsm_native_cot_te': ['facebook__mbart-large-50'], 'arc_de': ['facebook__mbart-large-50'], 'hellaswag_kn': ['facebook__mbart-large-50'], 'm_mmlu_bn': ['facebook__mbart-large-50'], 'xnli_hi': ['facebook__mbart-large-50'], 'hellaswag_hy': ['facebook__mbart-large-50'], 'mgsm_direct_bn': ['facebook__mbart-large-50'], 'hellaswag_te': ['facebook__mbart-large-50'], 'lambada_openai_mt_stablelm_de': ['facebook__mbart-large-50'], 'arc_te': ['facebook__mbart-large-50'], 'truthfulqa_ta_mc1': ['facebook__mbart-large-50'], 'm_mmlu_eu': ['facebook__mbart-large-50'], 'paws_ja': ['facebook__mbart-large-50'], 'xcopa_id': ['facebook__mbart-large-50'], 'paws_en': ['facebook__mbart-large-50'], 'hellaswag_de': ['facebook__mbart-large-50'], 'xnli_ar': ['facebook__mbart-large-50'], 'truthfulqa_vi_mc1': ['facebook__mbart-large-50'], 'hellaswag_da': ['facebook__mbart-large-50'], 'arc_eu': ['facebook__mbart-large-50'], 'mgsm_direct_fr': ['facebook__mbart-large-50'], 'xcopa_ht': ['facebook__mbart-large-50'], 'truthfulqa_ar_mc1': ['facebook__mbart-large-50'], 'm_mmlu_nb': ['facebook__mbart-large-50'], 'm_mmlu_sr': ['facebook__mbart-large-50'], 'm_mmlu_uk': ['facebook__mbart-large-50'], 'arc_sv': ['facebook__mbart-large-50'], 'xnli_tr': ['facebook__mbart-large-50'], 'xwinograd_ja': ['facebook__mbart-large-50'], 'm_mmlu_gu': ['facebook__mbart-large-50'], 'xstorycloze_ar': ['facebook__mbart-large-50'], 'arc_hr': ['facebook__mbart-large-50'], 'arc_zh': ['facebook__mbart-large-50'], 'xnli_ru': ['facebook__mbart-large-50'], 'xnli_de': ['facebook__mbart-large-50'], 'xnli_zh': ['facebook__mbart-large-50'], 'mgsm_direct_ja': ['facebook__mbart-large-50'], 'truthfulqa_ne_mc1': ['facebook__mbart-large-50'], 'truthfulqa_sr_mc1': ['facebook__mbart-large-50'], 'mgsm_direct_en': ['facebook__mbart-large-50'], 'truthfulqa_es_mc1': ['facebook__mbart-large-50'], 'xnli_el': ['facebook__mbart-large-50'], 'arc_uk': ['facebook__mbart-large-50'], 'truthfulqa_eu_mc1': ['facebook__mbart-large-50'], 'm_mmlu_hi': ['facebook__mbart-large-50'], 'hellaswag_id': ['facebook__mbart-large-50'], 'xcopa_th': ['facebook__mbart-large-50'], 'mgsm_native_cot_ru': ['facebook__mbart-large-50'], 'hellaswag_it': ['facebook__mbart-large-50'], 'truthfulqa_da_mc1': ['facebook__mbart-large-50'], 'hellaswag_eu': ['facebook__mbart-large-50'], 'xnli_th': ['facebook__mbart-large-50'], 'truthfulqa_te_mc1': ['facebook__mbart-large-50'], 'truthfulqa_hu_mc1': ['facebook__mbart-large-50'], 'hellaswag_nl': ['facebook__mbart-large-50'], 'hellaswag_ar': ['facebook__mbart-large-50'], 'm_mmlu_ru': ['facebook__mbart-large-50'], 'arc_it': ['facebook__mbart-large-50'], 'xstorycloze_my': ['facebook__mbart-large-50'], 'm_mmlu_it': ['facebook__mbart-large-50'], 'xstorycloze_en': ['facebook__mbart-large-50'], 'm_mmlu_en': ['facebook__mbart-large-50'], 'arc_pt': ['facebook__mbart-large-50'], 'hellaswag_gu': ['facebook__mbart-large-50'], 'xwinograd_pt': ['facebook__mbart-large-50'], 'truthfulqa_hr_mc1': ['facebook__mbart-large-50'], 'xnli_vi': ['facebook__mbart-large-50'], 'xnli_sw': ['facebook__mbart-large-50'], 'xnli_en': ['facebook__mbart-large-50'], 'xwinograd_ru': ['facebook__mbart-large-50'], 'arc_ml': ['facebook__mbart-large-50'], 'arc_vi': ['facebook__mbart-large-50'], 'arc_ru': ['facebook__mbart-large-50'], 'm_mmlu_hu': ['facebook__mbart-large-50'], 'arc_da': ['facebook__mbart-large-50'], 'xnli_fr': ['facebook__mbart-large-50'], 'arc_hi': ['facebook__mbart-large-50'], 'mgsm_native_cot_zh': ['facebook__mbart-large-50'], 'm_mmlu_de': ['facebook__mbart-large-50'], 'arc_kn': ['facebook__mbart-large-50'], 'm_mmlu_hr': ['facebook__mbart-large-50'], 'mgsm_direct_te': ['facebook__mbart-large-50'], 'truthfulqa_id_mc1': ['facebook__mbart-large-50'], 'truthfulqa_ru_mc1': ['facebook__mbart-large-50'], 'truthfulqa_kn_mc1': ['facebook__mbart-large-50'], 'paws_fr': ['facebook__mbart-large-50'], 'arc_sk': ['facebook__mbart-large-50'], 'xstorycloze_sw': ['facebook__mbart-large-50'], 'xcopa_zh': ['facebook__mbart-large-50'], 'xstorycloze_id': ['facebook__mbart-large-50'], 'truthfulqa_ml_mc1': ['facebook__mbart-large-50'], 'mgsm_native_cot_sw': ['facebook__mbart-large-50'], 'truthfulqa_bn_mc1': ['facebook__mbart-large-50'], 'paws_zh': ['facebook__mbart-large-50'], 'arc_sr': ['facebook__mbart-large-50'], 'truthfulqa_sv_mc1': ['facebook__mbart-large-50'], 'hellaswag_sr': ['facebook__mbart-large-50'], 'xcopa_sw': ['facebook__mbart-large-50'], 'xnli_es': ['facebook__mbart-large-50'], 'm_mmlu_is': ['facebook__mbart-large-50'], 'm_mmlu_mr': ['facebook__mbart-large-50'], 'hellaswag_hi': ['facebook__mbart-large-50'], 'mgsm_direct_es': ['facebook__mbart-large-50'], 'truthfulqa_sk_mc1': ['facebook__mbart-large-50'], 'm_mmlu_zh': ['facebook__mbart-large-50'], 'xcopa_vi': ['facebook__mbart-large-50'], 'xstorycloze_ru': ['facebook__mbart-large-50'], 'm_mmlu_ml': ['facebook__mbart-large-50'], 'xstorycloze_eu': ['facebook__mbart-large-50'], 'lambada_openai_mt_stablelm_es': ['facebook__mbart-large-50'], 'xnli_ur': ['facebook__mbart-large-50'], 'paws_ko': ['facebook__mbart-large-50'], 'mgsm_native_cot_es': ['facebook__mbart-large-50'], 'hellaswag_ne': ['facebook__mbart-large-50'], 'hellaswag_sk': ['facebook__mbart-large-50']}
Error encountered 17 time(s): torch.OutOfMemoryError: CUDA out of memory.	{'hellaswag_sr': ['google__mt5-xl'], 'hellaswag_pt': ['google__mt5-xl'], 'hellaswag_es': ['google__mt5-xl'], 'hellaswag_ne': ['google__mt5-xl'], 'hellaswag_ro': ['google__mt5-xl'], 'xstorycloze_te': ['SeaLLMs__SeaLLM-7B-v2'], 'truthfulqa_it_mc1': ['dice-research__lola_v1'], 'hellaswag_nl': ['google__mt5-xl'], 'truthfulqa_pt_mc1': ['dice-research__lola_v1'], 'hellaswag_ar': ['google__mt5-xl'], 'hellaswag_ml': ['google__mt5-xl'], 'hellaswag_sv': ['google__mt5-xl'], 'hellaswag_bn': ['google__mt5-xl'], 'hellaswag_mr': ['google__mt5-xl'], 'hellaswag_de': ['google__mt5-xl'], 'hellaswag_ru': ['google__mt5-xl'], 'truthfulqa_zh_mc1': ['dice-research__lola_v1']}