Total 311/311 errors found.
Error encountered 100 time(s): RuntimeError: CUDA error: CUBLAS_STATUS_EXECUTION_FAILED	{'m_mmlu_da': ['cis-lmu__glot500-base', 'FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl'], 'm_mmlu_gu': ['cis-lmu__glot500-base', 'FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl'], 'm_mmlu_fr': ['cis-lmu__glot500-base', 'facebook__xlm-roberta-xl', 'FacebookAI__xlm-roberta-large'], 'm_mmlu_en': ['facebook__xlm-roberta-xl', 'FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base'], 'm_mmlu_te': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl'], 'm_mmlu_pt': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl', 'cis-lmu__glot500-base'], 'm_mmlu_zh': ['facebook__xlm-roberta-xl', 'cis-lmu__glot500-base', 'FacebookAI__xlm-roberta-large'], 'm_mmlu_ne': ['cis-lmu__glot500-base', 'FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl'], 'm_mmlu_sk': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl', 'cis-lmu__glot500-base'], 'm_mmlu_ru': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl', 'cis-lmu__glot500-base'], 'm_mmlu_hi': ['FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base', 'facebook__xlm-roberta-xl'], 'm_mmlu_uk': ['FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base', 'facebook__xlm-roberta-xl'], 'm_mmlu_sv': ['cis-lmu__glot500-base', 'FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl'], 'm_mmlu_bn': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl', 'cis-lmu__glot500-base'], 'm_mmlu_hr': ['facebook__xlm-roberta-xl', 'FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base'], 'm_mmlu_it': ['cis-lmu__glot500-base', 'FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl'], 'm_mmlu_ml': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl', 'cis-lmu__glot500-base'], 'arc_es': ['google__mt5-large'], 'truthfulqa_ro_mc1': ['google__mt5-large'], 'm_mmlu_is': ['facebook__xlm-roberta-xl', 'FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base'], 'm_mmlu_de': ['FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base', 'facebook__xlm-roberta-xl'], 'm_mmlu_kn': ['facebook__xlm-roberta-xl', 'FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base'], 'm_mmlu_ro': ['FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base', 'facebook__xlm-roberta-xl'], 'm_mmlu_nl': ['cis-lmu__glot500-base', 'FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl'], 'm_mmlu_ca': ['facebook__xlm-roberta-xl', 'cis-lmu__glot500-base', 'FacebookAI__xlm-roberta-large'], 'm_mmlu_id': ['cis-lmu__glot500-base', 'facebook__xlm-roberta-xl', 'FacebookAI__xlm-roberta-large'], 'm_mmlu_ta': ['FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base', 'facebook__xlm-roberta-xl'], 'm_mmlu_ar': ['FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl', 'cis-lmu__glot500-base'], 'm_mmlu_eu': ['facebook__xlm-roberta-xl', 'FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base'], 'm_mmlu_hu': ['cis-lmu__glot500-base', 'FacebookAI__xlm-roberta-large', 'facebook__xlm-roberta-xl'], 'm_mmlu_mr': ['cis-lmu__glot500-base', 'facebook__xlm-roberta-xl', 'FacebookAI__xlm-roberta-large'], 'm_mmlu_sr': ['facebook__xlm-roberta-xl', 'cis-lmu__glot500-base', 'FacebookAI__xlm-roberta-large'], 'm_mmlu_vi': ['FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base', 'facebook__xlm-roberta-xl'], 'm_mmlu_es': ['facebook__xlm-roberta-xl', 'cis-lmu__glot500-base', 'FacebookAI__xlm-roberta-large'], 'm_mmlu_hy': ['FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base', 'facebook__xlm-roberta-xl']}
Error encountered 21 time(s): RuntimeError: CUDA error: uncorrectable ECC error encountered	{'arc_ru': ['cis-lmu__glot500-base'], 'truthfulqa_vi_mc1': ['facebook__m2m100_1.2B'], 'm_mmlu_pt': ['bigscience__bloom-1b1'], 'mgsm_native_cot_en': ['google__mt5-large'], 'arc_te': ['google__mt5-large'], 'hellaswag_te': ['dice-research__lola_v1'], 'truthfulqa_uk_mc1': ['google__mt5-large'], 'truthfulqa_ta_mc1': ['bigscience__bloom-1b7'], 'mgsm_direct_te': ['bigscience__bloom-1b7'], 'mgsm_native_cot_sw': ['cis-lmu__glot500-base'], 'mgsm_direct_th': ['cis-lmu__glot500-base'], 'mgsm_native_cot_fr': ['google__mt5-xl'], 'mgsm_native_cot_ja': ['facebook__m2m100_1.2B'], 'truthfulqa_es_mc1': ['facebook__m2m100_1.2B'], 'm_mmlu_ml': ['ai-forever__mGPT'], 'truthfulqa_da_mc1': ['facebook__xlm-roberta-xl'], 'mgsm_direct_ru': ['ai-forever__mGPT'], 'mgsm_direct_zh': ['bigscience__bloom-1b7'], 'hellaswag_ca': ['bigscience__bloom-1b7'], 'm_mmlu_ro': ['dice-research__lola_v1'], 'mgsm_native_cot_bn': ['FacebookAI__xlm-roberta-large']}
Error encountered 106 time(s): RuntimeError: CUDA error: device-side assert triggered	{'m_mmlu_gu': ['facebook__mbart-large-50'], 'arc_ru': ['facebook__mbart-large-50'], 'xwinograd_zh': ['facebook__mbart-large-50'], 'truthfulqa_nl_mc1': ['facebook__mbart-large-50'], 'mgsm_direct_ja': ['facebook__mbart-large-50'], 'truthfulqa_vi_mc1': ['facebook__mbart-large-50'], 'hellaswag_ne': ['facebook__mbart-large-50'], 'mgsm_direct_de': ['facebook__mbart-large-50'], 'mgsm_native_cot_en': ['facebook__mbart-large-50'], 'm_mmlu_fr': ['facebook__mbart-large-50'], 'mgsm_native_cot_zh': ['facebook__mbart-large-50'], 'm_mmlu_te': ['cis-lmu__glot500-base'], 'xcopa_tr': ['facebook__mbart-large-50'], 'arc_gu': ['facebook__mbart-large-50'], 'hellaswag_vi': ['facebook__mbart-large-50'], 'm_mmlu_en': ['facebook__mbart-large-50'], 'xstorycloze_ru': ['facebook__mbart-large-50'], 'lambada_openai_mt_stablelm_it': ['facebook__mbart-large-50'], 'hellaswag_ar': ['facebook__mbart-large-50'], 'hellaswag_ru': ['facebook__mbart-large-50'], 'hellaswag_gu': ['facebook__mbart-large-50'], 'hellaswag_hi': ['facebook__mbart-large-50'], 'm_mmlu_ne': ['facebook__mbart-large-50'], 'lambada_openai_mt_stablelm_de': ['facebook__mbart-large-50'], 'm_mmlu_zh': ['facebook__mbart-large-50'], 'mgsm_direct_en': ['facebook__mbart-large-50'], 'xnli_ru': ['facebook__mbart-large-50'], 'xwinograd_ru': ['facebook__mbart-large-50'], 'mgsm_direct_es': ['facebook__mbart-large-50'], 'hellaswag_nl': ['facebook__mbart-large-50'], 'truthfulqa_zh_mc1': ['facebook__mbart-large-50'], 'arc_ne': ['facebook__mbart-large-50'], 'hellaswag_fr': ['facebook__mbart-large-50'], 'xnli_hi': ['facebook__mbart-large-50'], 'mgsm_native_cot_es': ['facebook__mbart-large-50'], 'm_mmlu_hi': ['facebook__mbart-large-50'], 'm_mmlu_ru': ['facebook__mbart-large-50'], 'arc_de': ['facebook__mbart-large-50'], 'mgsm_native_cot_fr': ['facebook__mbart-large-50'], 'hellaswag_de': ['facebook__mbart-large-50'], 'truthfulqa_ne_mc1': ['facebook__mbart-large-50'], 'xstorycloze_my': ['facebook__mbart-large-50'], 'xstorycloze_es': ['facebook__mbart-large-50'], 'lambada_openai_mt_stablelm_es': ['facebook__mbart-large-50'], 'mgsm_native_cot_ja': ['facebook__mbart-large-50'], 'xnli_vi': ['facebook__mbart-large-50'], 'arc_es': ['facebook__mbart-large-50'], 'truthfulqa_es_mc1': ['facebook__mbart-large-50'], 'xnli_tr': ['facebook__mbart-large-50'], 'arc_hi': ['facebook__mbart-large-50'], 'arc_ar': ['facebook__mbart-large-50'], 'truthfulqa_ru_mc1': ['facebook__mbart-large-50'], 'xwinograd_ja': ['facebook__mbart-large-50'], 'm_mmlu_it': ['facebook__mbart-large-50'], 'mgsm_native_cot_de': ['facebook__mbart-large-50'], 'truthfulqa_ro_mc1': ['facebook__mbart-large-50'], 'truthfulqa_hi_mc1': ['facebook__mbart-large-50'], 'paws_zh': ['facebook__mbart-large-50'], 'xnli_en': ['facebook__mbart-large-50'], 'paws_ja': ['facebook__mbart-large-50'], 'hellaswag_it': ['facebook__mbart-large-50'], 'truthfulqa_fr_mc1': ['facebook__mbart-large-50'], 'xnli_es': ['facebook__mbart-large-50'], 'xwinograd_fr': ['facebook__mbart-large-50'], 'm_mmlu_de': ['facebook__mbart-large-50'], 'arc_fr': ['facebook__mbart-large-50'], 'lambada_openai_mt_stablelm_fr': ['facebook__mbart-large-50'], 'truthfulqa_ar_mc1': ['facebook__mbart-large-50'], 'arc_vi': ['facebook__mbart-large-50'], 'xnli_de': ['facebook__mbart-large-50'], 'm_mmlu_nl': ['facebook__mbart-large-50'], 'paws_de': ['facebook__mbart-large-50'], 'hellaswag_es': ['facebook__mbart-large-50'], 'arc_nl': ['facebook__mbart-large-50'], 'arc_zh': ['facebook__mbart-large-50'], 'lambada_openai_mt_stablelm_en': ['facebook__mbart-large-50'], 'm_mmlu_ar': ['facebook__mbart-large-50'], 'truthfulqa_gu_mc1': ['facebook__mbart-large-50'], 'xstorycloze_zh': ['facebook__mbart-large-50'], 'xwinograd_en': ['facebook__mbart-large-50'], 'paws_ko': ['facebook__mbart-large-50'], 'm_mmlu_ro': ['facebook__mbart-large-50'], 'mgsm_native_cot_ru': ['facebook__mbart-large-50'], 'paws_en': ['facebook__mbart-large-50'], 'xcopa_vi': ['facebook__mbart-large-50'], 'paws_es': ['facebook__mbart-large-50'], 'paws_fr': ['facebook__mbart-large-50'], 'xstorycloze_hi': ['facebook__mbart-large-50'], 'xcopa_et': ['facebook__mbart-large-50'], 'xnli_zh': ['facebook__mbart-large-50'], 'arc_it': ['facebook__mbart-large-50'], 'hellaswag_ro': ['facebook__mbart-large-50'], 'truthfulqa_it_mc1': ['facebook__mbart-large-50'], 'mgsm_direct_ru': ['facebook__mbart-large-50'], 'xcopa_it': ['facebook__mbart-large-50'], 'xstorycloze_ar': ['facebook__mbart-large-50'], 'mgsm_direct_fr': ['facebook__mbart-large-50'], 'mgsm_direct_zh': ['facebook__mbart-large-50'], 'xstorycloze_en': ['facebook__mbart-large-50'], 'xnli_ar': ['facebook__mbart-large-50'], 'xnli_fr': ['facebook__mbart-large-50'], 'arc_ro': ['facebook__mbart-large-50'], 'xcopa_zh': ['facebook__mbart-large-50'], 'truthfulqa_de_mc1': ['facebook__mbart-large-50'], 'm_mmlu_vi': ['facebook__mbart-large-50'], 'm_mmlu_es': ['facebook__mbart-large-50']}
Error encountered 66 time(s): AssertionError	{'hellaswag_te': ['bigscience__bloom-1b1', 'google__mt5-large', 'bigscience__bloom-1b7', 'google__mt5-xl', 'FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base'], 'hellaswag_ne': ['bigscience__bloom-1b7', 'FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base', 'facebook__m2m100_1.2B', 'google__mt5-large', 'bigscience__bloom-1b1'], 'hellaswag_gu': ['facebook__m2m100_1.2B', 'bigscience__bloom-1b1', 'bigscience__bloom-1b7', 'FacebookAI__xlm-roberta-large', 'google__mt5-large', 'google__mt5-xl', 'cis-lmu__glot500-base'], 'hellaswag_nl': ['google__mt5-large', 'facebook__m2m100_1.2B', 'FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base'], 'hellaswag_mr': ['bigscience__bloom-1b7', 'google__mt5-large', 'bigscience__bloom-1b1', 'FacebookAI__xlm-roberta-large', 'facebook__m2m100_1.2B', 'cis-lmu__glot500-base'], 'hellaswag_ml': ['facebook__m2m100_1.2B', 'bigscience__bloom-1b7', 'FacebookAI__xlm-roberta-large', 'bigscience__bloom-1b1', 'google__mt5-large', 'cis-lmu__glot500-base'], 'hellaswag_bn': ['bigscience__bloom-1b1', 'facebook__m2m100_1.2B', 'bigscience__bloom-1b7', 'google__mt5-large', 'FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base'], 'lambada_openai_mt_stablelm_fr': ['FacebookAI__xlm-roberta-large', 'google__mt5-xl', 'facebook__m2m100_1.2B', 'cis-lmu__glot500-base', 'google__mt5-large'], 'hellaswag_eu': ['FacebookAI__xlm-roberta-large', 'bigscience__bloom-1b1', 'bigscience__bloom-1b7', 'google__mt5-large', 'google__mt5-xl', 'cis-lmu__glot500-base'], 'hellaswag_kn': ['bigscience__bloom-1b1', 'bigscience__bloom-1b7', 'google__mt5-large', 'FacebookAI__xlm-roberta-large', 'cis-lmu__glot500-base', 'facebook__m2m100_1.2B', 'google__mt5-xl'], 'hellaswag_ta': ['bigscience__bloom-1b7', 'bigscience__bloom-1b1', 'facebook__m2m100_1.2B', 'FacebookAI__xlm-roberta-large', 'google__mt5-xl', 'google__mt5-large', 'cis-lmu__glot500-base']}
Error encountered 2 time(s): FileNotFoundError: No such file or directory	{'xnli_bg': ['facebook__xlm-roberta-xl'], 'xcopa_sw': ['bigscience__bloom-1b7']}
Error encountered 16 time(s): torch.OutOfMemoryError: CUDA out of memory.	{'hellaswag_ne': ['google__mt5-xl'], 'truthfulqa_pt_mc1': ['dice-research__lola_v1'], 'hellaswag_ru': ['google__mt5-xl'], 'hellaswag_ar': ['google__mt5-xl'], 'hellaswag_nl': ['google__mt5-xl'], 'truthfulqa_zh_mc1': ['dice-research__lola_v1'], 'hellaswag_mr': ['google__mt5-xl'], 'hellaswag_de': ['google__mt5-xl'], 'hellaswag_pt': ['google__mt5-xl'], 'hellaswag_ml': ['google__mt5-xl'], 'hellaswag_bn': ['google__mt5-xl'], 'hellaswag_sv': ['google__mt5-xl'], 'hellaswag_es': ['google__mt5-xl'], 'truthfulqa_it_mc1': ['dice-research__lola_v1'], 'hellaswag_sr': ['google__mt5-xl'], 'hellaswag_ro': ['google__mt5-xl']}