{ "metadata": { "title": "Systematizing the Evaluation of Generalist Multimodal Large Language Models (SoK)", "author": "Alireza Abbaszadeh", "affiliation": "Department of Computer Engineering, Islamic Azad University, Mashhad, Iran", "total_benchmarks": 169, "year_range": "2015-2026", "version": "2.0.0", "last_updated": "2026-10-19" }, "summary_statistics": { "modality_totals": { "Text": 147, "Image": 103, "Audio": 31, "Video": 44, "3D": 24, "Sensor": 37, "Action": 51 }, "capability_totals": { "Perception": 169, "Grounding": 135, "Reasoning": 157, "Instruction Following": 63, "Domain Knowledge": 49, "Long-Horizon Memory": 53, "Planning": 55, "Tool Use": 35, "Embodiment": 47 }, "judge_counts": { "auto": 161, "human": 4, "llm": 4, "hybrid": 0 }, "matrix_7x9": { "modalities": [ "Text", "Image", "Audio", "Video", "3D", "Sensor", "Action" ], "capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following", "Domain Knowledge", "Long-Horizon Memory", "Planning", "Tool Use", "Embodiment" ], "grid": [ [ 147, 123, 139, 57, 47, 47, 42, 29, 37 ], [ 103, 97, 93, 39, 30, 19, 38, 19, 36 ], [ 31, 15, 28, 18, 7, 7, 8, 13, 3 ], [ 44, 37, 44, 10, 10, 32, 13, 0, 11 ], [ 24, 19, 21, 9, 4, 2, 14, 4, 15 ], [ 37, 30, 34, 27, 5, 12, 33, 5, 34 ], [ 51, 42, 51, 42, 6, 22, 51, 15, 46 ] ] } }, "benchmarks": [ { "id": "mmbench2023", "name": "MMBench", "symbol": "MMB", "year": 2023, "venue": "ECCV", "url": "https://arxiv.org/abs/2304.04675", "family": "Vision-Language", "task_family": "ExamQA", "domain": "General", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Single-turn multiple-choice questions grouped into categories; accuracy aggregated across ~3000 Qs.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following", "Domain Knowledge" ], "cap_count": 5, "cap_score": 0.5556, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "mmbench2023", "citation": { "title": "MMBench: Is Your Multi-modal Model an All-around Player?", "authors": "Liu, Yuan and Duan, Haodong and Zhang, Yuanhan and Li, Bo and Zhang, Songyang and Zhao, Wangbo and Yuan, Yike and Wang, Jiaqi and He, Conghui and Liu, Ziwei and others", "year": 2024, "venue": "European Conference on Computer Vision (ECCV)", "bibtex": "@inproceedings{mmbench2023,\n title={MMBench: Is Your Multi-modal Model an All-around Player?},\n author={Liu, Yuan and Duan, Haodong and Zhang, Yuanhan and Li, Bo and Zhang, Songyang and Zhao, Wangbo and Yuan, Yike and Wang, Jiaqi and He, Conghui and Liu, Ziwei and others},\n booktitle={European Conference on Computer Vision (ECCV)},\n year={2024},\n url={https://arxiv.org/abs/2307.06281}\n}" } }, { "id": "mme2023", "name": "MME", "symbol": "MME", "year": 2023, "venue": "arXiv", "url": "https://arxiv.org/abs/2306.13394", "family": "Vision-Language", "task_family": "Diagnostic", "domain": "General", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "14 sub-dimensions covering perception and reasoning with automatic scoring scripts.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following" ], "cap_count": 4, "cap_score": 0.4444, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "mme2023", "citation": { "title": "MME: A Comprehensive Evaluation Benchmark for Multimodal Large Language Models", "authors": "Fu, Chaoyou and Chen, Peixian and Shen, Yunhang and Qin, Yulei and Zhang, Mengdan and Lin, Xu and Yang, Jinrui and Zheng, Xiawu and Li, Ke and Sun, Xing and others", "year": 2023, "venue": "arXiv preprint arXiv:2306.13394", "bibtex": "@article{mme2023,\n title={MME: A Comprehensive Evaluation Benchmark for Multimodal Large Language Models},\n author={Fu, Chaoyou and Chen, Peixian and Shen, Yunhang and Qin, Yulei and Zhang, Mengdan and Lin, Xu and Yang, Jinrui and Zheng, Xiawu and Li, Ke and Sun, Xing and others},\n journal={arXiv preprint arXiv:2306.13394},\n year={2023},\n url={https://arxiv.org/abs/2306.13394}\n}" } }, { "id": "seedbench2023", "name": "SEED-Bench", "symbol": "SDB", "year": 2023, "venue": "CVPR", "url": "https://arxiv.org/abs/2304.11565", "family": "Video & Temporal", "task_family": "ExamQA", "domain": "General", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Multiple-choice prompts across 19 categories for images and short videos.", "active_modalities": [ "Text", "Image", "Video" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following" ], "cap_count": 4, "cap_score": 0.4444, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "seedbench2023", "citation": { "title": "SEED-Bench: Benchmarking Multimodal LLMs with Generative Comprehension", "authors": "Li, Bohao and Wang, Rui and Wang, Guangzhi and Ge, Yuying and Ge, Yixiao and Shan, Ying", "year": 2024, "venue": "Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)", "bibtex": "@inproceedings{seedbench2023,\n title={SEED-Bench: Benchmarking Multimodal LLMs with Generative Comprehension},\n author={Li, Bohao and Wang, Rui and Wang, Guangzhi and Ge, Yuying and Ge, Yixiao and Shan, Ying},\n booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},\n year={2024},\n url={https://arxiv.org/abs/2307.16125}\n}" } }, { "id": "mmmu2023", "name": "MMMU", "symbol": "MMU", "year": 2023, "venue": "CVPR", "url": "https://arxiv.org/abs/2311.16502", "family": "Vision-Language", "task_family": "ExamQA", "domain": "Academic", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "12k multidisciplinary questions mixing diagrams", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Reasoning", "Domain Knowledge" ], "cap_count": 3, "cap_score": 0.3333, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "mmmu2023", "citation": { "title": "MMMU: A Massive Multi-discipline Multimodal Understanding and Reasoning Benchmark for Expert AGI", "authors": "Yue, Xiang and Ni, Yuansheng and Zhang, Kai and Zheng, Tianyu and Liu, Ruoqi and Ge, Yiran and Chen, Samuel and Wang, Yuxuan and Shen, Wangchunshu and others", "year": 2024, "venue": "Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)", "bibtex": "@inproceedings{mmmu2023,\n title={MMMU: A Massive Multi-discipline Multimodal Understanding and Reasoning Benchmark for Expert AGI},\n author={Yue, Xiang and Ni, Yuansheng and Zhang, Kai and Zheng, Tianyu and Liu, Ruoqi and Ge, Yiran and Chen, Samuel and Wang, Yuxuan and Shen, Wangchunshu and others},\n booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},\n year={2024},\n url={https://arxiv.org/abs/2311.16502}\n}" } }, { "id": "videomme2024", "name": "Video-MME", "symbol": "VME", "year": 2024, "venue": "CVPR", "url": "https://arxiv.org/abs/2406.10053", "family": "Video & Temporal", "task_family": "VideoQA", "domain": "General", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Evaluates spatial-temporal reasoning on videos >2 minutes with MCQ prompts scored automatically.", "active_modalities": [ "Text", "Video" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Long-Horizon Memory" ], "cap_count": 4, "cap_score": 0.4444, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "videomme2024", "citation": { "title": "Video-MME: The First-Ever Comprehensive Evaluation Benchmark of Multi-modal LLMs in Video Analysis", "authors": "Fu, Chaoyou and Dai, Yuhan and Luo, Yongdong and Li, Lei and Wang, Shuhuai and Shen, Yunhang and Lin, Xu and Zhang, Mengdan and Sun, Xing and others", "year": 2025, "venue": "Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)", "bibtex": "@inproceedings{videomme2024,\n title={Video-MME: The First-Ever Comprehensive Evaluation Benchmark of Multi-modal LLMs in Video Analysis},\n author={Fu, Chaoyou and Dai, Yuhan and Luo, Yongdong and Li, Lei and Wang, Shuhuai and Shen, Yunhang and Lin, Xu and Zhang, Mengdan and Sun, Xing and others},\n booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},\n year={2025},\n url={https://arxiv.org/abs/2405.21075}\n}" } }, { "id": "lvbench2024", "name": "LVBench", "symbol": "LVB", "year": 2024, "venue": "ICCV", "url": "https://arxiv.org/abs/2402.16671", "family": "Video & Temporal", "task_family": "VideoQA", "domain": "General", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Long video QA benchmark focusing on memory-intensive narratives; majority MC questions.", "active_modalities": [ "Text", "Video" ], "active_capabilities": [ "Perception", "Reasoning", "Long-Horizon Memory" ], "cap_count": 3, "cap_score": 0.3333, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "lvbench2024", "citation": { "title": "LVBench: An Extreme Long Video Understanding Benchmark", "authors": "Wang, Weihan and He, Zehai and Hong, Wenyi and Cheng, Yean and Zhang, Xiaohan and Ji, Jiashuo and Wang, Xiaohan and Long, Yuxuan and others", "year": 2025, "venue": "Proceedings of the IEEE/CVF International Conference on Computer Vision (ICCV)", "bibtex": "@inproceedings{lvbench2024,\n title={LVBench: An Extreme Long Video Understanding Benchmark},\n author={Wang, Weihan and He, Zehai and Hong, Wenyi and Cheng, Yean and Zhang, Xiaohan and Ji, Jiashuo and Wang, Xiaohan and Long, Yuxuan and others},\n booktitle={Proceedings of the IEEE/CVF International Conference on Computer Vision (ICCV)},\n year={2025},\n url={https://arxiv.org/abs/2406.08035}\n}" } }, { "id": "audiobench2024", "name": "AudioBench", "symbol": "AUB", "year": 2024, "venue": "NAACL", "url": "https://arxiv.org/abs/2403.08849", "family": "Audio & Speech", "task_family": "AudioQA", "domain": "General", "metrics": "Accuracy;BLEU", "judge_type": "hybrid", "protocol_notes": "Mix of structured QA and generative captioning scored via accuracy and BLEU using scripts plus LLM reviewers.", "active_modalities": [ "Text", "Audio" ], "active_capabilities": [ "Perception", "Reasoning" ], "cap_count": 2, "cap_score": 0.2222, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "audiobench2024", "citation": { "title": "AudioBench: A Universal Benchmark for Audio Large Language Models", "authors": "Wang, Bin and Zou, Xunlong and Lin, Geyu and Sun, Shuo and Liu, Zhuohan and Zhang, Mengdan and others", "year": 2025, "venue": "Proceedings of the North American Chapter of the Association for Computational Linguistics (NAACL)", "bibtex": "@inproceedings{audiobench2024,\n title={AudioBench: A Universal Benchmark for Audio Large Language Models},\n author={Wang, Bin and Zou, Xunlong and Lin, Geyu and Sun, Shuo and Liu, Zhuohan and Zhang, Mengdan and others},\n booktitle={Proceedings of the North American Chapter of the Association for Computational Linguistics (NAACL)},\n year={2025},\n url={https://arxiv.org/abs/2406.16020}\n}" } }, { "id": "chartqa2020", "name": "ChartQA", "symbol": "CQA", "year": 2020, "venue": "ACL", "url": "https://arxiv.org/abs/2004.09441", "family": "Doc & Diagrams", "task_family": "ChartQA", "domain": "DataViz", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Short answer chart reasoning scored via exact match / relaxed accuracy.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning" ], "cap_count": 3, "cap_score": 0.3333, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "chartqa2020", "citation": { "title": "ChartQA: A Benchmark for Question Answering on Charts and Tables", "authors": "Masry, Ahmed and others", "year": 2020, "venue": "ACL", "bibtex": "@inproceedings{chartqa2020,\n title={ChartQA: A Benchmark for Question Answering on Charts and Tables},\n author={Masry, Ahmed and others},\n booktitle={ACL},\n year={2020},\n url={https://arxiv.org/abs/2004.09441}\n}" } }, { "id": "scanqa2022", "name": "ScanQA", "symbol": "SCA", "year": 2022, "venue": "CVPR", "url": "https://arxiv.org/abs/2206.01740", "family": "3D & Spatial", "task_family": "3DQA", "domain": "3D", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Open-ended QA grounded on ScanNet rooms; answers normalized for automatic matching.", "active_modalities": [ "Text", "3D" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning" ], "cap_count": 3, "cap_score": 0.3333, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "scanqa2022", "citation": { "title": "ScanQA: 3D Question Answering for Spatial Scene Understanding", "authors": "Azuma, Daichi and Miyanishi, Taiki and Kurita, Shuhei and Kawanabe, Motoaki", "year": 2022, "venue": "Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)", "bibtex": "@inproceedings{scanqa2022,\n title={ScanQA: 3D Question Answering for Spatial Scene Understanding},\n author={Azuma, Daichi and Miyanishi, Taiki and Kurita, Shuhei and Kawanabe, Motoaki},\n booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},\n year={2022},\n url={https://arxiv.org/abs/2112.10482}\n}" } }, { "id": "alfred2019", "name": "ALFRED", "symbol": "ALF", "year": 2020, "venue": "CVPR", "url": "https://arxiv.org/abs/1912.01734", "family": "Embodied & Agents", "task_family": "InstructionFollowing", "domain": "Embodied", "metrics": "SuccessRate", "judge_type": "auto", "protocol_notes": "Simulation-based embodied benchmark measuring goal-conditioned success and path efficiency.", "active_modalities": [ "Text", "Image", "Sensor", "Action" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following", "Planning", "Embodiment" ], "cap_count": 6, "cap_score": 0.6667, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "alfred2020", "citation": { "title": "ALFRED: A Benchmark for Interpreting Grounded Instructions for Everyday Tasks", "authors": "Shridhar, Mohit and others", "year": 2020, "venue": "CVPR", "bibtex": "@inproceedings{alfred2020,\n title={ALFRED: A Benchmark for Interpreting Grounded Instructions for Everyday Tasks},\n author={Shridhar, Mohit and others},\n booktitle={CVPR},\n year={2020}\n}" } }, { "id": "egoschema2023", "name": "EgoSchema", "symbol": "EGO", "year": 2023, "venue": "NeurIPS", "url": "https://arxiv.org/abs/2308.06314", "family": "Video & Temporal", "task_family": "VideoQA", "domain": "Egocentric", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Long egocentric videos (~10 min) with temporal MCQ prompts scored automatically.", "active_modalities": [ "Text", "Video" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Long-Horizon Memory" ], "cap_count": 4, "cap_score": 0.4444, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "egoschema2023", "citation": { "title": "EgoSchema: A Diagnostic Benchmark for Very Long-form Video Language Understanding", "authors": "Mangalam, Karttikeya and Akshulakov, Raiymbek and Malik, Jitendra", "year": 2023, "venue": "Advances in Neural Information Processing Systems (NeurIPS)", "bibtex": "@inproceedings{egoschema2023,\n title={EgoSchema: A Diagnostic Benchmark for Very Long-form Video Language Understanding},\n author={Mangalam, Karttikeya and Akshulakov, Raiymbek and Malik, Jitendra},\n booktitle={Advances in Neural Information Processing Systems (NeurIPS)},\n year={2023},\n url={https://arxiv.org/abs/2308.09126}\n}" } }, { "id": "teach2021", "name": "TEACh", "symbol": "TCH", "year": 2021, "venue": "NeurIPS", "url": "https://arxiv.org/abs/2105.08805", "family": "Embodied & Agents", "task_family": "InstructionFollowing", "domain": "Household", "metrics": "SuccessRate", "judge_type": "auto", "protocol_notes": "Multi-turn dialog + embodied execution benchmark using simulated homes; success and path metrics computed automatically.", "active_modalities": [ "Text", "Image", "Sensor", "Action" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following", "Long-Horizon Memory", "Planning", "Embodiment" ], "cap_count": 7, "cap_score": 0.7778, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "teach2021", "citation": { "title": "TEACh: Task-Driven Embodied Agents that Chat", "authors": "Padmakumar, Vishal and others", "year": 2021, "venue": "NeurIPS", "bibtex": "@inproceedings{teach2021,\n title={TEACh: Task-Driven Embodied Agents that Chat},\n author={Padmakumar, Vishal and others},\n booktitle={NeurIPS},\n year={2021},\n url={https://arxiv.org/abs/2105.08805}\n}" } }, { "id": "alexaarena2023", "name": "Alexa Arena", "symbol": "ALE", "year": 2023, "venue": "NeurIPSDemo", "url": "https://arxiv.org/abs/2308.05221", "family": "Embodied & Agents", "task_family": "InstructionFollowing", "domain": "Household", "metrics": "MissionSuccessRate", "judge_type": "auto", "protocol_notes": "SimBot challenge platform with 46k dialogue-guided multi-room tasks and EvalAI scoring over mission success and dialog efficiency on unseen homes.", "active_modalities": [ "Text", "Image", "Video", "Action" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following", "Long-Horizon Memory", "Planning", "Embodiment" ], "cap_count": 7, "cap_score": 0.7778, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "alexaarena2023", "citation": { "title": "Alexa Arena: A User-Centric Interactive Platform for Embodied AI", "authors": "Gao, Qiaozi and Thattai, Govind and Shakiah, Suhaila and Gao, Xiaofeng and others", "year": 2023, "venue": "Advances in Neural Information Processing Systems (NeurIPS)", "bibtex": "@inproceedings{alexaarena2023,\n title={Alexa Arena: A User-Centric Interactive Platform for Embodied AI},\n author={Gao, Qiaozi and Thattai, Govind and Shakiah, Suhaila and Gao, Xiaofeng and others},\n booktitle={Advances in Neural Information Processing Systems (NeurIPS)},\n year={2023},\n url={https://arxiv.org/abs/2303.01586}\n}" } }, { "id": "mathvista2023", "name": "MathVista", "symbol": "MAT", "year": 2023, "venue": "NeurIPS", "url": "https://arxiv.org/abs/2310.07975", "family": "Vision-Language", "task_family": "MathQA", "domain": "STEM", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Aggregated accuracy over MC and free-form answers requiring diagram parsing and optional tool/calc use.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Domain Knowledge", "Tool Use" ], "cap_count": 5, "cap_score": 0.5556, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "mathvista2023", "citation": { "title": "MathVista: Evaluating Multimodal Mathematical Reasoning of Foundation Models", "authors": "Lu, Pan and others", "year": 2023, "venue": "NeurIPS", "bibtex": "@article{mathvista2023,\n title={MathVista: Evaluating Multimodal Mathematical Reasoning of Foundation Models},\n author={Lu, Pan and others},\n journal={NeurIPS},\n year={2023},\n url={https://arxiv.org/abs/2310.07975}\n}" } }, { "id": "tgifqa", "name": "TGIF-QA", "symbol": "TGI", "year": 2017, "venue": "CVPR", "url": "https://arxiv.org/abs/1704.04497", "family": "Video & Temporal", "task_family": "VideoQA", "domain": "General", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Early video QA benchmark with multiple question types (frame QA, repetition count, state transition, and action).", "active_modalities": [ "Text", "Video" ], "active_capabilities": [ "Perception", "Reasoning" ], "cap_count": 2, "cap_score": 0.2222, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "tgifqa2017", "citation": { "title": "TGIF-QA: Toward Spatio-Temporal Reasoning in Visual Question Answering", "authors": "Jang, Yunseok and others", "year": 2017, "venue": "CVPR", "bibtex": "@inproceedings{tgifqa2017,\n title={TGIF-QA: Toward Spatio-Temporal Reasoning in Visual Question Answering},\n author={Jang, Yunseok and others},\n booktitle={CVPR},\n year={2017},\n url={https://arxiv.org/abs/1704.04497}\n}" } }, { "id": "tvqa", "name": "TVQA", "symbol": "TVQ", "year": 2018, "venue": "EMNLP", "url": "https://arxiv.org/abs/1809.01696", "family": "Video & Temporal", "task_family": "VideoQA", "domain": "General", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Multiple-choice QA benchmark on TV show videos with aligned subtitles; tests joint reasoning over visual frames and dialogue text.", "active_modalities": [ "Text", "Video" ], "active_capabilities": [ "Perception", "Reasoning" ], "cap_count": 2, "cap_score": 0.2222, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "tvqa2018", "citation": { "title": "TVQA: Localized, Compositional Video Question Answering", "authors": "Lei, Jie and others", "year": 2018, "venue": "EMNLP", "bibtex": "@inproceedings{tvqa2018,\n title={TVQA: Localized, Compositional Video Question Answering},\n author={Lei, Jie and others},\n booktitle={EMNLP},\n year={2018},\n url={https://arxiv.org/abs/1809.01696}\n}" } }, { "id": "activitynetqa", "name": "ActivityNet-QA", "symbol": "ACT", "year": 2019, "venue": "AAAI", "url": "https://arxiv.org/abs/1906.02467", "family": "Video & Temporal", "task_family": "VideoQA", "domain": "General", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Open-ended QA benchmark built on ActivityNet videos.", "active_modalities": [ "Text", "Video" ], "active_capabilities": [ "Perception", "Reasoning", "Long-Horizon Memory" ], "cap_count": 3, "cap_score": 0.3333, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "activitynetqa2019", "citation": { "title": "ActivityNet-QA: A Dataset for Understanding Complex Web Videos", "authors": "Yu, Zhou and others", "year": 2019, "venue": "AAAI", "bibtex": "@inproceedings{activitynetqa2019,\n title={ActivityNet-QA: A Dataset for Understanding Complex Web Videos},\n author={Yu, Zhou and others},\n booktitle={AAAI},\n year={2019},\n url={https://arxiv.org/abs/1906.02467}\n}" } }, { "id": "nextqa", "name": "NExT-QA", "symbol": "NEX", "year": 2021, "venue": "CVPR", "url": "https://arxiv.org/abs/2105.08276", "family": "Video & Temporal", "task_family": "VideoQA", "domain": "General", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Video QA benchmark emphasizing causal and temporal reasoning about complex events in open-domain videos.", "active_modalities": [ "Text", "Video" ], "active_capabilities": [ "Perception", "Reasoning", "Long-Horizon Memory" ], "cap_count": 3, "cap_score": 0.3333, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "nextqa2021", "citation": { "title": "NExT-QA: Next Phase of Question-Answering to Explaining Temporal Actions", "authors": "Xiao, Fanyi and others", "year": 2021, "venue": "CVPR", "bibtex": "@inproceedings{nextqa2021,\n title={NExT-QA: Next Phase of Question-Answering to Explaining Temporal Actions},\n author={Xiao, Fanyi and others},\n booktitle={CVPR},\n year={2021},\n url={https://arxiv.org/abs/2105.08276}\n}" } }, { "id": "avsd", "name": "AVSD", "symbol": "AVS", "year": 2019, "venue": "CVPR", "url": "https://arxiv.org/abs/1901.09107", "family": "Audio & Speech", "task_family": "Dialog", "domain": "General", "metrics": "BLEU;METEOR;CIDEr", "judge_type": "auto", "protocol_notes": "Audio-Visual Scene-aware Dialog benchmark with multi-turn QA about videos; evaluation uses standard dialog generation metrics.", "active_modalities": [ "Text", "Audio", "Video" ], "active_capabilities": [ "Perception", "Reasoning", "Instruction Following", "Long-Horizon Memory" ], "cap_count": 4, "cap_score": 0.4444, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "avsd2019", "citation": { "title": "Audio-Visual Scene-Aware Dialog", "authors": "Alamri, Huda and others", "year": 2019, "venue": "CVPR", "bibtex": "@inproceedings{avsd2019,\n title={Audio-Visual Scene-Aware Dialog},\n author={Alamri, Huda and others},\n booktitle={CVPR},\n year={2019},\n url={https://arxiv.org/abs/1901.09107}\n}" } }, { "id": "sqa3d", "name": "SQA3D", "symbol": "SQ3", "year": 2023, "venue": "ICLR", "url": "https://arxiv.org/abs/2210.07474", "family": "3D & Spatial", "task_family": "3DQA", "domain": "3D", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "3D spatial QA benchmark querying objects in ScanNet 3D scenes.", "active_modalities": [ "Text", "3D" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning" ], "cap_count": 3, "cap_score": 0.3333, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "sqa3d2023", "citation": { "title": "SQA3D: Situated Question Answering in 3D Scenes", "authors": "Ma, Xiaojian and Yong, Silong and Zheng, Zilong and Li, Qing and Liang, Yitao and Zhu, Song-Chun and Huang, Siyuan", "year": 2023, "venue": "International Conference on Learning Representations (ICLR)", "bibtex": "@inproceedings{sqa3d2023,\n title={SQA3D: Situated Question Answering in 3D Scenes},\n author={Ma, Xiaojian and Yong, Silong and Zheng, Zilong and Li, Qing and Liang, Yitao and Zhu, Song-Chun and Huang, Siyuan},\n booktitle={International Conference on Learning Representations (ICLR)},\n year={2023},\n url={https://arxiv.org/abs/2210.07474}\n}" } }, { "id": "embodiedqa", "name": "EmbodiedQA", "symbol": "EMB", "year": 2018, "venue": "CVPR", "url": "https://arxiv.org/abs/1711.11543", "family": "Embodied & Agents", "task_family": "EmbodiedQA", "domain": "Embodied", "metrics": "Accuracy;SuccessRate", "judge_type": "auto", "protocol_notes": "Embodied question answering benchmark where an agent navigates a 3D environment to gather observations before answering natural language questions.", "active_modalities": [ "Text", "Video", "3D", "Sensor", "Action" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following", "Long-Horizon Memory", "Planning", "Embodiment" ], "cap_count": 7, "cap_score": 0.7778, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "embodiedqa2018", "citation": { "title": "Embodied Question Answering", "authors": "Das, Abhishek and others", "year": 2018, "venue": "CVPR", "bibtex": "@inproceedings{embodiedqa2018,\n title={Embodied Question Answering},\n author={Das, Abhishek and others},\n booktitle={CVPR},\n year={2018},\n url={https://arxiv.org/abs/1711.11543}\n}" } }, { "id": "r2r", "name": "R2R-RoomToRoom", "symbol": "R2R", "year": 2018, "venue": "CVPR", "url": "https://arxiv.org/abs/1711.07280", "family": "Embodied & Agents", "task_family": "Navigation", "domain": "Indoor", "metrics": "SuccessRate;SPL", "judge_type": "auto", "protocol_notes": "Vision-and-language navigation benchmark with panoramic indoor images and natural language instructions; metrics include success rate and path length.", "active_modalities": [ "Text", "Image", "Sensor", "Action" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following", "Long-Horizon Memory", "Planning", "Embodiment" ], "cap_count": 7, "cap_score": 0.7778, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "r2r2018", "citation": { "title": "Vision-and-Language Navigation: Interpreting Visually-Grounded Navigation Instructions in Real Environments", "authors": "Anderson, Peter and others", "year": 2018, "venue": "CVPR", "bibtex": "@inproceedings{r2r2018,\n title={Vision-and-Language Navigation: Interpreting Visually-Grounded Navigation Instructions in Real Environments},\n author={Anderson, Peter and others},\n booktitle={CVPR},\n year={2018},\n url={https://arxiv.org/abs/1711.07280}\n}" } }, { "id": "scienceqa", "name": "ScienceQA", "symbol": "SQA", "year": 2022, "venue": "NeurIPS", "url": "https://arxiv.org/abs/2209.09513", "family": "Vision-Language", "task_family": "ExamQA", "domain": "STEM", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Multimodal science QA benchmark with images, diagrams, and textual context.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Domain Knowledge" ], "cap_count": 4, "cap_score": 0.4444, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "scienceqa2022", "citation": { "title": "ScienceQA: A Multi-Modal Dataset for Question Answering in Science", "authors": "Lu, Pan and others", "year": 2022, "venue": "NeurIPS", "bibtex": "@article{scienceqa2022,\n title={ScienceQA: A Multi-Modal Dataset for Question Answering in Science},\n author={Lu, Pan and others},\n journal={NeurIPS},\n year={2022},\n url={https://arxiv.org/abs/2209.09513}\n}" } }, { "id": "docvqa2021", "name": "DocVQA", "symbol": "DVQ", "year": 2021, "venue": "WACV", "url": "https://arxiv.org/abs/2007.00398", "family": "Doc & Diagrams", "task_family": "DocQA", "domain": "DocUnderstanding", "metrics": "Accuracy;F1", "judge_type": "auto", "protocol_notes": "Document visual QA benchmark on 12k+ scanned forms and invoices; automatic evaluation reports accuracy and token-level F1 against human answers.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Domain Knowledge" ], "cap_count": 4, "cap_score": 0.4444, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "docvqa2021", "citation": { "title": "DocVQA: A Dataset for Document Visual Question Answering", "authors": "Mathew, Minesh and others", "year": 2021, "venue": "WACV", "bibtex": "@inproceedings{docvqa2021,\n title={DocVQA: A Dataset for Document Visual Question Answering},\n author={Mathew, Minesh and others},\n booktitle={WACV},\n year={2021},\n url={https://arxiv.org/abs/2007.00398}\n}" } }, { "id": "msrvttqa", "name": "MSRVTT-QA", "symbol": "MSR", "year": 2017, "venue": "ACM MM", "url": "https://dl.acm.org/doi/10.1145/3123266.3123427", "family": "Video & Temporal", "task_family": "VideoQA", "domain": "General", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Video QA benchmark built on MSR-VTT clips; evaluates recognition and reasoning over short open-domain videos via question answering.", "active_modalities": [ "Text", "Video" ], "active_capabilities": [ "Perception", "Reasoning" ], "cap_count": 2, "cap_score": 0.2222, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "msrvttqa2017", "citation": { "title": "MSRVTT-QA: Visual Question Answering on Multi-Modal Video", "authors": "Xu, Jun and others", "year": 2017, "venue": "ACM MM", "bibtex": "@inproceedings{msrvttqa2017,\n title={MSRVTT-QA: Visual Question Answering on Multi-Modal Video},\n author={Xu, Jun and others},\n booktitle={ACM MM},\n year={2017},\n url={https://dl.acm.org/doi/10.1145/3123266.3123427}\n}" } }, { "id": "ego4dqa", "name": "Ego4D-QA", "symbol": "EG2", "year": 2022, "venue": "CVPR", "url": "https://arxiv.org/abs/2110.07058", "family": "Video & Temporal", "task_family": "VideoQA", "domain": "Egocentric", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Egocentric video QA tasks from the Ego4D suite with long first-person videos and temporally grounded questions.", "active_modalities": [ "Text", "Video" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Long-Horizon Memory" ], "cap_count": 4, "cap_score": 0.4444, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "ego4dqa2022", "citation": { "title": "Ego4D: Egocentric Video Dataset and Benchmark Suite", "authors": "Grauman, Kristen and others", "year": 2022, "venue": "CVPR", "bibtex": "@inproceedings{ego4dqa2022,\n title={Ego4D: Egocentric Video Dataset and Benchmark Suite},\n author={Grauman, Kristen and others},\n booktitle={CVPR},\n year={2022},\n url={https://arxiv.org/abs/2110.07058}\n}" } }, { "id": "musicavqa2022", "name": "Music-AVQA", "symbol": "MUS", "year": 2022, "venue": "CVPR", "url": "https://arxiv.org/abs/2203.00243", "family": "Audio & Speech", "task_family": "AVQA", "domain": "Music", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Audio-visual question answering dataset with 9k musical performance videos and diverse question types; evaluated via automatic accuracy on multiple-choice answers.", "active_modalities": [ "Text", "Image", "Audio" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning" ], "cap_count": 3, "cap_score": 0.3333, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "musicavqa2022", "citation": { "title": "Music-AVQA: A Benchmark for Music-Aware Audio-Visual Question Answering", "authors": "Li, Xinyu and others", "year": 2022, "venue": "CVPR", "bibtex": "@inproceedings{musicavqa2022,\n title={Music-AVQA: A Benchmark for Music-Aware Audio-Visual Question Answering},\n author={Li, Xinyu and others},\n booktitle={CVPR},\n year={2022},\n url={https://arxiv.org/abs/2203.00243}\n}" } }, { "id": "movieqa2016", "name": "MovieQA", "symbol": "MOV", "year": 2016, "venue": "CVPR", "url": "https://arxiv.org/abs/1512.06900", "family": "Video & Temporal", "task_family": "VideoQA", "domain": "Entertainment", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Story-driven movie question answering benchmark spanning scripts, plots, and videos.", "active_modalities": [ "Text", "Video" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Domain Knowledge", "Long-Horizon Memory" ], "cap_count": 5, "cap_score": 0.5556, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "movieqa2016", "citation": { "title": "MovieQA: Understanding Stories in Movies through Question-Answering", "authors": "Tapaswi, Makarand and others", "year": 2016, "venue": "CVPR", "bibtex": "@inproceedings{movieqa2016,\n title={MovieQA: Understanding Stories in Movies through Question-Answering},\n author={Tapaswi, Makarand and others},\n booktitle={CVPR},\n year={2016},\n url={https://arxiv.org/abs/1512.06900}\n}" } }, { "id": "pororoqa2017", "name": "PororoQA", "symbol": "POR", "year": 2017, "venue": "IJCAI", "url": "https://arxiv.org/abs/1709.06145", "family": "Video & Temporal", "task_family": "VideoQA", "domain": "Kids", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Children's cartoon QA benchmark with annotated dialogues and video summaries; evaluated via automatic multiple-choice accuracy across episode-level questions.", "active_modalities": [ "Text", "Video" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Long-Horizon Memory" ], "cap_count": 4, "cap_score": 0.4444, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "pororoqa2017", "citation": { "title": "DeepStory: Video Story QA by Deep Embedded Memory Networks", "authors": "Kim, Kyung-Min and Heo, Min-Oh and Choi, Seong-Ho and Zhang, Byoung-Tak", "year": 2017, "venue": "Proceedings of the International Joint Conference on Artificial Intelligence (IJCAI)", "bibtex": "@inproceedings{pororoqa2017,\n title={DeepStory: Video Story QA by Deep Embedded Memory Networks},\n author={Kim, Kyung-Min and Heo, Min-Oh and Choi, Seong-Ho and Zhang, Byoung-Tak},\n booktitle={Proceedings of the International Joint Conference on Artificial Intelligence (IJCAI)},\n year={2017},\n url={https://arxiv.org/abs/1707.00836}\n}" } }, { "id": "apibank2023", "name": "API-Bank", "symbol": "API", "year": 2023, "venue": "arXiv", "url": "https://arxiv.org/abs/2304.08244", "family": "Vision-Language", "task_family": "ToolUse", "domain": "General", "metrics": "Accuracy", "judge_type": "LLM", "protocol_notes": "Tool-use benchmark covering 53 everyday APIs with multi-step reasoning traces; automatic evaluator checks whether generated tool calls satisfy the goal.", "active_modalities": [ "Text" ], "active_capabilities": [ "Perception", "Reasoning", "Tool Use" ], "cap_count": 3, "cap_score": 0.3333, "cost_tier": 2, "cost_tier_label": "Tier 2: $50-$150 (Multi-Turn / API)", "paper_citation_key": "apibank2023", "citation": { "title": "API-Bank", "authors": "Minghao Li and Feifan Song and Bowen Yu and Haiyang Yu and Zhoujun Li and Fei Huang and Yongbin Li", "year": 2023, "venue": "Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing (EMNLP)", "bibtex": "@inproceedings{apibank2023,\n title={{API-Bank}: A Comprehensive Benchmark for Tool-Augmented {LLMs}},\n author={Minghao Li and Feifan Song and Bowen Yu and Haiyang Yu and Zhoujun Li and Fei Huang and Yongbin Li},\n booktitle={Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing (EMNLP)},\n year={2023},\n howpublished={\\url{https://arxiv.org/abs/2304.08244}},\n}" } }, { "id": "talk2nav2020", "name": "Talk2Nav", "symbol": "TAL", "year": 2020, "venue": "IJCV", "url": "https://arxiv.org/abs/2006.07328", "family": "Embodied & Agents", "task_family": "Navigation", "domain": "Embodied", "metrics": "SuccessRate", "judge_type": "auto", "protocol_notes": "Speech-driven navigation dataset where agents follow spoken instructions through photorealistic environments; evaluated via automatic goal success and path length metrics.", "active_modalities": [ "Audio", "Sensor", "Action" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following", "Long-Horizon Memory", "Planning", "Embodiment" ], "cap_count": 7, "cap_score": 0.7778, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "talk2nav2020", "citation": { "title": "Talk2Nav: Long-Range Vision-and-Language Navigation with Dual Attention and Spatial Memory", "authors": "Vasudevan, Arun Balajee and Dai, Dengxin and Van Gool, Luc", "year": 2021, "venue": "International Journal of Computer Vision (IJCV)", "bibtex": "@article{talk2nav2020,\n title={Talk2Nav: Long-Range Vision-and-Language Navigation with Dual Attention and Spatial Memory},\n author={Vasudevan, Arun Balajee and Dai, Dengxin and Van Gool, Luc},\n journal={International Journal of Computer Vision (IJCV)},\n year={2021},\n url={https://arxiv.org/abs/1910.02029}\n}" } }, { "id": "clevrer2020", "name": "CLEVRER", "symbol": "CLE", "year": 2020, "venue": "ICLR", "url": "https://arxiv.org/abs/1910.01442", "family": "Video & Temporal", "task_family": "VideoQA", "domain": "Synthetic", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Synthetic physics reasoning video benchmark with descriptive questions about collisions, causes, and counterfactuals; used to probe temporal and causal reasoning.", "active_modalities": [ "Text", "Video" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Domain Knowledge", "Long-Horizon Memory" ], "cap_count": 5, "cap_score": 0.5556, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "clevrer2020", "citation": { "title": "CLEVRER", "authors": "Kexin Yi and Chuang Gan and Yunzhu Li and Pushmeet Kohli and Jiajun Wu and Antonio Torralba and Joshua B. Tenenbaum", "year": 2020, "venue": "International Conference on Learning Representations (ICLR)", "bibtex": "@inproceedings{clevrer2020,\n title={{CLEVRER}: Collision Events for Video Representation and Reasoning},\n author={Kexin Yi and Chuang Gan and Yunzhu Li and Pushmeet Kohli and Jiajun Wu and Antonio Torralba and Joshua B. Tenenbaum},\n booktitle={International Conference on Learning Representations (ICLR)},\n year={2020},\n howpublished={\\url{https://arxiv.org/abs/1910.01442}},\n}" } }, { "id": "clotho", "name": "Clotho-AudioCaption", "symbol": "CLO", "year": 2020, "venue": "ICASSP", "url": "https://arxiv.org/abs/1910.09387", "family": "Audio & Speech", "task_family": "Captioning", "domain": "General", "metrics": "BLEU;METEOR;CIDEr", "judge_type": "auto", "protocol_notes": "Audio captioning benchmark with natural language descriptions for diverse environmental sounds; evaluated with standard captioning metrics.", "active_modalities": [ "Text", "Audio" ], "active_capabilities": [ "Perception" ], "cap_count": 1, "cap_score": 0.1111, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "clotho2020", "citation": { "title": "Clotho: An Audio Captioning Dataset", "authors": "Konstantinos Drossos and Samuel Lipping and Tuomas Virtanen", "year": 2020, "venue": "IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)", "bibtex": "@inproceedings{clotho2020,\n title={Clotho: An Audio Captioning Dataset},\n author={Konstantinos Drossos and Samuel Lipping and Tuomas Virtanen},\n booktitle={IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},\n year={2020},\n howpublished={\\url{https://arxiv.org/abs/1910.09387}},\n}" } }, { "id": "vlnce", "name": "VLN-CE", "symbol": "VLN", "year": 2020, "venue": "CVPR", "url": "https://arxiv.org/abs/2005.09323", "family": "Embodied & Agents", "task_family": "Navigation", "domain": "Indoor", "metrics": "SuccessRate;SPL", "judge_type": "auto", "protocol_notes": "Vision-language navigation in continuous environments benchmark using realistic simulation; measures success and path efficiency for embodied agents.", "active_modalities": [ "Text", "Image", "Sensor", "Action" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following", "Long-Horizon Memory", "Planning", "Embodiment" ], "cap_count": 7, "cap_score": 0.7778, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "vlnce2020", "citation": { "title": "Beyond the Nav-Graph: Vision-and-Language Navigation in Continuous Environments", "authors": "Jacob Krantz and Erik Wijmans and Arjun Majumdar and Dhruv Batra and Stefan Lee", "year": 2020, "venue": "European Conference on Computer Vision (ECCV)", "bibtex": "@inproceedings{vlnce2020,\n title={Beyond the Nav-Graph: Vision-and-Language Navigation in Continuous Environments},\n author={Jacob Krantz and Erik Wijmans and Arjun Majumdar and Dhruv Batra and Stefan Lee},\n booktitle={European Conference on Computer Vision (ECCV)},\n year={2020},\n howpublished={\\url{https://arxiv.org/abs/2005.09323}},\n}" } }, { "id": "touchdown", "name": "Touchdown", "symbol": "TOU", "year": 2019, "venue": "CVPR", "url": "https://arxiv.org/abs/1811.12354", "family": "Embodied & Agents", "task_family": "Navigation", "domain": "Urban", "metrics": "SuccessRate;SPL", "judge_type": "auto", "protocol_notes": "Urban vision-language navigation benchmark on street-view imagery with long natural language instructions.", "active_modalities": [ "Text", "Image", "Sensor", "Action" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following", "Long-Horizon Memory", "Planning", "Embodiment" ], "cap_count": 7, "cap_score": 0.7778, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "touchdown2019", "citation": { "title": "Touchdown: Natural Language Navigation and Spatial Reasoning in Visual Street Environments", "authors": "Howard Chen and Alane Suhr and Kemikeme Mkih and Noah Snavely and Yoav Artzi", "year": 2019, "venue": "IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)", "bibtex": "@inproceedings{touchdown2019,\n title={Touchdown: Natural Language Navigation and Spatial Reasoning in Visual Street Environments},\n author={Howard Chen and Alane Suhr and Kemikeme Mkih and Noah Snavely and Yoav Artzi},\n booktitle={IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},\n year={2019},\n howpublished={\\url{https://arxiv.org/abs/1811.12354}},\n}" } }, { "id": "infographicvqa", "name": "InfographicVQA", "symbol": "INF", "year": 2021, "venue": "CVPR", "url": "https://arxiv.org/abs/2104.06490", "family": "Vision-Language", "task_family": "VQA", "domain": "DataViz", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Visual QA benchmark on infographics combining text, icons, and diagrams.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning" ], "cap_count": 3, "cap_score": 0.3333, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "infographicvqa2021", "citation": { "title": "InfographicVQA", "authors": "Mathew, Minesh and others", "year": 2021, "venue": "CVPR", "bibtex": "@inproceedings{infographicvqa2021,\n title={InfographicVQA},\n author={Mathew, Minesh and others},\n booktitle={CVPR},\n year={2021},\n url={https://arxiv.org/abs/2104.06490}\n}" } }, { "id": "vqav2", "name": "VQAv2", "symbol": "VQA", "year": 2016, "venue": "arXiv", "url": "https://arxiv.org/abs/1612.00837", "family": "Vision-Language", "task_family": "VQA", "domain": "General", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Balanced visual QA benchmark with multiple-choice and open-ended answers; widely reused for evaluating vision-language models and LVLMs.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning" ], "cap_count": 3, "cap_score": 0.3333, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "vqav22016", "citation": { "title": "VQA", "authors": "Goyal, Yash and others", "year": 2016, "venue": "arXiv preprint arXiv:1612.00837", "bibtex": "@article{vqav22016,\n title={{VQA} v2.0: Towards Reducing Language Bias in Visual Question Answering},\n author={Goyal, Yash and others},\n journal={arXiv preprint arXiv:1612.00837},\n year={2016},\n url={https://arxiv.org/abs/1612.00837}\n}" } }, { "id": "gqa", "name": "GQA", "symbol": "GQA", "year": 2019, "venue": "arXiv", "url": "https://arxiv.org/abs/1902.09506", "family": "Vision-Language", "task_family": "VQA", "domain": "General", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Compositional visual reasoning benchmark grounded in scene graphs; tests fine-grained grounding and multi-step reasoning.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning" ], "cap_count": 3, "cap_score": 0.3333, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "gqa2019", "citation": { "title": "GQA: A New Dataset for Real-World Visual Reasoning and Compositional Question Answering", "authors": "Hudson, Drew and Manning, Christopher D.", "year": 2019, "venue": "CVPR", "bibtex": "@article{gqa2019,\n title={GQA: A New Dataset for Real-World Visual Reasoning and Compositional Question Answering},\n author={Hudson, Drew and Manning, Christopher D.},\n journal={CVPR},\n year={2019},\n url={https://arxiv.org/abs/1902.09506}\n}" } }, { "id": "vizwizvqa", "name": "VizWiz-VQA", "symbol": "VIZ", "year": 2018, "venue": "arXiv", "url": "https://arxiv.org/abs/1802.08218", "family": "Vision-Language", "task_family": "VQA", "domain": "Accessibility", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Visual QA benchmark built from images captured by blind people; emphasizes robust perception and grounding under challenging conditions.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding" ], "cap_count": 2, "cap_score": 0.2222, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "vizwizvqa2018", "citation": { "title": "VizWiz Grand Challenge: Answering Visual Questions from Blind People", "authors": "Gurari, Danna and others", "year": 2018, "venue": "CVPR", "bibtex": "@article{vizwizvqa2018,\n title={VizWiz Grand Challenge: Answering Visual Questions from Blind People},\n author={Gurari, Danna and others},\n journal={CVPR},\n year={2018},\n url={https://arxiv.org/abs/1802.08218}\n}" } }, { "id": "textvqa", "name": "TextVQA", "symbol": "TEX", "year": 2019, "venue": "arXiv", "url": "https://arxiv.org/abs/1904.08920", "family": "Vision-Language", "task_family": "TextVQA", "domain": "General", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Visual QA benchmark requiring reading text in the image; probes OCR-augmented multimodal reasoning.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning" ], "cap_count": 3, "cap_score": 0.3333, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "textvqa2019", "citation": { "title": "Towards VQA Models That Can Read", "authors": "Singh, Amanpreet and Natarajan, Vivek and Shah, Meet and Jiang, Yu and Chen, Xinlei and Batra, Dhruv and Parikh, Devi and Rohrbach, Marcus", "year": 2019, "venue": "Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)", "bibtex": "@inproceedings{textvqa2019,\n title={Towards VQA Models That Can Read},\n author={Singh, Amanpreet and Natarajan, Vivek and Shah, Meet and Jiang, Yu and Chen, Xinlei and Batra, Dhruv and Parikh, Devi and Rohrbach, Marcus},\n booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},\n year={2019},\n url={https://arxiv.org/abs/1904.08920}\n}" } }, { "id": "okvqa", "name": "OK-VQA", "symbol": "OKV", "year": 2019, "venue": "CVPR", "url": "https://arxiv.org/abs/1906.00067", "family": "Vision-Language", "task_family": "VQA", "domain": "General", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Open-domain knowledge VQA benchmark where answering correctly typically requires external world knowledge beyond the image.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Domain Knowledge" ], "cap_count": 4, "cap_score": 0.4444, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "okvqa2019", "citation": { "title": "OK-VQA", "authors": "Marino, Kenneth and others", "year": 2019, "venue": "CVPR", "bibtex": "@inproceedings{okvqa2019,\n title={OK-{VQA}: A Visual Question Answering Benchmark Requiring External Knowledge},\n author={Marino, Kenneth and others},\n booktitle={CVPR},\n year={2019},\n url={https://arxiv.org/abs/1906.00067}\n}" } }, { "id": "aokvqa", "name": "A-OKVQA", "symbol": "AOK", "year": 2022, "venue": "ECCV", "url": "https://arxiv.org/abs/2206.01718", "family": "Vision-Language", "task_family": "VQA", "domain": "General", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Augmented OK-VQA with richer answer types and rationales; used to benchmark commonsense and knowledge-intensive visual QA for LVLMs.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Domain Knowledge" ], "cap_count": 4, "cap_score": 0.4444, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "aokvqa2022", "citation": { "title": "A-OKVQA: A Benchmark for Visual Question Answering Using World Knowledge", "authors": "Schwenk, Dustin and others", "year": 2022, "venue": "ECCV", "bibtex": "@inproceedings{aokvqa2022,\n title={A-OKVQA: A Benchmark for Visual Question Answering Using World Knowledge},\n author={Schwenk, Dustin and others},\n booktitle={ECCV},\n year={2022},\n url={https://arxiv.org/abs/2206.01718}\n}" } }, { "id": "textcaps", "name": "TextCaps", "symbol": "TE2", "year": 2020, "venue": "ECCV", "url": "https://arxiv.org/abs/2003.12462", "family": "Vision-Language", "task_family": "Captioning", "domain": "General", "metrics": "BLEU;CIDEr", "judge_type": "auto", "protocol_notes": "Image captioning benchmark that focuses on reading and describing text in images.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding" ], "cap_count": 2, "cap_score": 0.2222, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "textcaps2020", "citation": { "title": "TextCaps: A Dataset for Image Captioning with Reading Comprehension", "authors": "Sidorov, Oleksii and others", "year": 2020, "venue": "ECCV", "bibtex": "@inproceedings{textcaps2020,\n title={TextCaps: A Dataset for Image Captioning with Reading Comprehension},\n author={Sidorov, Oleksii and others},\n booktitle={ECCV},\n year={2020},\n url={https://arxiv.org/abs/2003.12462}\n}" } }, { "id": "iconqa", "name": "IconQA", "symbol": "ICO", "year": 2021, "venue": "NeurIPS", "url": "https://arxiv.org/abs/2110.13214", "family": "Vision-Language", "task_family": "ExamQA", "domain": "STEM", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Diagram and icon-based science QA benchmark targeting multimodal reasoning over textbook-style figures and charts.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Domain Knowledge" ], "cap_count": 4, "cap_score": 0.4444, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "iconqa2021", "citation": { "title": "ICONQA: A New Benchmark for Abstract Diagram Understanding and Reasoning", "authors": "Lu, Pan and others", "year": 2021, "venue": "NeurIPS", "bibtex": "@inproceedings{iconqa2021,\n title={ICONQA: A New Benchmark for Abstract Diagram Understanding and Reasoning},\n author={Lu, Pan and others},\n booktitle={NeurIPS},\n year={2021},\n url={https://arxiv.org/abs/2110.13214}\n}" } }, { "id": "audiocaps", "name": "AudioCaps", "symbol": "AUD", "year": 2019, "venue": "NAACL", "url": "https://aclanthology.org/N19-1011/", "family": "Audio & Speech", "task_family": "Captioning", "domain": "General", "metrics": "BLEU;METEOR;CIDEr", "judge_type": "auto", "protocol_notes": "Audio captioning benchmark with human-written captions for audio clips; reused for audio-language model evaluation.", "active_modalities": [ "Text", "Audio" ], "active_capabilities": [ "Perception" ], "cap_count": 1, "cap_score": 0.1111, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "audiocaps2019", "citation": { "title": "AudioCaps: Generating Captions for Audio in The Wild", "authors": "Kim, Chanwoo and others", "year": 2019, "venue": "NAACL", "bibtex": "@inproceedings{audiocaps2019,\n title={AudioCaps: Generating Captions for Audio in The Wild},\n author={Kim, Chanwoo and others},\n booktitle={NAACL},\n year={2019},\n url={https://aclanthology.org/N19-1011/}\n}" } }, { "id": "referit3d", "name": "ReferIt3D", "symbol": "REF", "year": 2020, "venue": "ECCV", "url": "https://arxiv.org/abs/2003.10454", "family": "3D & Spatial", "task_family": "Grounding", "domain": "3D", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "3D referring expression benchmark that localizes objects in point clouds or meshes based on natural language descriptions.", "active_modalities": [ "Text", "Image", "3D" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning" ], "cap_count": 3, "cap_score": 0.3333, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "referit3d2020", "citation": { "title": "ReferIt3D: Neural Listeners for Fine-Grained 3D", "authors": "Achlioptas, Panos and others", "year": 2020, "venue": "ECCV", "bibtex": "@inproceedings{referit3d2020,\n title={ReferIt3D: Neural Listeners for Fine-Grained {3D} Object Identification in Real-World Scenes},\n author={Achlioptas, Panos and others},\n booktitle={ECCV},\n year={2020},\n url={https://arxiv.org/abs/2003.10454}\n}" } }, { "id": "scanrefer", "name": "ScanRefer", "symbol": "SC2", "year": 2020, "venue": "ECCV", "url": "https://arxiv.org/abs/1912.06231", "family": "3D & Spatial", "task_family": "Grounding", "domain": "3D", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "3D language grounding benchmark aligning textual descriptions with target objects in 3D scans.", "active_modalities": [ "Text", "3D" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning" ], "cap_count": 3, "cap_score": 0.3333, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "scanrefer2020", "citation": { "title": "ScanRefer", "authors": "Dave Zhenyu Chen and Angel X. Chang and Matthias Nie\\ss", "year": 2020, "venue": "European Conference on Computer Vision (ECCV)", "bibtex": "@inproceedings{scanrefer2020,\n title={{ScanRefer}: 3D Object Localization in {RGB-D} Scans using Natural Language},\n author={Dave Zhenyu Chen and Angel X. Chang and Matthias Nie{\\ss}ner},\n booktitle={European Conference on Computer Vision (ECCV)},\n year={2020},\n howpublished={\\url{https://arxiv.org/abs/1912.06231}},\n}" } }, { "id": "mscoco", "name": "MSCOCO-Caption", "symbol": "MSC", "year": 2015, "venue": "arXiv", "url": "https://arxiv.org/abs/1504.00325", "family": "Vision-Language", "task_family": "Captioning", "domain": "General", "metrics": "BLEU;CIDEr", "judge_type": "auto", "protocol_notes": "Image captioning benchmark based on MS COCO; widely reused to evaluate captioning and LVLM generative quality.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding" ], "cap_count": 2, "cap_score": 0.2222, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "mscoco2015", "citation": { "title": "Microsoft COCO", "authors": "Xinlei Chen and Hao Fang and Tsung-Yi Lin and Ramakrishna Vedantam and Saurabh Gupta and Piotr Doll'a", "year": 2015, "venue": "arXiv preprint arXiv:1504.00325", "bibtex": "@article{mscoco2015,\n title={Microsoft {COCO} Captions: Data Collection and Evaluation Server},\n author={Xinlei Chen and Hao Fang and Tsung-Yi Lin and Ramakrishna Vedantam and Saurabh Gupta and Piotr Doll{'a}r and C. Lawrence Zitnick},\n journal={arXiv preprint arXiv:1504.00325},\n year={2015},\n howpublished={\\url{https://arxiv.org/abs/1504.00325}},\n}" } }, { "id": "nocaps", "name": "NoCaps", "symbol": "NOC", "year": 2019, "venue": "ICCV", "url": "https://arxiv.org/abs/1812.08658", "family": "Vision-Language", "task_family": "Captioning", "domain": "General", "metrics": "BLEU;CIDEr", "judge_type": "auto", "protocol_notes": "Captioning benchmark evaluating generalization to novel objects beyond training categories; reused for LVLM captioning.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding" ], "cap_count": 2, "cap_score": 0.2222, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "nocaps2019", "citation": { "title": "nocaps: novel object captioning at scale", "authors": "Harsh Agrawal and Karan Desai and Yufei Wang and Xiaojuan Wang and Rishabh Jain and Vladan Petrovic and Stefan Lee and Dhruv Batra and Devi Parikh", "year": 2019, "venue": "IEEE/CVF International Conference on Computer Vision (ICCV)", "bibtex": "@inproceedings{nocaps2019,\n title={nocaps: novel object captioning at scale},\n author={Harsh Agrawal and Karan Desai and Yufei Wang and Xiaojuan Wang and Rishabh Jain and Vladan Petrovic and Stefan Lee and Dhruv Batra and Devi Parikh},\n booktitle={IEEE/CVF International Conference on Computer Vision (ICCV)},\n year={2019},\n howpublished={\\url{https://arxiv.org/abs/1812.08658}},\n}" } }, { "id": "flickr30k", "name": "Flickr30k-Entities", "symbol": "FLI", "year": 2015, "venue": "arXiv", "url": "https://arxiv.org/abs/1505.04870", "family": "Vision-Language", "task_family": "Grounding", "domain": "General", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Grounding benchmark linking caption phrases to image regions; used to evaluate region-level visual grounding.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning" ], "cap_count": 3, "cap_score": 0.3333, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "flickr30k2015", "citation": { "title": "Flickr30k", "authors": "Bryan A. Plummer and Liwei Wang and Christopher M. Cervantes and Juan C. Caicedo and Julia Hockenmaier and Svetlana Lazebnik", "year": 2015, "venue": "IEEE International Conference on Computer Vision (ICCV)", "bibtex": "@inproceedings{flickr30k2015,\n title={{Flickr30k} Entities: Collecting Region-to-Phrase Correspondences for Richer Image-to-Sentence Models},\n author={Bryan A. Plummer and Liwei Wang and Christopher M. Cervantes and Juan C. Caicedo and Julia Hockenmaier and Svetlana Lazebnik},\n booktitle={IEEE International Conference on Computer Vision (ICCV)},\n year={2015},\n howpublished={\\url{https://arxiv.org/abs/1505.04870}},\n}" } }, { "id": "refcoco", "name": "RefCOCO", "symbol": "RC1", "year": 2016, "venue": "ECCV", "url": "https://arxiv.org/abs/1603.03146", "family": "Vision-Language", "task_family": "Grounding", "domain": "General", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Referring expression comprehension benchmark on MS COCO images; evaluates grounding of language to object regions.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning" ], "cap_count": 3, "cap_score": 0.3333, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "refcoco2016", "citation": { "title": "ReferItGame: Referring to Objects in Photographs of Natural Scenes", "authors": "Kazemzadeh, Sahar and Ordonez, Vicente and Matten, Mark and Berg, Tamara L.", "year": 2014, "venue": "Proceedings of the 2014 Conference on Empirical Methods in Natural Language Processing (EMNLP)", "bibtex": "@inproceedings{refcoco2016,\n title={ReferItGame: Referring to Objects in Photographs of Natural Scenes},\n author={Kazemzadeh, Sahar and Ordonez, Vicente and Matten, Mark and Berg, Tamara L.},\n booktitle={Proceedings of the 2014 Conference on Empirical Methods in Natural Language Processing (EMNLP)},\n year={2014},\n url={https://aclanthology.org/D14-1086/}\n}" } }, { "id": "refcocoplus", "name": "RefCOCOplus", "symbol": "RE2", "year": 2016, "venue": "ECCV", "url": "https://arxiv.org/abs/1603.03146", "family": "Vision-Language", "task_family": "Grounding", "domain": "General", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Variant of RefCOCO emphasizing appearance-based referring expressions; used to stress-test fine-grained visual grounding.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning" ], "cap_count": 3, "cap_score": 0.3333, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "refcocoplus2016", "citation": { "title": "Modeling Context in Referring Expressions", "authors": "Yu, Licheng and Poirson, Patrick and Yang, Shan and Berg, Alexander C. and Berg, Tamara L.", "year": 2016, "venue": "European Conference on Computer Vision (ECCV)", "bibtex": "@inproceedings{refcocoplus2016,\n title={Modeling Context in Referring Expressions},\n author={Yu, Licheng and Poirson, Patrick and Yang, Shan and Berg, Alexander C. and Berg, Tamara L.},\n booktitle={European Conference on Computer Vision (ECCV)},\n year={2016},\n url={https://arxiv.org/abs/1608.00272}\n}" } }, { "id": "refcocog", "name": "RefCOCOg", "symbol": "RCG", "year": 2016, "venue": "ECCV", "url": "https://arxiv.org/abs/1603.03146", "family": "Vision-Language", "task_family": "Grounding", "domain": "General", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Referring expressions with longer, more descriptive phrases; commonly used for grounding and generation tasks.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning" ], "cap_count": 3, "cap_score": 0.3333, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "refcocog2016", "citation": { "title": "Generation and Comprehension of Unambiguous Object Descriptions", "authors": "Mao, Junhua and Huang, Jonathan and Toshev, Alexander and Camburu, Oana and Yuille, Alan L. and Murphy, Kevin", "year": 2016, "venue": "Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition (CVPR)", "bibtex": "@inproceedings{refcocog2016,\n title={Generation and Comprehension of Unambiguous Object Descriptions},\n author={Mao, Junhua and Huang, Jonathan and Toshev, Alexander and Camburu, Oana and Yuille, Alan L. and Murphy, Kevin},\n booktitle={Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition (CVPR)},\n year={2016},\n url={https://arxiv.org/abs/1511.02283}\n}" } }, { "id": "visual7w", "name": "Visual7W", "symbol": "VIS", "year": 2016, "venue": "CVPR", "url": "https://arxiv.org/abs/1511.03416", "family": "Vision-Language", "task_family": "VQA", "domain": "General", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Visual question answering benchmark with multiple-choice questions and localized answers; reused in LVLM evaluation.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning" ], "cap_count": 3, "cap_score": 0.3333, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "visual7w2016", "citation": { "title": "Visual7W", "authors": "Yuke Zhu and Oliver Groth and Michael Bernstein and Li Fei-Fei", "year": 2016, "venue": "IEEE Conference on Computer Vision and Pattern Recognition (CVPR)", "bibtex": "@inproceedings{visual7w2016,\n title={{Visual7W}: Grounded Question Answering in Images},\n author={Yuke Zhu and Oliver Groth and Michael Bernstein and Li Fei-Fei},\n booktitle={IEEE Conference on Computer Vision and Pattern Recognition (CVPR)},\n year={2016},\n howpublished={\\url{https://arxiv.org/abs/1511.03416}},\n}" } }, { "id": "nlvr2", "name": "NLVR2", "symbol": "NLV", "year": 2018, "venue": "ACL", "url": "https://arxiv.org/abs/1811.00491", "family": "Vision-Language", "task_family": "NLVR", "domain": "General", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Bilateral image-text reasoning benchmark for natural language visual reasoning; adopted for vision-language models.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning" ], "cap_count": 3, "cap_score": 0.3333, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "nlvr22018", "citation": { "title": "A Corpus for Reasoning About Natural Language Grounded in Photographs", "authors": "Alane Suhr and Stephanie Zhou and Allyson Lynch and Iris Zhang and Noah Snavely and Yoav Artzi", "year": 2019, "venue": "Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics (ACL)", "bibtex": "@inproceedings{nlvr22018,\n title={A Corpus for Reasoning About Natural Language Grounded in Photographs},\n author={Alane Suhr and Stephanie Zhou and Allyson Lynch and Iris Zhang and Noah Snavely and Yoav Artzi},\n booktitle={Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics (ACL)},\n year={2019},\n howpublished={\\url{https://arxiv.org/abs/1811.00491}},\n}" } }, { "id": "snlive", "name": "SNLI-VE", "symbol": "SNL", "year": 2019, "venue": "ICCV", "url": "https://arxiv.org/abs/1901.06706", "family": "Vision-Language", "task_family": "Entailment", "domain": "General", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Visual entailment benchmark extending SNLI to image-hypothesis pairs; evaluates multimodal inference.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Reasoning" ], "cap_count": 2, "cap_score": 0.2222, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "snlive2019", "citation": { "title": "Visual Entailment", "authors": "Ning Xie and Farley Lai and Derek Doran and Asim Kadav", "year": 2019, "venue": "IEEE/CVF Conference on Computer Vision and Pattern Recognition Workshops (CVPRW)", "bibtex": "@inproceedings{snlive2019,\n title={{Visual Entailment}: A Novel Task for Fine-Grained Image Understanding},\n author={Ning Xie and Farley Lai and Derek Doran and Asim Kadav},\n booktitle={IEEE/CVF Conference on Computer Vision and Pattern Recognition Workshops (CVPRW)},\n year={2019},\n howpublished={\\url{https://arxiv.org/abs/1901.06706}},\n}" } }, { "id": "slake2021", "name": "SLAKE", "symbol": "SLA", "year": 2021, "venue": "ISBI", "url": "https://arxiv.org/abs/2102.09542", "family": "Vision-Language", "task_family": "MedVQA", "domain": "Medical", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Bilingual medical VQA benchmark with 14k question-answer pairs and a physician-annotated knowledge base; evaluated via automatic exact-match accuracy.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Domain Knowledge" ], "cap_count": 4, "cap_score": 0.4444, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "slake2021", "citation": { "title": "SLAKE", "authors": "Bo Liu and Li-Ming Zhan and Li Xu and Lin Ma and Yan Shen and Xiao-Ming Wu", "year": 2021, "venue": "IEEE 18th International Symposium on Biomedical Imaging (ISBI)", "bibtex": "@inproceedings{slake2021,\n title={{SLAKE}: A Semantically-Labeled Knowledge-Enhanced Dataset for Medical Visual Question Answering},\n author={Bo Liu and Li-Ming Zhan and Li Xu and Lin Ma and Yan Shen and Xiao-Ming Wu},\n booktitle={IEEE 18th International Symposium on Biomedical Imaging (ISBI)},\n year={2021},\n howpublished={\\url{https://arxiv.org/abs/2102.09542}},\n}" } }, { "id": "plotqa2020", "name": "PlotQA", "symbol": "PLO", "year": 2020, "venue": "WACV", "url": "https://arxiv.org/abs/1909.00997", "family": "Doc & Diagrams", "task_family": "ChartQA", "domain": "DataViz", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Large-scale plot reasoning benchmark with 28.9M QA pairs over 224k charts; evaluation combines vocabulary answers and numerical reasoning.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Domain Knowledge", "Tool Use" ], "cap_count": 5, "cap_score": 0.5556, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "plotqa2020", "citation": { "title": "PlotQA", "authors": "Nitesh Methani and Pradeepika Khapra and Anoop Kunchukuttan and Pratyush Kumar and Mitesh M. Khapra", "year": 2020, "venue": "IEEE/CVF Winter Conference on Applications of Computer Vision (WACV)", "bibtex": "@inproceedings{plotqa2020,\n title={{PlotQA}: Reasoning over Scientific Plots using Visual Attention},\n author={Nitesh Methani and Pradeepika Khapra and Anoop Kunchukuttan and Pratyush Kumar and Mitesh M. Khapra},\n booktitle={IEEE/CVF Winter Conference on Applications of Computer Vision (WACV)},\n year={2020},\n howpublished={\\url{https://arxiv.org/abs/1909.00997}},\n}" } }, { "id": "ai2d", "name": "AI2D", "symbol": "AI2", "year": 2016, "venue": "Dataset", "url": "https://allenai.org/data/ai2d", "family": "Vision-Language", "task_family": "ExamQA", "domain": "STEM", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Diagram-based science QA benchmark with textbook diagrams and questions; reused to test multimodal science reasoning.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Domain Knowledge" ], "cap_count": 4, "cap_score": 0.4444, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "ai2d2016", "citation": { "title": "A Diagram is Worth a Dozen Images", "authors": "Aniruddha Kembhavi and Michael Salvato and Eric Kolve and Minjoon Seo and Hannaneh Hajishirzi and Ali Farhadi", "year": 2016, "venue": "European Conference on Computer Vision (ECCV)", "bibtex": "@inproceedings{ai2d2016,\n title={A Diagram is Worth a Dozen Images},\n author={Aniruddha Kembhavi and Michael Salvato and Eric Kolve and Minjoon Seo and Hannaneh Hajishirzi and Ali Farhadi},\n booktitle={European Conference on Computer Vision (ECCV)},\n year={2016},\n howpublished={\\url{https://allenai.org/data/ai2d}},\n}" } }, { "id": "tqa", "name": "TQA", "symbol": "TQA", "year": 2017, "venue": "CVPR", "url": "https://arxiv.org/abs/1705.03851", "family": "Vision-Language", "task_family": "ExamQA", "domain": "STEM", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Textbook Question Answering benchmark combining diagrams and text; probes science reasoning in multimodal settings.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Domain Knowledge" ], "cap_count": 4, "cap_score": 0.4444, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "tqa2017", "citation": { "title": "Are You Smarter Than a Sixth Grader? Textbook Question Answering for Multimodal Machine Comprehension", "authors": "Aniruddha Kembhavi and Minjoon Seo and Dustin Schwenk and Jonghyun Choi and Ali Farhadi and Hannaneh Hajishirzi", "year": 2017, "venue": "IEEE Conference on Computer Vision and Pattern Recognition (CVPR)", "bibtex": "@inproceedings{tqa2017,\n title={Are You Smarter Than a Sixth Grader? Textbook Question Answering for Multimodal Machine Comprehension},\n author={Aniruddha Kembhavi and Minjoon Seo and Dustin Schwenk and Jonghyun Choi and Ali Farhadi and Hannaneh Hajishirzi},\n booktitle={IEEE Conference on Computer Vision and Pattern Recognition (CVPR)},\n year={2017},\n howpublished={\\url{https://arxiv.org/abs/1705.03851}},\n}" } }, { "id": "babyai2019", "name": "BabyAI", "symbol": "BAB", "year": 2019, "venue": "ICLR", "url": "https://arxiv.org/abs/1810.08272", "family": "Embodied & Agents", "task_family": "InstructionFollowing", "domain": "Embodied", "metrics": "SuccessRate", "judge_type": "auto", "protocol_notes": "Gridworld instruction-following platform with 19 curriculum levels; evaluation reports success rate and sample efficiency using automatic goal checks.", "active_modalities": [ "Text", "Image", "Sensor", "Action" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following", "Planning", "Embodiment" ], "cap_count": 6, "cap_score": 0.6667, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "babyai2019", "citation": { "title": "BabyAI", "authors": "Maxime Chevalier-Boisvert and Dzmitry Bahdanau and Salem Lahlou and Lucas Willems and Chitwan Saharia and Thien Huu Nguyen and Yoshua Bengio", "year": 2019, "venue": "International Conference on Learning Representations (ICLR)", "bibtex": "@inproceedings{babyai2019,\n title={{BabyAI}: A Platform to Study the Sample Efficiency of Grounded Language Learning},\n author={Maxime Chevalier-Boisvert and Dzmitry Bahdanau and Salem Lahlou and Lucas Willems and Chitwan Saharia and Thien Huu Nguyen and Yoshua Bengio},\n booktitle={International Conference on Learning Representations (ICLR)},\n year={2019},\n howpublished={\\url{https://arxiv.org/abs/1810.08272}},\n}" } }, { "id": "behavior1k2024", "name": "BEHAVIOR-1K", "symbol": "BEH", "year": 2024, "venue": "CoRL", "url": "https://arxiv.org/abs/2403.09227", "family": "Embodied & Agents", "task_family": "HouseholdManipulation", "domain": "Embodied", "metrics": "SuccessRate", "judge_type": "auto", "protocol_notes": "Human-centered benchmark of 1000 long-horizon household activities in OmniGibson simulation; mobile manipulation agents measured via automatic success/step metrics.", "active_modalities": [ "Text", "Image", "Sensor", "Action" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following", "Planning", "Embodiment" ], "cap_count": 6, "cap_score": 0.6667, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "behavior1k2024", "citation": { "title": "BEHAVIOR-1K", "authors": "Chengshu Li and Ruohan Zhang and Josiah Wong and Cem Gokmen and Sanjana Srivastava and Roberto Mart\\'i", "year": 2023, "venue": "Conference on Robot Learning (CoRL)", "bibtex": "@inproceedings{behavior1k2024,\n title={{BEHAVIOR-1K}: A Benchmark for Embodied {AI} with 1,000 Everyday Activities and Realistic Simulation},\n author={Chengshu Li and Ruohan Zhang and Josiah Wong and Cem Gokmen and Sanjana Srivastava and Roberto Mart{\\'i}n-Mart{\\'i}n and Chen Wang and Gabrael Levine and Michael Lingelbach and Jiankai Sun and Mona Shen and Minjune Hwang and Manasi Sharma and Arman Aydin and Dhruva Bansal and Samuel Hunter and Kyungho Kim and Alan Lou and Caleb R. Matthews and Ivan Villa-Reyes and Te-Lin Wu and Dylan Zhang and Fei-Fei Li and Silvio Savarese and Danfei Xu and Jiajun Wu and C. Karen Liu and Jeanie Shen},\n booktitle={Conference on Robot Learning (CoRL)},\n year={2023},\n howpublished={\\url{https://arxiv.org/abs/2403.09227}},\n}" } }, { "id": "alfworld2021", "name": "ALFWorld", "symbol": "AL2", "year": 2021, "venue": "ICLR", "url": "https://arxiv.org/abs/2010.03768", "family": "Embodied & Agents", "task_family": "InstructionFollowing", "domain": "Embodied", "metrics": "SuccessRate", "judge_type": "auto", "protocol_notes": "Aligns TextWorld policies with ALFRED environments; agents execute text-derived plans in 3D visual scenes and are evaluated via automatic task success and episode length.", "active_modalities": [ "Text", "Sensor", "Action" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following", "Long-Horizon Memory", "Planning", "Embodiment" ], "cap_count": 7, "cap_score": 0.7778, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "alfworld2021", "citation": { "title": "ALFWorld", "authors": "Mohit Shridhar and Xingdi Yuan and Marc-Alexandre C\\^o", "year": 2021, "venue": "International Conference on Learning Representations (ICLR)", "bibtex": "@inproceedings{alfworld2021,\n title={{ALFWorld}: Aligning Text and Embodied Environments for Interactive Learning},\n author={Mohit Shridhar and Xingdi Yuan and Marc-Alexandre C{\\^o}t{\\'e} and Yonatan Bisk and Daniel Fried and Matthew Hausknecht},\n booktitle={International Conference on Learning Representations (ICLR)},\n year={2021},\n howpublished={\\url{https://arxiv.org/abs/2010.03768}},\n}" } }, { "id": "mmreact2023", "name": "MM-REACT", "symbol": "MMR", "year": 2023, "venue": "arXiv", "url": "https://arxiv.org/abs/2303.11381", "family": "Vision-Language", "task_family": "ToolUse", "domain": "General", "metrics": "Accuracy", "judge_type": "LLM", "protocol_notes": "Evaluates multimodal tool-calling prompts where ChatGPT orchestrates external vision experts; success judged by LLM or human grading of generated answers.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Tool Use" ], "cap_count": 4, "cap_score": 0.4444, "cost_tier": 2, "cost_tier_label": "Tier 2: $50-$150 (Multi-Turn / API)", "paper_citation_key": "mmreact2023", "citation": { "title": "MM-REACT", "authors": "Jianwei Yang and Hao Zhang and Feng Li and Xueyan Zou and Chunyuan Li and Jianfeng Gao", "year": 2023, "venue": "arXiv preprint arXiv:2303.11381", "bibtex": "@article{mmreact2023,\n title={{MM-REACT}: Prompting {ChatGPT} for Multimodal Reasoning and Action},\n author={Jianwei Yang and Hao Zhang and Feng Li and Xueyan Zou and Chunyuan Li and Jianfeng Gao},\n journal={arXiv preprint arXiv:2303.11381},\n year={2023},\n howpublished={\\url{https://arxiv.org/abs/2303.11381}},\n}" } }, { "id": "avqa2022", "name": "AVQA", "symbol": "AVQ", "year": 2022, "venue": "ACM MM", "url": "https://dl.acm.org/doi/10.1145/3503161.3548291", "family": "Audio & Speech", "task_family": "AVQA", "domain": "General", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Audio-visual QA benchmark over real videos built on VGGSound clips; evaluates multimodal reasoning with automatic accuracy on thousands of question-answer pairs.", "active_modalities": [ "Text", "Audio", "Video" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning" ], "cap_count": 3, "cap_score": 0.3333, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "avqa2022", "citation": { "title": "AVQA", "authors": "Pinzhen Yang and Lu Wang and Linchao Zhu and Yi Yang", "year": 2022, "venue": "ACM International Conference on Multimedia (ACM MM)", "bibtex": "@inproceedings{avqa2022,\n title={{AVQA}: A Dataset for Audio-Visual Question Answering on Videos},\n author={Pinzhen Yang and Lu Wang and Linchao Zhu and Yi Yang},\n booktitle={ACM International Conference on Multimedia (ACM MM)},\n year={2022},\n howpublished={\\url{https://dl.acm.org/doi/10.1145/3503161.3548291}},\n}" } }, { "id": "toolbench2023", "name": "ToolBench", "symbol": "TOO", "year": 2023, "venue": "ICLR", "url": "https://arxiv.org/abs/2307.16789", "family": "Vision-Language", "task_family": "ToolUse", "domain": "General", "metrics": "Accuracy", "judge_type": "LLM", "protocol_notes": "Instruction-tuning dataset and evaluation for tool-augmented LLMs covering 16k APIs; success judged automatically via ToolEval on multi-step API call execution.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Tool Use" ], "cap_count": 4, "cap_score": 0.4444, "cost_tier": 2, "cost_tier_label": "Tier 2: $50-$150 (Multi-Turn / API)", "paper_citation_key": "toolbench2023", "citation": { "title": "ToolLLM", "authors": "Yujia Qin and Shihao Liang and Yining Ye and Kunlun Zhu and Lan Yan and Yaxi Lu and Yankai Lin and Pei-Hao Su and Shengding Hu and Jie Zhou and Zhiyuan Liu and Maosong Sun", "year": 2024, "venue": "International Conference on Learning Representations (ICLR)", "bibtex": "@inproceedings{toolbench2023,\n title={{ToolLLM}: Facilitating Large Language Models to Master 16000+ Real-world {APIs}},\n author={Yujia Qin and Shihao Liang and Yining Ye and Kunlun Zhu and Lan Yan and Yaxi Lu and Yankai Lin and Pei-Hao Su and Shengding Hu and Jie Zhou and Zhiyuan Liu and Maosong Sun},\n booktitle={International Conference on Learning Representations (ICLR)},\n year={2024},\n howpublished={\\url{https://arxiv.org/abs/2307.16789}},\n}" } }, { "id": "dramaqa2021", "name": "DramaQA", "symbol": "DRA", "year": 2021, "venue": "AAAI", "url": "https://aaai.org/ocs/index.php/AAAI/AAAI21/paper/view/16176", "family": "Video & Temporal", "task_family": "VideoQA", "domain": "Entertainment", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Character-centered long-form TV drama QA benchmark with hierarchical questions and dense annotations; evaluation uses automatic accuracy on multiple-choice answers across episodes.", "active_modalities": [ "Text", "Image", "Video" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Domain Knowledge", "Long-Horizon Memory" ], "cap_count": 5, "cap_score": 0.5556, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "dramaqa2021", "citation": { "title": "DramaQA", "authors": "Seongho Choi and Kyoung-Woon On and Yu-Jung Heo and Ahjeong Seo and Youwon Jang and Minsu Lee and Byoung-Tak Zhang", "year": 2021, "venue": "AAAI Conference on Artificial Intelligence (AAAI)", "bibtex": "@inproceedings{dramaqa2021,\n title={{DramaQA}: Character-Centered Video Story Understanding with Hierarchical {QA}},\n author={Seongho Choi and Kyoung-Woon On and Yu-Jung Heo and Ahjeong Seo and Youwon Jang and Minsu Lee and Byoung-Tak Zhang},\n booktitle={AAAI Conference on Artificial Intelligence (AAAI)},\n year={2021},\n howpublished={\\url{https://aaai.org/ocs/index.php/AAAI/AAAI21/paper/view/16176}},\n}" } }, { "id": "qvhighlights2021", "name": "QVHighlights", "symbol": "QVH", "year": 2021, "venue": "NeurIPS", "url": "https://arxiv.org/abs/2107.09609", "family": "Video & Temporal", "task_family": "VideoRetrieval", "domain": "General", "metrics": "mAP", "judge_type": "auto", "protocol_notes": "Long-video highlight detection benchmark with natural language queries and moment annotations; evaluated via mean Average Precision on localized predictions.", "active_modalities": [ "Text", "Video" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Long-Horizon Memory" ], "cap_count": 4, "cap_score": 0.4444, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "qvhighlights2021", "citation": { "title": "QVHighlights", "authors": "Jie Lei and Tamara L. Berg and Mohit Bansal", "year": 2021, "venue": "Advances in Neural Information Processing Systems (NeurIPS)", "bibtex": "@inproceedings{qvhighlights2021,\n title={{QVHighlights}: Detecting Moments and Highlights in Videos via Natural Language Queries},\n author={Jie Lei and Tamara L. Berg and Mohit Bansal},\n booktitle={Advances in Neural Information Processing Systems (NeurIPS)},\n year={2021},\n howpublished={\\url{https://arxiv.org/abs/2107.09609}},\n}" } }, { "id": "rxr2021", "name": "RxR", "symbol": "RXR", "year": 2021, "venue": "CVPR", "url": "https://arxiv.org/abs/2010.07954", "family": "Embodied & Agents", "task_family": "Navigation", "domain": "Embodied", "metrics": "SuccessRate;SPL", "judge_type": "auto", "protocol_notes": "Room-Across-Room multilingual navigation benchmark with panoramic RGB-D scenes; agents follow long natural-language instructions and are scored via success rate and SPL.", "active_modalities": [ "Text", "Image", "Sensor", "Action" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following", "Long-Horizon Memory", "Planning", "Embodiment" ], "cap_count": 7, "cap_score": 0.7778, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "rxr2021", "citation": { "title": "Room-Across-Room", "authors": "Alexander Ku and Peter Anderson and Roma Patel and Eugene Ie and Jason Baldridge", "year": 2020, "venue": "Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)", "bibtex": "@inproceedings{rxr2021,\n title={{Room-Across-Room}: Multilingual Vision-and-Language Navigation with Dense Spatiotemporal Grounding},\n author={Alexander Ku and Peter Anderson and Roma Patel and Eugene Ie and Jason Baldridge},\n booktitle={Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)},\n year={2020},\n howpublished={\\url{https://arxiv.org/abs/2010.07954}},\n}" } }, { "id": "visdial2017", "name": "VisDial", "symbol": "VI2", "year": 2017, "venue": "CVPR", "url": "https://arxiv.org/abs/1611.08669", "family": "Vision-Language", "task_family": "DialogQA", "domain": "General", "metrics": "NDCG", "judge_type": "auto", "protocol_notes": "Visual dialogue benchmark with multi-turn question answering grounded in images; evaluated via NDCG and recall over ranked answer candidates.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Long-Horizon Memory" ], "cap_count": 4, "cap_score": 0.4444, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "visdial2017", "citation": { "title": "Visual Dialog", "authors": "Abhishek Das and Satwik Kottur and Khushi Gupta and Avi Singh and Deshraj Yadav and Jos'e", "year": 2017, "venue": "IEEE Conference on Computer Vision and Pattern Recognition (CVPR)", "bibtex": "@inproceedings{visdial2017,\n title={Visual Dialog},\n author={Abhishek Das and Satwik Kottur and Khushi Gupta and Avi Singh and Deshraj Yadav and Jos{'e} M. F. Moura and Devi Parikh and Dhruv Batra},\n booktitle={IEEE Conference on Computer Vision and Pattern Recognition (CVPR)},\n year={2017},\n howpublished={\\url{https://arxiv.org/abs/1611.08669}},\n}" } }, { "id": "mind2web2023", "name": "Mind2Web", "symbol": "MIN", "year": 2023, "venue": "NeurIPS", "url": "https://arxiv.org/abs/2306.06070", "family": "Embodied & Agents", "task_family": "WebAgent", "domain": "Web", "metrics": "SuccessRate", "judge_type": "auto", "protocol_notes": "Generalist web-agent benchmark with crowdsourced tasks and DOM interactions; agents follow text instructions on unseen websites and are scored automatically by task completion.", "active_modalities": [ "Text", "Action" ], "active_capabilities": [ "Perception", "Reasoning", "Instruction Following", "Long-Horizon Memory", "Planning", "Embodiment" ], "cap_count": 6, "cap_score": 0.6667, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "mind2web2023", "citation": { "title": "Mind2Web", "authors": "Xiang Deng and Yu Gu and Boyuan Zheng and Shiyan Chen and Samuel Stevens and Boshi Wang and Huan Sun and Yu Su", "year": 2023, "venue": "Advances in Neural Information Processing Systems (NeurIPS)", "bibtex": "@inproceedings{mind2web2023,\n title={{Mind2Web}: Towards a Generalist Agent for the Web},\n author={Xiang Deng and Yu Gu and Boyuan Zheng and Shiyan Chen and Samuel Stevens and Boshi Wang and Huan Sun and Yu Su},\n booktitle={Advances in Neural Information Processing Systems (NeurIPS)},\n year={2023},\n howpublished={\\url{https://arxiv.org/abs/2306.06070}},\n}" } }, { "id": "webarena2023", "name": "WebArena", "symbol": "WEB", "year": 2023, "venue": "ICLR", "url": "https://arxiv.org/abs/2307.13854", "family": "Embodied & Agents", "task_family": "WebAgent", "domain": "Web", "metrics": "SuccessRate", "judge_type": "auto", "protocol_notes": "Realistic web environment benchmark spanning diverse sites (shopping, forums, documentation) where agents execute DOM-level actions; success measured automatically against scripted goals.", "active_modalities": [ "Text", "Action" ], "active_capabilities": [ "Perception", "Reasoning", "Instruction Following", "Long-Horizon Memory", "Planning", "Embodiment" ], "cap_count": 6, "cap_score": 0.6667, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "webarena2023", "citation": { "title": "WebArena", "authors": "Shuyan Zhou and Frank F. Xu and Hao Zhu and Xuhui Zhou and Robert Lo and Abishek Sridhar and Xianyi Cheng and Yonatan Bisk and Daniel Fried and Uri Alon and Graham Neubig", "year": 2024, "venue": "International Conference on Learning Representations (ICLR)", "bibtex": "@inproceedings{webarena2023,\n title={{WebArena}: A Realistic Web Environment for Building Autonomous Agents},\n author={Shuyan Zhou and Frank F. Xu and Hao Zhu and Xuhui Zhou and Robert Lo and Abishek Sridhar and Xianyi Cheng and Yonatan Bisk and Daniel Fried and Uri Alon and Graham Neubig},\n booktitle={International Conference on Learning Representations (ICLR)},\n year={2024},\n howpublished={\\url{https://arxiv.org/abs/2307.13854}},\n}" } }, { "id": "scienceworld2022", "name": "ScienceWorld", "symbol": "SCI", "year": 2022, "venue": "EMNLPFindings", "url": "https://arxiv.org/abs/2212.08656", "family": "Embodied & Agents", "task_family": "InstructionFollowing", "domain": "Science", "metrics": "SuccessRate;Completion", "judge_type": "auto", "protocol_notes": "Text-based science lab with 1000+ procedural tasks; automatic checker validates multi-step goals.", "active_modalities": [ "Text", "Sensor", "Action" ], "active_capabilities": [ "Perception", "Reasoning", "Instruction Following", "Domain Knowledge", "Planning", "Embodiment" ], "cap_count": 6, "cap_score": 0.6667, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "scienceworld2022", "citation": { "title": "ScienceWorld", "authors": "Ruoyao Wang and Peter Jansen and Marc-Alexandre C\\^o", "year": 2022, "venue": "Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing (EMNLP Findings)", "bibtex": "@inproceedings{scienceworld2022,\n title={{ScienceWorld}: Is Your Agent Smarter Than a 5th Grader?},\n author={Ruoyao Wang and Peter Jansen and Marc-Alexandre C{\\^o}t{\\'e} and Prithviraj Ammanabrolu},\n booktitle={Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing (EMNLP Findings)},\n year={2022},\n howpublished={\\url{https://arxiv.org/abs/2212.08656}},\n}" } }, { "id": "webshop2022", "name": "WebShop", "symbol": "WE2", "year": 2022, "venue": "ICLR", "url": "https://arxiv.org/abs/2208.08464", "family": "Embodied & Agents", "task_family": "WebAgent", "domain": "Ecommerce", "metrics": "SuccessRate", "judge_type": "auto", "protocol_notes": "Shopping assistant benchmark with DOM-level actions; success computed automatically by goal and budget satisfaction.", "active_modalities": [ "Text", "Image", "Action" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following", "Planning", "Tool Use", "Embodiment" ], "cap_count": 7, "cap_score": 0.7778, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "webshop2022", "citation": { "title": "WebShop", "authors": "Shunyu Yao and Howard Chen and John Yang and Karthik Narasimhan", "year": 2022, "venue": "Advances in Neural Information Processing Systems (NeurIPS)", "bibtex": "@inproceedings{webshop2022,\n title={{WebShop}: Towards Scalable Real-World Web Interaction with Grounded Language Agents},\n author={Shunyu Yao and Howard Chen and John Yang and Karthik Narasimhan},\n booktitle={Advances in Neural Information Processing Systems (NeurIPS)},\n year={2022},\n howpublished={\\url{https://arxiv.org/abs/2208.08464}},\n}" } }, { "id": "xwebagentbench2025", "name": "X-WebAgentBench", "symbol": "XWE", "year": 2025, "venue": "ACL-Findings", "url": "https://arxiv.org/abs/2505.15372", "family": "Embodied & Agents", "task_family": "WebAgent", "domain": "Ecommerce", "metrics": "TaskScore", "judge_type": "auto", "protocol_notes": "Multilingual extension of WebShop with 2.8k instructions across 14 languages; models plan DOM actions per language and success is measured automatically against target shopping goals.", "active_modalities": [ "Text", "Action" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following", "Domain Knowledge", "Planning", "Tool Use", "Embodiment" ], "cap_count": 8, "cap_score": 0.8889, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "xwebagentbench2025", "citation": { "title": "X-WebAgentBench: A Multilingual Interactive Web Benchmark for Evaluating Global Agentic Systems", "authors": "Wang, Peng and Tao, Ruihan and Chen, Qiguang and Hu, Mengkang and Qin, Libo", "year": 2025, "venue": "arXiv preprint arXiv:2505.15372", "bibtex": "@article{xwebagentbench2025,\n title={X-WebAgentBench: A Multilingual Interactive Web Benchmark for Evaluating Global Agentic Systems},\n author={Wang, Peng and Tao, Ruihan and Chen, Qiguang and Hu, Mengkang and Qin, Libo},\n journal={arXiv preprint arXiv:2505.15372},\n year={2025}\n}" } }, { "id": "agentbench2024", "name": "AgentBench", "symbol": "AGE", "year": 2024, "venue": "ICLR", "url": "https://arxiv.org/abs/2403.17752", "family": "Embodied & Agents", "task_family": "WebAgent", "domain": "General", "metrics": "SuccessRate", "judge_type": "auto", "protocol_notes": "Browser-based tool-use evaluation covering office automation and shopping tasks with automatic checkers per domain.", "active_modalities": [ "Text", "Image", "Action" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following", "Planning", "Tool Use", "Embodiment" ], "cap_count": 7, "cap_score": 0.7778, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "agentbench2024", "citation": { "title": "AgentBench", "authors": "Xiao Liu and Hao Yu and Hanchen Zhang and Yifan Xu and Xuanyu Lei and Hanyu Lai and Yu Gu and Hangdong Ding and Kaiwen Men and Kejuan Yang and Shudan Zhang and Xiang Deng and Aohan Zeng and Zhengxiao Du and Chenhui Zhang and Sheng Shen and Tianjun Zhang and Yu Su and Huan Sun and Minlie Huang and Yuxiao Dong and Jie Tang", "year": 2024, "venue": "International Conference on Learning Representations (ICLR)", "bibtex": "@inproceedings{agentbench2024,\n title={{AgentBench}: Evaluating {LLMs} as Agents},\n author={Xiao Liu and Hao Yu and Hanchen Zhang and Yifan Xu and Xuanyu Lei and Hanyu Lai and Yu Gu and Hangdong Ding and Kaiwen Men and Kejuan Yang and Shudan Zhang and Xiang Deng and Aohan Zeng and Zhengxiao Du and Chenhui Zhang and Sheng Shen and Tianjun Zhang and Yu Su and Huan Sun and Minlie Huang and Yuxiao Dong and Jie Tang},\n booktitle={International Conference on Learning Representations (ICLR)},\n year={2024},\n howpublished={\\url{https://arxiv.org/abs/2403.17752}},\n}" } }, { "id": "chart2code2023", "name": "Chart2Code", "symbol": "CHA", "year": 2023, "venue": "NAACL", "url": "https://arxiv.org/abs/2304.02017", "family": "Doc & Diagrams", "task_family": "ChartProgramming", "domain": "DataViz", "metrics": "ExecutionAccuracy;BLEU", "judge_type": "auto", "protocol_notes": "Generate plotting code from charts; scripts execute predicted code to compare rendered outputs plus BLEU on textual summaries.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Domain Knowledge", "Tool Use" ], "cap_count": 5, "cap_score": 0.5556, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "chart2code2023", "citation": { "title": "Plot2Code: A Comprehensive Benchmark for Evaluating Multi-modal Large Language Models in Code Generation from Scientific Plots", "authors": "Wu, Chengyue and Ge, Yixiao and Guo, Qiushan and Wang, Jiahao and others", "year": 2025, "venue": "Findings of the Association for Computational Linguistics: NAACL", "bibtex": "@inproceedings{chart2code2023,\n title={Plot2Code: A Comprehensive Benchmark for Evaluating Multi-modal Large Language Models in Code Generation from Scientific Plots},\n author={Wu, Chengyue and Ge, Yixiao and Guo, Qiushan and Wang, Jiahao and others},\n booktitle={Findings of the Association for Computational Linguistics: NAACL},\n year={2025},\n url={https://arxiv.org/abs/2405.07990}\n}" } }, { "id": "vggsoundavqa2021", "name": "VGGSound-AVQA", "symbol": "VGG", "year": 2021, "venue": "ICCVW", "url": "https://arxiv.org/abs/2105.04899", "family": "Audio & Speech", "task_family": "AVQA", "domain": "General", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Audio-visual QA dataset built on VGGSound clips with multiple question types and automatic accuracy scoring.", "active_modalities": [ "Text", "Image", "Audio", "Video" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Domain Knowledge" ], "cap_count": 4, "cap_score": 0.4444, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "vggsoundavqa2021", "citation": { "title": "Audio-Visual Question Answering with VGGSound", "authors": "Honglie Chen and Weidi Xie and Andrea Vedaldi and Andrew Zisserman", "year": 2021, "venue": "IEEE/CVF International Conference on Computer Vision Workshops (ICCVW)", "bibtex": "@inproceedings{vggsoundavqa2021,\n title={Audio-Visual Question Answering with {VGGSound}},\n author={Honglie Chen and Weidi Xie and Andrea Vedaldi and Andrew Zisserman},\n booktitle={IEEE/CVF International Conference on Computer Vision Workshops (ICCVW)},\n year={2021},\n howpublished={\\url{https://arxiv.org/abs/2105.04899}},\n}" } }, { "id": "audiosetzsavqa2023", "name": "AudioSet-ZS-AVQA", "symbol": "AU2", "year": 2023, "venue": "arXiv", "url": "https://arxiv.org/abs/2308.06585", "family": "Audio & Speech", "task_family": "AudioQA", "domain": "General", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Zero-shot audio QA benchmark using AudioSet tags and tool hints; answers verified automatically via canonical labels.", "active_modalities": [ "Text", "Audio" ], "active_capabilities": [ "Perception", "Reasoning", "Domain Knowledge", "Tool Use" ], "cap_count": 4, "cap_score": 0.4444, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "audiosetzsavqa2023", "citation": { "title": "Zero-Shot Audio-Visual Question Answering with AudioSet", "authors": "Juncheng Li and Minghe Gao and Xin He and Ziyi Liu and Shiliang Zhang", "year": 2023, "venue": "arXiv preprint arXiv:2308.06585", "bibtex": "@article{audiosetzsavqa2023,\n title={Zero-Shot Audio-Visual Question Answering with {AudioSet}},\n author={Juncheng Li and Minghe Gao and Xin He and Ziyi Liu and Shiliang Zhang},\n journal={arXiv preprint arXiv:2308.06585},\n year={2023},\n howpublished={\\url{https://arxiv.org/abs/2308.06585}},\n}" } }, { "id": "spokensquad2018", "name": "SpokenSQuAD", "symbol": "SPO", "year": 2018, "venue": "Interspeech", "url": "https://arxiv.org/abs/1804.00320", "family": "Audio & Speech", "task_family": "AudioQA", "domain": "General", "metrics": "F1;EM", "judge_type": "auto", "protocol_notes": "Speech rendering of SQuAD passages and questions; evaluate text answers against references using EM and F1 after ASR.", "active_modalities": [ "Text", "Audio" ], "active_capabilities": [ "Perception", "Reasoning", "Domain Knowledge" ], "cap_count": 3, "cap_score": 0.3333, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "spokensquad2018", "citation": { "title": "Spoken SQuAD: A Study of Mitigating the Impact of Speech Recognition Errors on Listening Comprehension", "authors": "Lee, Chia-Hsuan and Wu, Szu-Lin and Liu, Chi-Liang and Lee, Hung-yi", "year": 2018, "venue": "Interspeech", "bibtex": "@inproceedings{spokensquad2018,\n title={Spoken SQuAD: A Study of Mitigating the Impact of Speech Recognition Errors on Listening Comprehension},\n author={Lee, Chia-Hsuan and Wu, Szu-Lin and Liu, Chi-Liang and Lee, Hung-yi},\n booktitle={Interspeech},\n year={2018},\n url={https://arxiv.org/abs/1804.00320}\n}" } }, { "id": "virtualhome2018", "name": "VirtualHome", "symbol": "VIR", "year": 2018, "venue": "CVPR", "url": "https://arxiv.org/abs/1806.07011", "family": "Embodied & Agents", "task_family": "Manipulation", "domain": "Household", "metrics": "ProgramExecution;SuccessRate", "judge_type": "auto", "protocol_notes": "Synthetic household program benchmark; agents execute action programs in Unity scenes and success is computed automatically per task.", "active_modalities": [ "Text", "Image", "3D", "Sensor", "Action" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following", "Domain Knowledge", "Planning", "Embodiment" ], "cap_count": 7, "cap_score": 0.7778, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "virtualhome2018", "citation": { "title": "VirtualHome", "authors": "Xavier Puig and Kevin Ra and Marko Boben and Jiaman Li and Tingwu Wang and Sanja Fidler and Antonio Torralba", "year": 2018, "venue": "IEEE Conference on Computer Vision and Pattern Recognition (CVPR)", "bibtex": "@inproceedings{virtualhome2018,\n title={{VirtualHome}: Simulating Household Activities via Programs},\n author={Xavier Puig and Kevin Ra and Marko Boben and Jiaman Li and Tingwu Wang and Sanja Fidler and Antonio Torralba},\n booktitle={IEEE Conference on Computer Vision and Pattern Recognition (CVPR)},\n year={2018},\n howpublished={\\url{https://arxiv.org/abs/1806.07011}},\n}" } }, { "id": "objaverseqa2023", "name": "ObjaverseQA", "symbol": "OBJ", "year": 2023, "venue": "arXiv", "url": "https://arxiv.org/abs/2305.11624", "family": "3D & Spatial", "task_family": "3DQA", "domain": "3D", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Question answering over Objaverse 3D assets with renderings and meshes; evaluation via automatic exact match accuracy.", "active_modalities": [ "Text", "Image", "3D" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Domain Knowledge" ], "cap_count": 4, "cap_score": 0.4444, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "objaverseqa2023", "citation": { "title": "Objaverse-QA", "authors": "Jiacheng Chen and Ruizhi Shao and Eric Yi", "year": 2023, "venue": "arXiv preprint arXiv:2305.11624", "bibtex": "@article{objaverseqa2023,\n title={{Objaverse-QA}: A Multi-Modal Benchmark for Grounded 3D Question Answering},\n author={Jiacheng Chen and Ruizhi Shao and Eric Yi},\n journal={arXiv preprint arXiv:2305.11624},\n year={2023},\n howpublished={\\url{https://arxiv.org/abs/2305.11624}},\n}" } }, { "id": "moviechatqa2023", "name": "MovieChatQA", "symbol": "MO2", "year": 2023, "venue": "CVPR", "url": "https://arxiv.org/abs/2304.08485", "family": "Video & Temporal", "task_family": "VideoQA", "domain": "Entertainment", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Long-form movie QA benchmark with subtitles and multi-choice answers scored automatically for memory and story reasoning.", "active_modalities": [ "Text", "Video" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Domain Knowledge", "Long-Horizon Memory" ], "cap_count": 5, "cap_score": 0.5556, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "moviechatqa2023", "citation": { "title": "MovieChat", "authors": "Chaoyi Song and Jidong Tian and Jing Zhang and Shaofei Cai and Baobao Chang", "year": 2024, "venue": "IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)", "bibtex": "@inproceedings{moviechatqa2023,\n title={{MovieChat}: From Dense Token to Sparse Memory for Long Video Understanding},\n author={Chaoyi Song and Jidong Tian and Jing Zhang and Shaofei Cai and Baobao Chang},\n booktitle={IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},\n year={2024},\n howpublished={\\url{https://arxiv.org/abs/2304.08485}},\n}" } }, { "id": "charadessta2017", "name": "Charades-STA", "symbol": "CH2", "year": 2017, "venue": "ICCV", "url": "https://arxiv.org/abs/1705.02101", "family": "Video & Temporal", "task_family": "VideoGrounding", "domain": "General", "metrics": "mIoU;R@1", "judge_type": "auto", "protocol_notes": "Temporal grounding benchmark localizing natural language queries in Charades videos; evaluation via IoU and recall metrics.", "active_modalities": [ "Text", "Video" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning" ], "cap_count": 3, "cap_score": 0.3333, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "charadessta2017", "citation": { "title": "Temporal Grounding of Natural Language in Untrimmed Videos", "authors": "Jiyang Gao and Chen Sun and Zhenheng Yang and Ram Nevatia", "year": 2017, "venue": "IEEE International Conference on Computer Vision (ICCV)", "bibtex": "@inproceedings{charadessta2017,\n title={Temporal Grounding of Natural Language in Untrimmed Videos},\n author={Jiyang Gao and Chen Sun and Zhenheng Yang and Ram Nevatia},\n booktitle={IEEE International Conference on Computer Vision (ICCV)},\n year={2017},\n howpublished={\\url{https://arxiv.org/abs/1705.02101}},\n}" } }, { "id": "didemo2017", "name": "DiDeMo", "symbol": "DID", "year": 2017, "venue": "CVPR", "url": "https://arxiv.org/abs/1704.04130", "family": "Video & Temporal", "task_family": "VideoGrounding", "domain": "General", "metrics": "R@1;R@5;mIoU", "judge_type": "auto", "protocol_notes": "Temporal moment localization dataset of crowd videos with open vocabulary queries; automatic ranking metrics over intervals.", "active_modalities": [ "Text", "Video" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning" ], "cap_count": 3, "cap_score": 0.3333, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "didemo2017", "citation": { "title": "Localizing Moments in Video with Natural Language", "authors": "Lisa Anne Hendricks and Oliver Wang and Ruben Shechtman and Josef Sivic and Trevor Darrell and Bryan Russell", "year": 2017, "venue": "IEEE International Conference on Computer Vision (ICCV)", "bibtex": "@inproceedings{didemo2017,\n title={Localizing Moments in Video with Natural Language},\n author={Lisa Anne Hendricks and Oliver Wang and Ruben Shechtman and Josef Sivic and Trevor Darrell and Bryan Russell},\n booktitle={IEEE International Conference on Computer Vision (ICCV)},\n year={2017},\n howpublished={\\url{https://arxiv.org/abs/1704.04130}},\n}" } }, { "id": "how2qa2019", "name": "How2QA", "symbol": "HOW", "year": 2019, "venue": "EMNLP", "url": "https://arxiv.org/abs/1904.00969", "family": "Audio & Speech", "task_family": "VideoQA", "domain": "Instructional", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Instructional HowTo100M QA benchmark combining narration audio subtitles and video frames; automatic accuracy against reference answers.", "active_modalities": [ "Text", "Audio", "Video" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Domain Knowledge", "Long-Horizon Memory" ], "cap_count": 5, "cap_score": 0.5556, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "how2qa2019", "citation": { "title": "Towards Automatic Evaluation of Natural Language Explanations on Instructional Videos", "authors": "Luowei Zhou and Chenliang Xu and Jason J. Corso", "year": 2019, "venue": "Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing (EMNLP)", "bibtex": "@inproceedings{how2qa2019,\n title={Towards Automatic Evaluation of Natural Language Explanations on Instructional Videos},\n author={Luowei Zhou and Chenliang Xu and Jason J. Corso},\n booktitle={Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing (EMNLP)},\n year={2019},\n howpublished={\\url{https://arxiv.org/abs/1904.00969}},\n}" } }, { "id": "gaia2023", "name": "GAIA", "symbol": "GAI", "year": 2023, "venue": "ICLR", "url": "https://arxiv.org/abs/2310.12931", "family": "Vision-Language", "task_family": "ComprehensiveEval", "domain": "General", "metrics": "LLMScore", "judge_type": "LLM", "protocol_notes": "Complex assistant tasks mixing text and charts scored by GPT-4 with rubric covering reasoning planning and tool usage.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following", "Domain Knowledge", "Long-Horizon Memory", "Planning", "Tool Use" ], "cap_count": 8, "cap_score": 0.8889, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "gaia2023", "citation": { "title": "GAIA", "authors": "Gr'e", "year": 2024, "venue": "International Conference on Learning Representations (ICLR)", "bibtex": "@inproceedings{gaia2023,\n title={{GAIA}: A Benchmark for General {AI} Assistants},\n author={Gr{'e}goire Mialon and Cl{'e}mentine Fourrier and Craig Swift and Thomas Wolf and Yann LeCun and Thomas Scialom},\n booktitle={International Conference on Learning Representations (ICLR)},\n year={2024},\n howpublished={\\url{https://arxiv.org/abs/2310.12931}},\n}" } }, { "id": "textworld2018", "name": "TextWorld", "symbol": "TE3", "year": 2018, "venue": "NeurIPS", "url": "https://arxiv.org/abs/1806.11532", "family": "Embodied & Agents", "task_family": "TextAdventure", "domain": "General", "metrics": "SuccessRate", "judge_type": "auto", "protocol_notes": "Interactive text-game suite with quests and inventory sensors; success tracked by engine through goal states.", "active_modalities": [ "Text", "Sensor", "Action" ], "active_capabilities": [ "Perception", "Reasoning", "Instruction Following", "Domain Knowledge", "Long-Horizon Memory", "Planning", "Embodiment" ], "cap_count": 7, "cap_score": 0.7778, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "textworld2018", "citation": { "title": "TextWorld", "authors": "Marc-Alexandre C\\^o", "year": 2018, "venue": "Computer Games Workshop at IJCAI", "bibtex": "@inproceedings{textworld2018,\n title={{TextWorld}: A Learning Environment for Text-based Games},\n author={Marc-Alexandre C{\\^o}t{\\'e} and {'A}kos K{'a}d{'a}r and Xingdi Yuan and Ben Kybartas and Tavian Barnes and Emery Fine and James Moore and Matthew Hausknecht and Layla El Asri and Mahmoud Adada and Wendy Tay and Adam Trischler},\n booktitle={Computer Games Workshop at IJCAI},\n year={2018},\n howpublished={\\url{https://arxiv.org/abs/1806.11532}},\n}" } }, { "id": "procthor2022", "name": "ProcTHOR", "symbol": "PRO", "year": 2022, "venue": "CVPR", "url": "https://arxiv.org/abs/2206.11281", "family": "Embodied & Agents", "task_family": "Manipulation", "domain": "Household", "metrics": "SuccessRate", "judge_type": "auto", "protocol_notes": "Procedurally generated household tasks with thousands of 3D layouts; agents follow language goals and success is computed automatically.", "active_modalities": [ "Text", "Image", "3D", "Sensor", "Action" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following", "Domain Knowledge", "Planning", "Embodiment" ], "cap_count": 7, "cap_score": 0.7778, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "procthor2022", "citation": { "title": "ProcTHOR", "authors": "Matt Deitke and Eli VanderBilt and Alvaro Herrasti and Luca Weihs and Jordi Salvador and Kiana Ehsani and Winson Han and Eric Kolve and Ali Farhadi and Aniruddha Kembhavi and Roozbeh Mottaghi", "year": 2022, "venue": "Advances in Neural Information Processing Systems (NeurIPS)", "bibtex": "@inproceedings{procthor2022,\n title={{ProcTHOR}: Large-Scale Embodied {AI} Using Procedural Generation},\n author={Matt Deitke and Eli VanderBilt and Alvaro Herrasti and Luca Weihs and Jordi Salvador and Kiana Ehsani and Winson Han and Eric Kolve and Ali Farhadi and Aniruddha Kembhavi and Roozbeh Mottaghi},\n booktitle={Advances in Neural Information Processing Systems (NeurIPS)},\n year={2022},\n howpublished={\\url{https://arxiv.org/abs/2206.11281}},\n}" } }, { "id": "reverie2020", "name": "REVERIE", "symbol": "REV", "year": 2020, "venue": "CVPR", "url": "https://arxiv.org/abs/1904.10151", "family": "Embodied & Agents", "task_family": "Navigation", "domain": "Indoor", "metrics": "SuccessRate;SPL", "judge_type": "auto", "protocol_notes": "Remote embodied referring expression navigation benchmark requiring goal grounding and navigation scored via success and SPL metrics.", "active_modalities": [ "Text", "Image", "Sensor", "Action" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following", "Planning", "Embodiment" ], "cap_count": 6, "cap_score": 0.6667, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "reverie2020", "citation": { "title": "REVERIE", "authors": "Yuankai Qi and Qi Wu and Peter Anderson and Xin Wang and William Yang Wang and Chunhua Shen and Anton van den Hengel", "year": 2020, "venue": "IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)", "bibtex": "@inproceedings{reverie2020,\n title={{REVERIE}: Remote Embodied Visual Referring Expression in Real Indoor Environments},\n author={Yuankai Qi and Qi Wu and Peter Anderson and Xin Wang and William Yang Wang and Chunhua Shen and Anton van den Hengel},\n booktitle={IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},\n year={2020},\n howpublished={\\url{https://arxiv.org/abs/1904.10151}},\n}" } }, { "id": "soundspaces2022", "name": "SoundSpaces2", "symbol": "SOU", "year": 2022, "venue": "NeurIPS", "url": "https://arxiv.org/abs/2206.08312", "family": "Embodied & Agents", "task_family": "Navigation", "domain": "AudioNavigation", "metrics": "SuccessRate;SPL", "judge_type": "auto", "protocol_notes": "Audio-goal navigation tasks with spatialized audio and RGB-D sensors; automatic evaluation of success and path efficiency.", "active_modalities": [ "Text", "Image", "Audio", "3D", "Sensor", "Action" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following", "Planning", "Embodiment" ], "cap_count": 6, "cap_score": 0.6667, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "soundspaces2022", "citation": { "title": "SoundSpaces 2.0", "authors": "Changan Chen and Carl Schissler and Sanchit Garg and Philip Kobernich and Alexander Clegg and Paul Calamia and Dhruv Batra and Philip W. Krahenbuhl and Kristen Grauman", "year": 2022, "venue": "Advances in Neural Information Processing Systems (NeurIPS)", "bibtex": "@inproceedings{soundspaces2022,\n title={{SoundSpaces 2.0}: A Platform for Audio-Visual Embodied {AI}},\n author={Changan Chen and Carl Schissler and Sanchit Garg and Philip Kobernich and Alexander Clegg and Paul Calamia and Dhruv Batra and Philip W. Krahenbuhl and Kristen Grauman},\n booktitle={Advances in Neural Information Processing Systems (NeurIPS)},\n year={2022},\n howpublished={\\url{https://arxiv.org/abs/2206.08312}},\n}" } }, { "id": "robothor2020", "name": "RoboTHOR", "symbol": "ROB", "year": 2020, "venue": "CVPR", "url": "https://arxiv.org/abs/2004.06799", "family": "Embodied & Agents", "task_family": "Navigation", "domain": "Embodied", "metrics": "SuccessRate;SPL", "judge_type": "auto", "protocol_notes": "Embodied navigation benchmark with RGB-D observations and instructions executed in sim plus real apartments; scored via success rate and SPL.", "active_modalities": [ "Text", "Image", "3D", "Sensor", "Action" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following", "Planning", "Embodiment" ], "cap_count": 6, "cap_score": 0.6667, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "robothor2020", "citation": { "title": "RoboTHOR", "authors": "Matt Deitke and Winson Han and Alvaro Herrasti and Aniruddha Kembhavi and Eric Kolve and Roozbeh Mottaghi and Jordi Salvador and Dustin Schwenk and Eli VanderBilt and Matthew Wallingford and Luca Weihs and Mark Yatskar and Ali Farhadi", "year": 2020, "venue": "IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)", "bibtex": "@inproceedings{robothor2020,\n title={{RoboTHOR}: An Open Simulation-to-Real Embodied {AI} Platform},\n author={Matt Deitke and Winson Han and Alvaro Herrasti and Aniruddha Kembhavi and Eric Kolve and Roozbeh Mottaghi and Jordi Salvador and Dustin Schwenk and Eli VanderBilt and Matthew Wallingford and Luca Weihs and Mark Yatskar and Ali Farhadi},\n booktitle={IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},\n year={2020},\n howpublished={\\url{https://arxiv.org/abs/2004.06799}},\n}" } }, { "id": "spokencoco2020", "name": "SpokenCOCO", "symbol": "SP2", "year": 2020, "venue": "Interspeech", "url": "https://arxiv.org/abs/2002.07984", "family": "Audio & Speech", "task_family": "Captioning", "domain": "General", "metrics": "BLEU;CIDEr", "judge_type": "auto", "protocol_notes": "Speech captioning extension of MSCOCO with human spoken descriptions; evaluated using standard caption metrics after ASR transcription.", "active_modalities": [ "Text", "Image", "Audio" ], "active_capabilities": [ "Perception", "Grounding" ], "cap_count": 2, "cap_score": 0.2222, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "spokencoco2020", "citation": { "title": "SpokenCOCO: 600k Spoken Captions Aligned to MSCOCO Images", "authors": "Hsu, Wei-Ning and others", "year": 2020, "venue": "arXiv preprint arXiv:2002.07984", "bibtex": "@article{spokencoco2020,\n title={SpokenCOCO: 600k Spoken Captions Aligned to MSCOCO Images},\n author={Hsu, Wei-Ning and others},\n journal={arXiv preprint arXiv:2002.07984},\n year={2020}\n}" } }, { "id": "medvidqa2022", "name": "MedVidQA", "symbol": "MED", "year": 2022, "venue": "arXiv", "url": "https://arxiv.org/abs/2203.06667", "family": "Video & Temporal", "task_family": "VideoQA", "domain": "Medical", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Multiple-choice QA over instructional medical videos; automatic accuracy computed on long-context clips.", "active_modalities": [ "Text", "Video" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Domain Knowledge", "Long-Horizon Memory" ], "cap_count": 5, "cap_score": 0.5556, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "medvidqa2022", "citation": { "title": "MedVidQA", "authors": "Deepak Gupta and Dina Demner-Fushman", "year": 2022, "venue": "Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (ACL)", "bibtex": "@inproceedings{medvidqa2022,\n title={{MedVidQA}: A Dataset for Medical Video Question Answering},\n author={Deepak Gupta and Dina Demner-Fushman},\n booktitle={Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (ACL)},\n year={2022},\n howpublished={\\url{https://arxiv.org/abs/2203.06667}},\n}" } }, { "id": "pathvqa2019", "name": "PathVQA", "symbol": "PAT", "year": 2019, "venue": "CVPR", "url": "https://openaccess.thecvf.com/content_CVPR_2019/html/He_PathVQA_3D_Pathology_VQA_for_Better_Diagram-Understanding_CVPR_2019_paper.html", "family": "Vision-Language", "task_family": "MedicalQA", "domain": "Medical", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Pathology diagram QA benchmark with curated question types and exact-match scoring.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Domain Knowledge" ], "cap_count": 4, "cap_score": 0.4444, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "pathvqa2019", "citation": { "title": "PathVQA", "authors": "Xuehai He and Zhuosi Dong and Shuai Zhang and Jinyu Liu and Ziyang Wang", "year": 2019, "venue": "IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)", "bibtex": "@inproceedings{pathvqa2019,\n title={{PathVQA}: 3D Pathology Visual Question Answering for Better Diagram Understanding},\n author={Xuehai He and Zhuosi Dong and Shuai Zhang and Jinyu Liu and Ziyang Wang},\n booktitle={IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},\n year={2019},\n howpublished={\\url{https://openaccess.thecvf.com/content_CVPR_2019/html/He_PathVQA_3D_Pathology_VQA_for_Better_Diagram-Understanding_CVPR_2019_paper.html}},\n}" } }, { "id": "vqarad2018", "name": "VQA-RAD", "symbol": "VQ2", "year": 2018, "venue": "MICCAI", "url": "https://osf.io/89ghr/", "family": "Vision-Language", "task_family": "MedicalQA", "domain": "Medical", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Radiology question answering dataset using medical images and text questions; accuracy computed over free-form answers with normalization.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Domain Knowledge" ], "cap_count": 4, "cap_score": 0.4444, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "vqarad2018", "citation": { "title": "A Dataset of Clinically Generated Visual Questions and Answers about Radiology Images", "authors": "Jason J. Lau and Soumya Gayen and Asma Ben Abacha and Dina Demner-Fushman", "year": 2018, "venue": "Scientific Data", "bibtex": "@inproceedings{vqarad2018,\n title={A Dataset of Clinically Generated Visual Questions and Answers about Radiology Images},\n author={Jason J. Lau and Soumya Gayen and Asma Ben Abacha and Dina Demner-Fushman},\n booktitle={Scientific Data},\n year={2018},\n howpublished={\\url{https://osf.io/89ghr/}},\n}" } }, { "id": "chartbench2023", "name": "ChartBench", "symbol": "CH3", "year": 2023, "venue": "arXiv", "url": "https://arxiv.org/abs/2312.15915", "family": "Doc & Diagrams", "task_family": "ChartQA", "domain": "DataViz", "metrics": "Accuracy;LLMScore", "judge_type": "hybrid", "protocol_notes": "Complex chart reasoning benchmark with executable checking and LLM graders for open responses.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Domain Knowledge", "Tool Use" ], "cap_count": 5, "cap_score": 0.5556, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "chartbench2023", "citation": { "title": "ChartBench", "authors": "Zhengzhuo Xu and Chun Yuan and Menghao Li", "year": 2023, "venue": "arXiv preprint arXiv:2312.15915", "bibtex": "@article{chartbench2023,\n title={{ChartBench}: A Benchmark for Complex Visual Reasoning on Charts},\n author={Zhengzhuo Xu and Chun Yuan and Menghao Li},\n journal={arXiv preprint arXiv:2312.15915},\n year={2023},\n howpublished={\\url{https://arxiv.org/abs/2312.15915}},\n}" } }, { "id": "tabfact2020", "name": "TabFact", "symbol": "TAB", "year": 2020, "venue": "ICLR", "url": "https://arxiv.org/abs/1909.02164", "family": "Vision-Language", "task_family": "FactVerification", "domain": "Data", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Large-scale table fact verification with executable programs; evaluation via classification accuracy on entailment labels.", "active_modalities": [ "Text" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Domain Knowledge", "Tool Use" ], "cap_count": 5, "cap_score": 0.5556, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "tabfact2020", "citation": { "title": "TabFact", "authors": "Wenhu Chen and Hongmin Wang and Jianshu Chen and Yunkai Zhang and Hong Wang and Shiyang Li and Xiyou Zhou and William Yang Wang", "year": 2020, "venue": "International Conference on Learning Representations (ICLR)", "bibtex": "@inproceedings{tabfact2020,\n title={{TabFact}: A Large-scale Dataset for Table-based Fact Verification},\n author={Wenhu Chen and Hongmin Wang and Jianshu Chen and Yunkai Zhang and Hong Wang and Shiyang Li and Xiyou Zhou and William Yang Wang},\n booktitle={International Conference on Learning Representations (ICLR)},\n year={2020},\n howpublished={\\url{https://arxiv.org/abs/1909.02164}},\n}" } }, { "id": "wikitables2015", "name": "WikiTableQuestions", "symbol": "WIK", "year": 2015, "venue": "ACL", "url": "https://arxiv.org/abs/1508.00305", "family": "Doc & Diagrams", "task_family": "TableQA", "domain": "Data", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Question answering over HTML tables with denotation accuracy measured by executing logical forms.", "active_modalities": [ "Text" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Domain Knowledge", "Tool Use" ], "cap_count": 5, "cap_score": 0.5556, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "wikitables2015", "citation": { "title": "Compositional Semantic Parsing on Semi-Structured Tables", "authors": "Panupong Pasupat and Percy Liang", "year": 2015, "venue": "Proceedings of the 53rd Annual Meeting of the Association for Computational Linguistics (ACL)", "bibtex": "@inproceedings{wikitables2015,\n title={Compositional Semantic Parsing on Semi-Structured Tables},\n author={Panupong Pasupat and Percy Liang},\n booktitle={Proceedings of the 53rd Annual Meeting of the Association for Computational Linguistics (ACL)},\n year={2015},\n howpublished={\\url{https://arxiv.org/abs/1508.00305}},\n}" } }, { "id": "hybridqa2020", "name": "HybridQA", "symbol": "HYB", "year": 2020, "venue": "ACL", "url": "https://arxiv.org/abs/2004.07347", "family": "Doc & Diagrams", "task_family": "TableQA", "domain": "Data", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Multi-hop QA combining tables and passages; evaluation via exact-match accuracy on open-form answers.", "active_modalities": [ "Text" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Domain Knowledge", "Long-Horizon Memory", "Tool Use" ], "cap_count": 6, "cap_score": 0.6667, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "hybridqa2020", "citation": { "title": "HybridQA", "authors": "Wenhu Chen and Hanwen Zha and Zhiyu Chen and Hao Xiong and Hong Wang and William Yang Wang", "year": 2020, "venue": "Findings of the Association for Computational Linguistics: EMNLP 2020", "bibtex": "@inproceedings{hybridqa2020,\n title={{HybridQA}: A Dataset of Multi-Hop Question Answering over Tabular and Textual Data},\n author={Wenhu Chen and Hanwen Zha and Zhiyu Chen and Hao Xiong and Hong Wang and William Yang Wang},\n booktitle={Findings of the Association for Computational Linguistics: EMNLP 2020},\n year={2020},\n howpublished={\\url{https://arxiv.org/abs/2004.07347}},\n}" } }, { "id": "dvqa2018", "name": "DVQA", "symbol": "DV2", "year": 2018, "venue": "CVPR", "url": "https://arxiv.org/abs/1801.08163", "family": "Doc & Diagrams", "task_family": "ChartQA", "domain": "DataViz", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Diagrammatic VQA on bar charts with automatic accuracy over generated tokens and numbers.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Domain Knowledge", "Tool Use" ], "cap_count": 5, "cap_score": 0.5556, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "dvqa2018", "citation": { "title": "DVQA", "authors": "Kushal Kafle and Brian Price and Scott Cohen and Christopher Kanan", "year": 2018, "venue": "IEEE Conference on Computer Vision and Pattern Recognition (CVPR)", "bibtex": "@inproceedings{dvqa2018,\n title={{DVQA}: Understanding Data Visualizations via Question Answering},\n author={Kushal Kafle and Brian Price and Scott Cohen and Christopher Kanan},\n booktitle={IEEE Conference on Computer Vision and Pattern Recognition (CVPR)},\n year={2018},\n howpublished={\\url{https://arxiv.org/abs/1801.08163}},\n}" } }, { "id": "tallyqa2019", "name": "TallyQA", "symbol": "TA2", "year": 2019, "venue": "CVPR", "url": "https://arxiv.org/abs/1810.12440", "family": "Vision-Language", "task_family": "CountingQA", "domain": "General", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Large counting-focused VQA dataset with natural and synthetic images; accuracy reported on simple and complex splits.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning" ], "cap_count": 3, "cap_score": 0.3333, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "tallyqa2019", "citation": { "title": "TallyQA", "authors": "Manoj Acharya and Kushal Kafle and Christopher Kanan", "year": 2019, "venue": "AAAI Conference on Artificial Intelligence (AAAI)", "bibtex": "@inproceedings{tallyqa2019,\n title={{TallyQA}: Answering Complex Counting Questions},\n author={Manoj Acharya and Kushal Kafle and Christopher Kanan},\n booktitle={AAAI Conference on Artificial Intelligence (AAAI)},\n year={2019},\n howpublished={\\url{https://arxiv.org/abs/1810.12440}},\n}" } }, { "id": "vcr2019", "name": "VCR", "symbol": "VCR", "year": 2019, "venue": "CVPR", "url": "https://arxiv.org/abs/1811.10830", "family": "Vision-Language", "task_family": "CommonsenseQA", "domain": "General", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Visual Commonsense Reasoning benchmark with question plus rationale multiple-choice scoring.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Domain Knowledge" ], "cap_count": 4, "cap_score": 0.4444, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "vcr2019", "citation": { "title": "From Recognition to Cognition: Visual Commonsense Reasoning", "authors": "Rowan Zellers and Yonatan Bisk and Ali Farhadi and Yejin Choi", "year": 2019, "venue": "IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)", "bibtex": "@inproceedings{vcr2019,\n title={From Recognition to Cognition: Visual Commonsense Reasoning},\n author={Rowan Zellers and Yonatan Bisk and Ali Farhadi and Yejin Choi},\n booktitle={IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},\n year={2019},\n howpublished={\\url{https://arxiv.org/abs/1811.10830}},\n}" } }, { "id": "vizwizcap2020", "name": "VizWiz-Captions", "symbol": "VI3", "year": 2020, "venue": "CVPR", "url": "https://vizwiz.org/tasks-and-datasets/image-captioning/", "family": "Vision-Language", "task_family": "Captioning", "domain": "Accessibility", "metrics": "BLEU;CIDEr", "judge_type": "auto", "protocol_notes": "Image captioning dataset collected from blind photographers; evaluation via standard caption metrics on server.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding" ], "cap_count": 2, "cap_score": 0.2222, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "vizwizcap2020", "citation": { "title": "Towards Addressing Bias in Image Captioning for People who are Blind", "authors": "Danna Gurari and Yinan Zhao and Meng Zhang and Nilavra Bhattacharya", "year": 2020, "venue": "European Conference on Computer Vision (ECCV)", "bibtex": "@inproceedings{vizwizcap2020,\n title={Towards Addressing Bias in Image Captioning for People who are Blind},\n author={Danna Gurari and Yinan Zhao and Meng Zhang and Nilavra Bhattacharya},\n booktitle={European Conference on Computer Vision (ECCV)},\n year={2020},\n howpublished={\\url{https://vizwiz.org/tasks-and-datasets/image-captioning/}},\n}" } }, { "id": "multidoc2dial2021", "name": "MultiDoc2Dial", "symbol": "MUL", "year": 2021, "venue": "EMNLP", "url": "https://arxiv.org/abs/2109.12595", "family": "Vision-Language", "task_family": "DialogQA", "domain": "CustomerSupport", "metrics": "BLEU;ROUGE", "judge_type": "auto", "protocol_notes": "Document-grounded multi-turn dialog benchmark requiring retrieval over multiple manuals with automatic BLEU and ROUGE scoring.", "active_modalities": [ "Text" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following", "Domain Knowledge", "Long-Horizon Memory" ], "cap_count": 6, "cap_score": 0.6667, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "multidoc2dial2021", "citation": { "title": "MultiDoc2Dial", "authors": "Song Feng and Siva Sankalp Patel and Hui Wan and Sachindra Joshi", "year": 2021, "venue": "Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing (EMNLP)", "bibtex": "@inproceedings{multidoc2dial2021,\n title={{MultiDoc2Dial}: Modeling Dialogues Grounded in Multiple Documents},\n author={Song Feng and Siva Sankalp Patel and Hui Wan and Sachindra Joshi},\n booktitle={Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing (EMNLP)},\n year={2021},\n howpublished={\\url{https://arxiv.org/abs/2109.12595}},\n}" } }, { "id": "cocotext2016", "name": "COCO-Text", "symbol": "COC", "year": 2016, "venue": "ECCV", "url": "https://arxiv.org/abs/1601.07140", "family": "Vision-Language", "task_family": "OCR", "domain": "General", "metrics": "Precision;Recall", "judge_type": "auto", "protocol_notes": "Large-scale dataset for text spotting in natural images; evaluation via precision and recall on word detection and recognition.", "active_modalities": [ "Image" ], "active_capabilities": [ "Perception", "Grounding" ], "cap_count": 2, "cap_score": 0.2222, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "cocotext2016", "citation": { "title": "COCO-Text", "authors": "Andreas Veit and Tomas Matera and Lukas Neumann and Jiri Matas and Serge Belongie", "year": 2016, "venue": "arXiv preprint arXiv:1601.07140", "bibtex": "@article{cocotext2016,\n title={{COCO-Text}: Dataset and Benchmark for Text Detection and Recognition in Natural Images},\n author={Andreas Veit and Tomas Matera and Lukas Neumann and Jiri Matas and Serge Belongie},\n journal={arXiv preprint arXiv:1601.07140},\n year={2016},\n howpublished={\\url{https://arxiv.org/abs/1601.07140}},\n}" } }, { "id": "recipeqa2018", "name": "RecipeQA", "symbol": "REC", "year": 2018, "venue": "EMNLP", "url": "https://arxiv.org/abs/1809.00812", "family": "Vision-Language", "task_family": "MultiModalQA", "domain": "Cooking", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Cooking comprehension benchmark mixing text steps and images; four tasks scored via accuracy on multiple-choice answers.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following", "Domain Knowledge", "Long-Horizon Memory" ], "cap_count": 6, "cap_score": 0.6667, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "recipeqa2018", "citation": { "title": "RecipeQA", "authors": "Semih Yagcioglu and Aykut Erdem and Erkut Erdem and Nazli Ikizler-Cinbis", "year": 2018, "venue": "Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing (EMNLP)", "bibtex": "@inproceedings{recipeqa2018,\n title={{RecipeQA}: A Challenge Dataset for Multimodal Comprehension of Cooking Recipes},\n author={Semih Yagcioglu and Aykut Erdem and Erkut Erdem and Nazli Ikizler-Cinbis},\n booktitle={Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing (EMNLP)},\n year={2018},\n howpublished={\\url{https://arxiv.org/abs/1809.00812}},\n}" } }, { "id": "hatefulmemes2020", "name": "HatefulMemes", "symbol": "HAT", "year": 2020, "venue": "NeurIPS", "url": "https://arxiv.org/abs/2005.04790", "family": "Vision-Language", "task_family": "MultimodalClassification", "domain": "Safety", "metrics": "ROC-AUC", "judge_type": "auto", "protocol_notes": "Binary meme toxicity dataset mixing text and image cues; leaderboard reports ROC-AUC and accuracy on hidden test labels.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Domain Knowledge" ], "cap_count": 4, "cap_score": 0.4444, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "hatefulmemes2020", "citation": { "title": "The Hateful Memes Challenge: Detecting Hate Speech in Multimodal Memes", "authors": "Douwe Kiela and Hamed Firooz and Aravind Mohan and Vedanuj Goswami and Amanpreet Singh and Pratik Ringshia and Davide Testuggine", "year": 2020, "venue": "Advances in Neural Information Processing Systems (NeurIPS)", "bibtex": "@inproceedings{hatefulmemes2020,\n title={The Hateful Memes Challenge: Detecting Hate Speech in Multimodal Memes},\n author={Douwe Kiela and Hamed Firooz and Aravind Mohan and Vedanuj Goswami and Amanpreet Singh and Pratik Ringshia and Davide Testuggine},\n booktitle={Advances in Neural Information Processing Systems (NeurIPS)},\n year={2020},\n howpublished={\\url{https://arxiv.org/abs/2005.04790}},\n}" } }, { "id": "mmimdb2018", "name": "MM-IMDb", "symbol": "MMI", "year": 2018, "venue": "ACMMM", "url": "https://arxiv.org/abs/1809.02135", "family": "Vision-Language", "task_family": "MultilabelClassification", "domain": "Entertainment", "metrics": "mAP;F1", "judge_type": "auto", "protocol_notes": "Movie genre classification from posters plus plots; multi-label metrics computed automatically on validation/test splits.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Domain Knowledge" ], "cap_count": 4, "cap_score": 0.4444, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "mmimdb2018", "citation": { "title": "MM-IMDb", "authors": "John Arevalo and Thamar Solorio and Manuel Montes-y-G\\'o", "year": 2017, "venue": "ACM International Conference on Multimedia Retrieval (ICMR)", "bibtex": "@inproceedings{mmimdb2018,\n title={{MM-IMDb}: Multimodal Dataset for Genre Prediction},\n author={John Arevalo and Thamar Solorio and Manuel Montes-y-G{\\'o}mez and Fabio A. Gonz{\\'a}lez},\n booktitle={ACM International Conference on Multimedia Retrieval (ICMR)},\n year={2017},\n howpublished={\\url{https://arxiv.org/abs/1809.02135}},\n}" } }, { "id": "fashioniq2019", "name": "FashionIQ", "symbol": "FAS", "year": 2019, "venue": "CVPR", "url": "https://arxiv.org/abs/1905.12794", "family": "Vision-Language", "task_family": "Retrieval", "domain": "Fashion", "metrics": "R@10", "judge_type": "auto", "protocol_notes": "Natural language based fashion retrieval benchmark combining reference images and human feedback; evaluation via recall at K.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following" ], "cap_count": 4, "cap_score": 0.4444, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "fashioniq2019", "citation": { "title": "The Fashion-IQ", "authors": "Hui Wu and Yupeng Gao and Xiaoxiao Guo and Alborz Rezazadeh and Scott Cohen and Zhaowen Wang and Suhang Wang", "year": 2021, "venue": "IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)", "bibtex": "@inproceedings{fashioniq2019,\n title={The {Fashion-IQ} Dataset: Retrieving Images by Combining Visual and Interactive Attributes},\n author={Hui Wu and Yupeng Gao and Xiaoxiao Guo and Alborz Rezazadeh and Scott Cohen and Zhaowen Wang and Suhang Wang},\n booktitle={IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},\n year={2021},\n howpublished={\\url{https://arxiv.org/abs/1905.12794}},\n}" } }, { "id": "vatex2019", "name": "VATEX", "symbol": "VAT", "year": 2019, "venue": "ICCV", "url": "https://arxiv.org/abs/1904.03493", "family": "Video & Temporal", "task_family": "VideoCaptioning", "domain": "General", "metrics": "BLEU;CIDEr", "judge_type": "auto", "protocol_notes": "Large multilingual video captioning dataset; scoring uses standard caption metrics on held-out videos.", "active_modalities": [ "Text", "Video" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Long-Horizon Memory" ], "cap_count": 4, "cap_score": 0.4444, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "vatex2019", "citation": { "title": "VATEX", "authors": "Xin Wang and Jiawei Wu and Junkun Chen and Lei Li and Yuan-Fang Wang and William Yang Wang", "year": 2019, "venue": "IEEE/CVF International Conference on Computer Vision (ICCV)", "bibtex": "@inproceedings{vatex2019,\n title={{VATEX}: A Large-Scale, High-Quality Multilingual Dataset for Video-and-Language Research},\n author={Xin Wang and Jiawei Wu and Junkun Chen and Lei Li and Yuan-Fang Wang and William Yang Wang},\n booktitle={IEEE/CVF International Conference on Computer Vision (ICCV)},\n year={2019},\n howpublished={\\url{https://arxiv.org/abs/1904.03493}},\n}" } }, { "id": "coin2019", "name": "COIN", "symbol": "COI", "year": 2019, "venue": "CVPR", "url": "https://arxiv.org/abs/1903.02874", "family": "Video & Temporal", "task_family": "InstructionalVideo", "domain": "Instructional", "metrics": "mAP", "judge_type": "auto", "protocol_notes": "Comprehensive instructional video dataset with 11k clips and step annotations; evaluation via segment-level mAP over predicted procedures.", "active_modalities": [ "Text", "Video" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following", "Domain Knowledge", "Long-Horizon Memory" ], "cap_count": 6, "cap_score": 0.6667, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "coin2019", "citation": { "title": "COIN", "authors": "Yansong Tang and Dajun Ding and Yongming Rao and Yue Zheng and Danyang Zhang and Lili Zhao and Jiwen Lu and Jie Zhou", "year": 2019, "venue": "IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)", "bibtex": "@inproceedings{coin2019,\n title={{COIN}: A Large-scale Dataset for Comprehensive Instructional Video Analysis},\n author={Yansong Tang and Dajun Ding and Yongming Rao and Yue Zheng and Danyang Zhang and Lili Zhao and Jiwen Lu and Jie Zhou},\n booktitle={IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},\n year={2019},\n howpublished={\\url{https://arxiv.org/abs/1903.02874}},\n}" } }, { "id": "moviefib2016", "name": "MovieFIB", "symbol": "MO3", "year": 2016, "venue": "arXiv", "url": "https://arxiv.org/abs/1611.07810", "family": "Video & Temporal", "task_family": "VideoQA", "domain": "Entertainment", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Fill-in-the-blank question answering on movie clips with automatic exact-match scoring.", "active_modalities": [ "Text", "Video" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Domain Knowledge", "Long-Horizon Memory" ], "cap_count": 5, "cap_score": 0.5556, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "moviefib2016", "citation": { "title": "A Large-Scale Fill-in-the-Blank Dataset for Video Understanding", "authors": "Tairu Qiu and Qing Luan and Wei Liu and Zicheng Liu", "year": 2016, "venue": "arXiv preprint arXiv:1611.07810", "bibtex": "@article{moviefib2016,\n title={A Large-Scale Fill-in-the-Blank Dataset for Video Understanding},\n author={Tairu Qiu and Qing Luan and Wei Liu and Zicheng Liu},\n journal={arXiv preprint arXiv:1611.07810},\n year={2016},\n howpublished={\\url{https://arxiv.org/abs/1611.07810}},\n}" } }, { "id": "activitynetcaptions2017", "name": "ActivityNetCaptions", "symbol": "AC2", "year": 2017, "venue": "CVPR", "url": "https://arxiv.org/abs/1705.00754", "family": "Video & Temporal", "task_family": "VideoCaptioning", "domain": "General", "metrics": "BLEU;CIDEr", "judge_type": "auto", "protocol_notes": "Dense event captioning benchmark over ActivityNet videos; evaluation uses standard caption metrics and temporal IoU alignment.", "active_modalities": [ "Text", "Video" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Long-Horizon Memory" ], "cap_count": 4, "cap_score": 0.4444, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "activitynetcaptions2017", "citation": { "title": "Dense-Captioning Events in Videos", "authors": "Ranjay Krishna and Kenji Hata and Frederic Ren and Li Fei-Fei and Juan Carlos Niebles", "year": 2017, "venue": "IEEE International Conference on Computer Vision (ICCV)", "bibtex": "@inproceedings{activitynetcaptions2017,\n title={Dense-Captioning Events in Videos},\n author={Ranjay Krishna and Kenji Hata and Frederic Ren and Li Fei-Fei and Juan Carlos Niebles},\n booktitle={IEEE International Conference on Computer Vision (ICCV)},\n year={2017},\n howpublished={\\url{https://arxiv.org/abs/1705.00754}},\n}" } }, { "id": "ego4dqa2022", "name": "Ego4D-NLQ", "symbol": "EG3", "year": 2022, "venue": "CVPR", "url": "https://arxiv.org/abs/2110.07058", "family": "Video & Temporal", "task_family": "VideoGrounding", "domain": "Egocentric", "metrics": "mAP", "judge_type": "auto", "protocol_notes": "Natural language query localization on multi-minute egocentric videos; evaluated via mean Average Precision over temporal segments.", "active_modalities": [ "Text", "Video" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Long-Horizon Memory" ], "cap_count": 4, "cap_score": 0.4444, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "ego4dqa2022", "citation": { "title": "Ego4D: Egocentric Video Dataset and Benchmark Suite", "authors": "Grauman, Kristen and others", "year": 2022, "venue": "CVPR", "bibtex": "@inproceedings{ego4dqa2022,\n title={Ego4D: Egocentric Video Dataset and Benchmark Suite},\n author={Grauman, Kristen and others},\n booktitle={CVPR},\n year={2022},\n url={https://arxiv.org/abs/2110.07058}\n}" } }, { "id": "scan2cap2020", "name": "Scan2Cap", "symbol": "SC3", "year": 2020, "venue": "CVPR", "url": "https://arxiv.org/abs/2012.02206", "family": "3D & Spatial", "task_family": "3DCaptioning", "domain": "3D", "metrics": "BLEU;CIDEr", "judge_type": "auto", "protocol_notes": "Dense captioning of RGB-D scans linking language to 3D regions; evaluation uses BLEU/CIDEr metrics on predicted sentences.", "active_modalities": [ "Text", "Image", "3D" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning" ], "cap_count": 3, "cap_score": 0.3333, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "scan2cap2020", "citation": { "title": "Scan2Cap", "authors": "Dave Zhenyu Chen and Angel X. Chang and Matthias Nie\\ss", "year": 2021, "venue": "IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)", "bibtex": "@inproceedings{scan2cap2020,\n title={{Scan2Cap}: Context-aware Dense Captioning in {RGB-D} Scans},\n author={Dave Zhenyu Chen and Angel X. Chang and Matthias Nie{\\ss}ner},\n booktitle={IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},\n year={2021},\n howpublished={\\url{https://arxiv.org/abs/2012.02206}},\n}" } }, { "id": "igibson2021", "name": "iGibson2", "symbol": "IGI", "year": 2021, "venue": "CoRL", "url": "https://arxiv.org/abs/2108.03272", "family": "Embodied & Agents", "task_family": "Manipulation", "domain": "Household", "metrics": "SuccessRate", "judge_type": "auto", "protocol_notes": "Object-centric embodied benchmark spanning navigation and manipulation tasks; automatic success computed per task and scene.", "active_modalities": [ "Text", "Image", "3D", "Sensor", "Action" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following", "Domain Knowledge", "Planning", "Embodiment" ], "cap_count": 7, "cap_score": 0.7778, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "igibson2021", "citation": { "title": "iGibson 2.0", "authors": "Chengshu Li and Fei Xia and Roberto Mart\\'i", "year": 2021, "venue": "Conference on Robot Learning (CoRL)", "bibtex": "@inproceedings{igibson2021,\n title={{iGibson 2.0}: Object-Centric Simulation for Robot Learning of Everyday Household Tasks},\n author={Chengshu Li and Fei Xia and Roberto Mart{\\'i}n-Mart{\\'i}n and Michael Lingelbach and Sanjana Srivastava and Bokui Shen and Kent Vainio and Cem Gokmen and Gokul Dharan and Manasi Sharma and Jiaming Song and Rishabh Joshi and C. Karen Liu and Silvio Savarese and Fei-Fei Li},\n booktitle={Conference on Robot Learning (CoRL)},\n year={2021},\n howpublished={\\url{https://arxiv.org/abs/2108.03272}},\n}" } }, { "id": "objectnav2019", "name": "ObjectNav-Habitat", "symbol": "OB2", "year": 2019, "venue": "CVPR", "url": "https://arxiv.org/abs/1904.00581", "family": "Embodied & Agents", "task_family": "Navigation", "domain": "Indoor", "metrics": "SuccessRate;SPL", "judge_type": "auto", "protocol_notes": "Habitat ObjectGoal navigation challenge where agents receive object names and must explore; metrics include success and SPL.", "active_modalities": [ "Text", "Image", "Sensor", "Action" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following", "Planning", "Embodiment" ], "cap_count": 6, "cap_score": 0.6667, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "objectnav2019", "citation": { "title": "On Evaluation of Embodied Agents Navigating to Objects", "authors": "Dhruv Batra and Aaron Gokaslan and Aniruddha Kembhavi and Stefan Lee and Dhruv Maurya and Rishabh Jain and Abhishek Das and Stefan Lee", "year": 2020, "venue": "arXiv preprint arXiv:2003.11531", "bibtex": "@article{objectnav2019,\n title={On Evaluation of Embodied Agents Navigating to Objects},\n author={Dhruv Batra and Aaron Gokaslan and Aniruddha Kembhavi and Stefan Lee and Dhruv Maurya and Rishabh Jain and Abhishek Das and Stefan Lee},\n journal={arXiv preprint arXiv:2003.11531},\n year={2020},\n howpublished={\\url{https://arxiv.org/abs/1904.00581}},\n}" } }, { "id": "lsmdc2016", "name": "LSMDC", "symbol": "LSM", "year": 2016, "venue": "ECCV", "url": "https://sites.google.com/site/describingmovies/lsmdc", "family": "Video & Temporal", "task_family": "VideoCaptioning", "domain": "Entertainment", "metrics": "BLEU;METEOR", "judge_type": "auto", "protocol_notes": "Large-scale movie description challenge featuring captioning and fill-in-the-blank tasks scored via standard caption metrics.", "active_modalities": [ "Text", "Video" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Domain Knowledge", "Long-Horizon Memory" ], "cap_count": 5, "cap_score": 0.5556, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "lsmdc2016", "citation": { "title": "A Large-Scale Video-and-Language Dataset for Movie Description", "authors": "Anna Rohrbach and Atousa Torabi and Marcus Rohrbach and Niket Tandon and Christopher Pal and Hugo Larochelle and Aaron Courville and Bernt Schiele", "year": 2016, "venue": "European Conference on Computer Vision (ECCV)", "bibtex": "@inproceedings{lsmdc2016,\n title={A Large-Scale Video-and-Language Dataset for Movie Description},\n author={Anna Rohrbach and Atousa Torabi and Marcus Rohrbach and Niket Tandon and Christopher Pal and Hugo Larochelle and Aaron Courville and Bernt Schiele},\n booktitle={European Conference on Computer Vision (ECCV)},\n year={2016},\n howpublished={\\url{https://sites.google.com/site/describingmovies/lsmdc}},\n}" } }, { "id": "funsd2019", "name": "FUNSD", "symbol": "FUN", "year": 2019, "venue": "ICDAR", "url": "https://guillaumejaume.github.io/FUNSD/", "family": "Vision-Language", "task_family": "DocUnderstanding", "domain": "Forms", "metrics": "F1", "judge_type": "auto", "protocol_notes": "Form understanding dataset with semantic entity labeling and relationship extraction; evaluation via token F1 and entity accuracy.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning" ], "cap_count": 3, "cap_score": 0.3333, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "funsd2019", "citation": { "title": "Towards Complex Document Understanding by Matching Text and Image", "authors": "Guillaume Jaume and Hazim Kemal Ekenel and Jean-Philippe Thiran", "year": 2019, "venue": "International Conference on Document Analysis and Recognition Workshops (ICDARW)", "bibtex": "@inproceedings{funsd2019,\n title={Towards Complex Document Understanding by Matching Text and Image},\n author={Guillaume Jaume and Hazim Kemal Ekenel and Jean-Philippe Thiran},\n booktitle={International Conference on Document Analysis and Recognition Workshops (ICDARW)},\n year={2019},\n howpublished={\\url{https://guillaumejaume.github.io/FUNSD/}},\n}" } }, { "id": "cord2019", "name": "CORD", "symbol": "COR", "year": 2019, "venue": "ICDAR", "url": "https://github.com/clovaai/cord", "family": "Vision-Language", "task_family": "DocUnderstanding", "domain": "Receipts", "metrics": "F1", "judge_type": "auto", "protocol_notes": "Consolidated receipt OCR dataset measuring key information extraction; scores computed via field-level F1.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning" ], "cap_count": 3, "cap_score": 0.3333, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "cord2019", "citation": { "title": "CORD", "authors": "Seunghyun Park and Seung Shin and Bado Lee and Junyeop Lee and Jaeheung Surh and Minjoon Seo and Hwalsuk Lee", "year": 2019, "venue": "Document Intelligence Workshop at NeurIPS", "bibtex": "@inproceedings{cord2019,\n title={{CORD}: A Consolidated Receipt Dataset for Post-OCR Parsing},\n author={Seunghyun Park and Seung Shin and Bado Lee and Junyeop Lee and Jaeheung Surh and Minjoon Seo and Hwalsuk Lee},\n booktitle={Document Intelligence Workshop at NeurIPS},\n year={2019},\n howpublished={\\url{https://github.com/clovaai/cord}},\n}" } }, { "id": "sroie2019", "name": "SROIE", "symbol": "SRO", "year": 2019, "venue": "ICDAR", "url": "https://rrc.cvc.uab.es/?ch=13", "family": "Vision-Language", "task_family": "DocUnderstanding", "domain": "Receipts", "metrics": "F1", "judge_type": "auto", "protocol_notes": "Scanned receipt OCR and information extraction benchmark from ICDAR 2019; evaluation via detection and key-value F1.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning" ], "cap_count": 3, "cap_score": 0.3333, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "sroie2019", "citation": { "title": "ICDAR2019", "authors": "Zheng Huang and Kai Chen and Jianhua He and Xiang Bai and Dimosthenis Karatzas and Shiguang Shan and Guan-Hua Wang", "year": 2019, "venue": "International Conference on Document Analysis and Recognition (ICDAR)", "bibtex": "@inproceedings{sroie2019,\n title={{ICDAR2019} Competition on Scanned Receipts {OCR} and Information Extraction},\n author={Zheng Huang and Kai Chen and Jianhua He and Xiang Bai and Dimosthenis Karatzas and Shiguang Shan and Guan-Hua Wang},\n booktitle={International Conference on Document Analysis and Recognition (ICDAR)},\n year={2019},\n howpublished={\\url{https://rrc.cvc.uab.es/?ch=13}},\n}" } }, { "id": "miniwobpp2017", "name": "MiniWoB++", "symbol": "MI2", "year": 2017, "venue": "ICLRW", "url": "https://github.com/google-research/google-research/tree/master/miniwob", "family": "Embodied & Agents", "task_family": "WebAgent", "domain": "Web", "metrics": "SuccessRate", "judge_type": "auto", "protocol_notes": "Suite of micro web tasks requiring DOM manipulation; completion judged automatically by scripts per environment.", "active_modalities": [ "Text", "Image", "Action" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following", "Planning", "Tool Use", "Embodiment" ], "cap_count": 7, "cap_score": 0.7778, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "miniwobpp2017", "citation": { "title": "Reinforcement Learning on the Web with MiniWoB++", "authors": "Evan Zheran Liu and Kelvin Guu and Panupong Pasupat and Tianlin Shi and Percy Liang", "year": 2018, "venue": "International Conference on Learning Representations Workshop (ICLRW)", "bibtex": "@inproceedings{miniwobpp2017,\n title={Reinforcement Learning on the Web with {MiniWoB++}},\n author={Evan Zheran Liu and Kelvin Guu and Panupong Pasupat and Tianlin Shi and Percy Liang},\n booktitle={International Conference on Learning Representations Workshop (ICLRW)},\n year={2018},\n howpublished={\\url{https://github.com/google-research/google-research/tree/master/miniwob}},\n}" } }, { "id": "mmfashion2018", "name": "MMFashion", "symbol": "MMF", "year": 2018, "venue": "ACMMM", "url": "https://arxiv.org/abs/1811.08391", "family": "Vision-Language", "task_family": "MultilabelClassification", "domain": "Fashion", "metrics": "mAP", "judge_type": "auto", "protocol_notes": "Large fashion attribute benchmarking dataset combining text tags and product images with automatic multi-label scoring.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning" ], "cap_count": 3, "cap_score": 0.3333, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "mmfashion2018", "citation": { "title": "MMFashion", "authors": "Xin Liu and Jiaqi Wang and Ziwei Liu and Dahua Lin", "year": 2020, "venue": "ACM International Conference on Multimedia (ACM MM)", "bibtex": "@inproceedings{mmfashion2018,\n title={{MMFashion}: An Open-Source Visual Fashion Analysis Toolbox in {PyTorch}},\n author={Xin Liu and Jiaqi Wang and Ziwei Liu and Dahua Lin},\n booktitle={ACM International Conference on Multimedia (ACM MM)},\n year={2020},\n howpublished={\\url{https://arxiv.org/abs/1811.08391}},\n}" } }, { "id": "wav2sql2023", "name": "Wav2SQL", "symbol": "WAV", "year": 2023, "venue": "ACL", "url": "https://arxiv.org/abs/2305.12552", "family": "Audio & Speech", "task_family": "Speech2SQL", "domain": "Data", "metrics": "ExecutionAccuracy", "judge_type": "auto", "protocol_notes": "Speech-to-SQL benchmark with 26k spoken questions mapped to executable SQL queries; evaluation via exact string match and database execution accuracy.", "active_modalities": [ "Text", "Audio" ], "active_capabilities": [ "Perception", "Reasoning", "Instruction Following", "Tool Use" ], "cap_count": 4, "cap_score": 0.4444, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "wav2sql2023", "citation": { "title": "Wav2SQL: A Large-Scale Speech-to-SQL Dataset for Spoken Language Interface to Databases", "authors": "Liu, Huadai and Huang, Rongjie and He, Jinzheng and Sun, Gang and Shen, Ran and Cheng, Xize and Zhao, Zhou", "year": 2024, "venue": "Findings of the Association for Computational Linguistics (ACL)", "bibtex": "@inproceedings{wav2sql2023,\n title={Wav2SQL: A Large-Scale Speech-to-SQL Dataset for Spoken Language Interface to Databases},\n author={Liu, Huadai and Huang, Rongjie and He, Jinzheng and Sun, Gang and Shen, Ran and Cheng, Xize and Zhao, Zhou},\n booktitle={Findings of the Association for Computational Linguistics (ACL)},\n year={2024},\n url={https://arxiv.org/abs/2305.12552}\n}" } }, { "id": "slurp2020", "name": "SLURP", "symbol": "SLU", "year": 2020, "venue": "EMNLP", "url": "https://arxiv.org/abs/2011.13205", "family": "Audio & Speech", "task_family": "SpokenNLU", "domain": "Assistant", "metrics": "IntentF1;SlotF1", "judge_type": "auto", "protocol_notes": "Large-scale spoken assistant dataset covering 18 domains with transcribed and spoken utterances; automatic scoring uses intent F1 and slot tagging F1 against API call annotations.", "active_modalities": [ "Text", "Audio" ], "active_capabilities": [ "Perception", "Reasoning", "Instruction Following", "Tool Use" ], "cap_count": 4, "cap_score": 0.4444, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "slurp2020", "citation": { "title": "SLURP: A Spoken Language Understanding Resource Package", "authors": "Bastianelli, Emanuele and others", "year": 2020, "venue": "EMNLP", "bibtex": "@inproceedings{slurp2020,\n title={SLURP: A Spoken Language Understanding Resource Package},\n author={Bastianelli, Emanuele and others},\n booktitle={EMNLP},\n year={2020}\n}" } }, { "id": "egoplanbench2023", "name": "EgoPlan-Bench", "symbol": "EP1", "year": 2023, "venue": "arXiv", "url": "https://arxiv.org/abs/2312.06722", "family": "Embodied & Agents", "task_family": "PlanningBench", "domain": "Egocentric", "metrics": "PlanAccuracy;LLMScore", "judge_type": "hybrid", "protocol_notes": "Egocentric video planning benchmark that asks models to produce multi-step action plans from wearable video clips; scored via action alignment metrics plus GPT-4 rubric grading.", "active_modalities": [ "Text", "Video", "Action" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following", "Long-Horizon Memory", "Planning", "Embodiment" ], "cap_count": 7, "cap_score": 0.7778, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "egoplanbench2023", "citation": { "title": "EgoPlan-Bench: Benchmarking Multimodal Large Language Models for Human-Level Planning", "authors": "Chen, Yi and Tang, Jie and others", "year": 2023, "venue": "arXiv preprint arXiv:2312.06722", "bibtex": "@article{egoplanbench2023,\n title={EgoPlan-Bench: Benchmarking Multimodal Large Language Models for Human-Level Planning},\n author={Chen, Yi and Tang, Jie and others},\n journal={arXiv preprint arXiv:2312.06722},\n year={2023}\n}" } }, { "id": "egoplanbench2_2024", "name": "EgoPlan-Bench2", "symbol": "EP2", "year": 2024, "venue": "arXiv", "url": "https://arxiv.org/abs/2412.04447", "family": "Embodied & Agents", "task_family": "PlanningBench", "domain": "Egocentric", "metrics": "PlanAccuracy", "judge_type": "auto", "protocol_notes": "Expanded Ego4D-based planning benchmark featuring diverse real-world egocentric clips and language goals; models predict the next action from multiple choices scored automatically.", "active_modalities": [ "Text", "Video", "Action" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following", "Long-Horizon Memory", "Planning", "Embodiment" ], "cap_count": 7, "cap_score": 0.7778, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "egoplanbench22024", "citation": { "title": "EgoPlan-Bench2: A Benchmark for Long-Horizon Egocentric Video Planning", "authors": "Qiu, Lu and Ge, Yuying and others", "year": 2024, "venue": "arXiv preprint arXiv:2412.04447", "bibtex": "@article{egoplanbench22024,\n title={EgoPlan-Bench2: A Benchmark for Long-Horizon Egocentric Video Planning},\n author={Qiu, Lu and Ge, Yuying and others},\n journal={arXiv preprint arXiv:2412.04447},\n year={2024},\n url={https://arxiv.org/abs/2412.04447}\n}" } }, { "id": "embodiedbench2025", "name": "EmbodiedBench", "symbol": "EM2", "year": 2025, "venue": "ICML", "url": "https://arxiv.org/abs/2502.09560", "family": "Embodied & Agents", "task_family": "EmbodiedSuite", "domain": "General", "metrics": "SuccessRate", "judge_type": "auto", "protocol_notes": "Comprehensive multi-domain embodied suite across navigation and manipulation envs with RGB/RGB-D inputs and language goals; evaluation uses automated task success in each domain.", "active_modalities": [ "Text", "Image", "Video", "Sensor", "Action" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following", "Long-Horizon Memory", "Planning", "Embodiment" ], "cap_count": 7, "cap_score": 0.7778, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "embodiedbench2025", "citation": { "title": "EmbodiedBench: A Comprehensive Benchmark for Multi-modal Embodied Agents", "authors": "Yang, Rui and Chen, Hanyang and Zhang, Junyu and Zhao, Mark and others", "year": 2025, "venue": "International Conference on Machine Learning (ICML)", "bibtex": "@inproceedings{embodiedbench2025,\n title={EmbodiedBench: A Comprehensive Benchmark for Multi-modal Embodied Agents},\n author={Yang, Rui and Chen, Hanyang and Zhang, Junyu and Zhao, Mark and others},\n booktitle={International Conference on Machine Learning (ICML)},\n year={2025},\n url={https://arxiv.org/abs/2502.09560}\n}" } }, { "id": "ovmm2023", "name": "OVMM-HomeRobot", "symbol": "OVM", "year": 2023, "venue": "CoRL", "url": "https://arxiv.org/abs/2306.11565", "family": "Embodied & Agents", "task_family": "MobileManipulation", "domain": "Household", "metrics": "SuccessRate", "judge_type": "auto", "protocol_notes": "Open-vocabulary mobile manipulation benchmark where an agent follows spoken prompts to find objects and place them in target receptacles using onboard RGB-D sensing; scores via success in sim and real runs.", "active_modalities": [ "Text", "Image", "Sensor", "Action" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following", "Long-Horizon Memory", "Planning", "Embodiment" ], "cap_count": 7, "cap_score": 0.7778, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "ovmm2023", "citation": { "title": "HomeRobot: Open-Vocabulary Mobile Manipulation", "authors": "Yenamandra, Sriram and Ramachandran, Arun and Yadav, Karmesh and Wang, Austin and others", "year": 2023, "venue": "Conference on Robot Learning (CoRL)", "bibtex": "@inproceedings{ovmm2023,\n title={HomeRobot: Open-Vocabulary Mobile Manipulation},\n author={Yenamandra, Sriram and Ramachandran, Arun and Yadav, Karmesh and Wang, Austin and others},\n booktitle={Conference on Robot Learning (CoRL)},\n year={2023},\n url={https://arxiv.org/abs/2306.11565}\n}" } }, { "id": "voiceagentbench2025", "name": "VoiceAgentBench", "symbol": "VOI", "year": 2025, "venue": "arXiv", "url": "https://arxiv.org/abs/2510.07978", "family": "Audio & Speech", "task_family": "SpeechToolUse", "domain": "General", "metrics": "ToolSelect;ToolStructure;ParamFill;RefusalRate", "judge_type": "hybrid", "protocol_notes": "5.5k multilingual spoken queries spanning single/multi-tool workflows and adversarial safety tasks; TS/TCS metrics are exact-match scripts while parameter filling and refusal checks rely on GPT-4o judges over ground-truth API graphs.", "active_modalities": [ "Audio" ], "active_capabilities": [ "Perception", "Reasoning", "Instruction Following", "Planning", "Tool Use" ], "cap_count": 5, "cap_score": 0.5556, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "voiceagentbench2025", "citation": { "title": "VoiceAgentBench: Benchmarking Multimodal Voice Agents for Real-World Task Execution", "authors": "Jain, Dhruv and Shukla, Harshit and Rajeev, Gautam and others", "year": 2025, "venue": "arXiv preprint arXiv:2510.07978", "bibtex": "@article{voiceagentbench2025,\n title={VoiceAgentBench: Benchmarking Multimodal Voice Agents for Real-World Task Execution},\n author={Jain, Dhruv and Shukla, Harshit and Rajeev, Gautam and others},\n journal={arXiv preprint arXiv:2510.07978},\n year={2025},\n url={https://arxiv.org/abs/2510.07978}\n}" } }, { "id": "speakql2020", "name": "SPEAKQL", "symbol": "SPE", "year": 2020, "venue": "SIGMOD", "url": "https://adalabucsd.github.io/speakql.html", "family": "Audio & Speech", "task_family": "Speech2SQL", "domain": "Data", "metrics": "ExecutionAccuracy", "judge_type": "auto", "protocol_notes": "Speech-to-SQL dataset where users dictate queries over databases; scoring via exact-match SQL or execution accuracy of generated statements.", "active_modalities": [ "Audio" ], "active_capabilities": [ "Perception", "Reasoning", "Instruction Following", "Tool Use" ], "cap_count": 4, "cap_score": 0.4444, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "speakql2020", "citation": { "title": "SpeakQL: Towards Speech-driven Multimodal Querying of Structured Data", "authors": "Shah, Vraj and Li, Side and Kumar, Arun and Saul, Lawrence K.", "year": 2020, "venue": "Proceedings of the ACM SIGMOD International Conference on Management of Data", "bibtex": "@inproceedings{speakql2020,\n title={SpeakQL: Towards Speech-driven Multimodal Querying of Structured Data},\n author={Shah, Vraj and Li, Side and Kumar, Arun and Saul, Lawrence K.},\n booktitle={Proceedings of the ACM SIGMOD International Conference on Management of Data},\n year={2020},\n url={https://adalabucsd.github.io/speakql.html}\n}" } }, { "id": "voicebench2024", "name": "VoiceBench", "symbol": "VO2", "year": 2024, "venue": "TACL", "url": "https://arxiv.org/abs/2410.17196", "family": "Audio & Speech", "task_family": "VoiceAssistant", "domain": "General", "metrics": "QAAccuracy;Robustness", "judge_type": "auto", "protocol_notes": "Aggregated speech-eval suite (CommonEval, SD-QA, AlpacaEval, IFEval, AdvBench) mixing real and synthetic utterances with noise/accents to assess factual QA, instruction following, and safety; pipeline ASR+LLM scores reported against end-to-end SpeechLMs.", "active_modalities": [ "Audio" ], "active_capabilities": [ "Perception", "Reasoning", "Instruction Following", "Domain Knowledge", "Tool Use" ], "cap_count": 5, "cap_score": 0.5556, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "voicebench2024", "citation": { "title": "VoiceBench: Benchmarking LLM-Based Voice Assistants across Real-World Dialog Tasks", "authors": "Chen, Yiming and Yue, Xianghu and Zhang, Chen and Gao, Xiaoxue and Tan, Robby T. and Li, Haizhou", "year": 2026, "venue": "Transactions of the Association for Computational Linguistics (TACL)", "bibtex": "@article{voicebench2024,\n title={VoiceBench: Benchmarking LLM-Based Voice Assistants across Real-World Dialog Tasks},\n author={Chen, Yiming and Yue, Xianghu and Zhang, Chen and Gao, Xiaoxue and Tan, Robby T. and Li, Haizhou},\n journal={Transactions of the Association for Computational Linguistics (TACL)},\n year={2026},\n url={https://arxiv.org/abs/2410.17196}\n}" } }, { "id": "voiceassistanteval2025", "name": "VoiceAssistant-Eval", "symbol": "VO3", "year": 2025, "venue": "arXiv", "url": "https://arxiv.org/abs/2509.22651", "family": "Audio & Speech", "task_family": "AssistantEval", "domain": "General", "metrics": "ContentScore;MOS;Consistency", "judge_type": "hybrid", "protocol_notes": "Benchmark of 10k listening/speaking/viewing tasks where assistants answer questions about audio clips, conduct spoken dialogues, and analyze images; scoring fuses LLM-judged content accuracy, MOS-based speech quality, and transcript-vs-intent consistency with open-source data.", "active_modalities": [ "Text", "Image", "Audio" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following", "Domain Knowledge", "Long-Horizon Memory" ], "cap_count": 6, "cap_score": 0.6667, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "voiceassistanteval2025", "citation": { "title": "VoiceAssistant-Eval: Benchmarking AI", "authors": "Wang, Ke and Ren, Houxing and Lu, Zimu and Zhan, Mingjie and Li, Hongsheng", "year": 2025, "venue": "arXiv preprint arXiv:2509.22651", "bibtex": "@article{voiceassistanteval2025,\n title={VoiceAssistant-Eval: Benchmarking {AI} Assistants across Listening, Speaking, and Viewing},\n author={Wang, Ke and Ren, Houxing and Lu, Zimu and Zhan, Mingjie and Li, Hongsheng},\n journal={arXiv preprint arXiv:2509.22651},\n year={2025}\n}" } }, { "id": "voxeval2025", "name": "VoxEval", "symbol": "VOX", "year": 2025, "venue": "ACL", "url": "https://arxiv.org/abs/2501.04962", "family": "Audio & Speech", "task_family": "SpeechQA", "domain": "General", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Purely spoken QA benchmark keeping both queries and responses in audio; evaluates world-knowledge reasoning and robustness to accents/noise by transcribing answers and scoring accuracy over ~280 audio QA pairs.", "active_modalities": [ "Audio" ], "active_capabilities": [ "Perception", "Reasoning", "Domain Knowledge" ], "cap_count": 3, "cap_score": 0.3333, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "voxeval2025", "citation": { "title": "VoxEval: A Comprehensive Benchmark for Spoken Large Language Models", "authors": "Cui, Wenqian and Jiao, Xiaoqi and Meng, Ziqiao and King, Irwin", "year": 2025, "venue": "Proceedings of the Association for Computational Linguistics (ACL)", "bibtex": "@inproceedings{voxeval2025,\n title={VoxEval: A Comprehensive Benchmark for Spoken Large Language Models},\n author={Cui, Wenqian and Jiao, Xiaoqi and Meng, Ziqiao and King, Irwin},\n booktitle={Proceedings of the Association for Computational Linguistics (ACL)},\n year={2025},\n url={https://arxiv.org/abs/2501.04962}\n}" } }, { "id": "cava2025", "name": "CAVA", "symbol": "CAV", "year": 2025, "venue": "Project", "url": "https://talkarena.org/cava", "family": "Audio & Speech", "task_family": "FunctionCalling", "domain": "Assistant", "metrics": "FunctionCallMatch", "judge_type": "auto", "protocol_notes": "Speech-in/speech-out benchmark where models map spoken requests to JSON API calls; leaderboard reports exact call match rate and latency across diverse assistant skills.", "active_modalities": [ "Text", "Audio" ], "active_capabilities": [ "Perception", "Reasoning", "Instruction Following", "Tool Use" ], "cap_count": 4, "cap_score": 0.4444, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "cava2025", "citation": { "title": "Comprehensive Assessment for Voice Assistants (CAVA)", "authors": "Stanford ILIAD and USC Collaborators", "year": 2025, "venue": "", "bibtex": "@misc{cava2025,\n title={Comprehensive Assessment for Voice Assistants (CAVA)},\n author = {{Stanford ILIAD and USC Collaborators}},\n year={2025},\n howpublished={\\url{https://talkarena.org/cava}},\n note={Accessed 2025-02-17}\n}" } }, { "id": "uitronspeech2025", "name": "UITron-Speech", "symbol": "UIT", "year": 2025, "venue": "arXiv", "url": "https://arxiv.org/abs/2506.11127", "family": "Embodied & Agents", "task_family": "WebAgent", "domain": "Apps", "metrics": "TaskSuccess;GroundingAccuracy", "judge_type": "auto", "protocol_notes": "Speech-driven GUI control benchmark pairing audio instructions with UI screenshots and DOM action traces; evaluation reports action success and grounding accuracy versus text-based agents.", "active_modalities": [ "Text", "Image", "Audio", "Action" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following", "Planning", "Tool Use", "Embodiment" ], "cap_count": 7, "cap_score": 0.7778, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "uitronspeech2025", "citation": { "title": "UITron-Speech: Desktop UI Automation via Spoken Language Grounding", "authors": "Han, Wenkang and others", "year": 2025, "venue": "arXiv preprint arXiv:2506.11127", "bibtex": "@article{uitronspeech2025,\n title={UITron-Speech: Desktop UI Automation via Spoken Language Grounding},\n author={Han, Wenkang and others},\n journal={arXiv preprint arXiv:2506.11127},\n year={2025},\n url={https://arxiv.org/abs/2506.11127}\n}" } }, { "id": "mshab2024", "name": "ManiSkill-HAB", "symbol": "MAN", "year": 2024, "venue": "ICLR", "url": "https://arxiv.org/abs/2412.13211", "family": "Embodied & Agents", "task_family": "Manipulation", "domain": "Household", "metrics": "SuccessRate", "judge_type": "auto", "protocol_notes": "Home Assistant Benchmark extension with GPU-accelerated simulator for long-horizon rearrangement using dual RGB-D cameras and low-level control; success defined by goal completion without unsafe collisions.", "active_modalities": [ "Image", "3D", "Sensor", "Action" ], "active_capabilities": [ "Perception", "Reasoning", "Instruction Following", "Planning", "Embodiment" ], "cap_count": 5, "cap_score": 0.5556, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "mshab2024", "citation": { "title": "ManiSkill-HAB: Benchmarking Mobile Manipulation in Habitat", "authors": "Shukla, Arth and Tao, Stone and Su, Hao", "year": 2025, "venue": "International Conference on Learning Representations (ICLR)", "bibtex": "@inproceedings{mshab2024,\n title={ManiSkill-HAB: Benchmarking Mobile Manipulation in Habitat},\n author={Shukla, Arth and Tao, Stone and Su, Hao},\n booktitle={International Conference on Learning Representations (ICLR)},\n year={2025},\n url={https://arxiv.org/abs/2412.13211}\n}" } }, { "id": "openxembodiment2023", "name": "Open X-Embodiment", "symbol": "OPE", "year": 2023, "venue": "arXiv", "url": "https://arxiv.org/abs/2310.08864", "family": "Embodied & Agents", "task_family": "Manipulation", "domain": "Robotics", "metrics": "PolicySuccess", "judge_type": "auto", "protocol_notes": "Aggregated 1M+ real-robot demonstrations over 22 embodiments with synchronized video, proprioception, and actions; evaluation reports multi-task success of RT-X policies transferred to held-out robots/tasks.", "active_modalities": [ "Image", "Video", "Sensor", "Action" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Planning", "Embodiment" ], "cap_count": 5, "cap_score": 0.5556, "cost_tier": 4, "cost_tier_label": "Tier 4: Extreme (Physical / Robotics)", "paper_citation_key": "openxembodiment2023", "citation": { "title": "Open X-Embodiment: Robotic Learning Datasets and Benchmarks at Scale", "authors": "Team, Open X-Embodiment and others", "year": 2023, "venue": "arXiv preprint arXiv:2310.08864", "bibtex": "@article{openxembodiment2023,\n title={Open X-Embodiment: Robotic Learning Datasets and Benchmarks at Scale},\n author={Team, Open X-Embodiment and others},\n journal={arXiv preprint arXiv:2310.08864},\n year={2023},\n url={https://arxiv.org/abs/2310.08864}\n}" } }, { "id": "languagetable2022", "name": "Language-Table", "symbol": "LAN", "year": 2022, "venue": "arXiv", "url": "https://arxiv.org/abs/2210.06407", "family": "Embodied & Agents", "task_family": "Manipulation", "domain": "Robotics", "metrics": "SuccessRate", "judge_type": "auto", "protocol_notes": "442k real-robot episodes where language instructions drive block rearrangement with RGB video and proprioception; success automatically checks final block configurations on held-out instructions.", "active_modalities": [ "Text", "Image", "Video", "Sensor", "Action" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following", "Planning", "Embodiment" ], "cap_count": 6, "cap_score": 0.6667, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "languagetable2022", "citation": { "title": "Interactive Language: Talking to Robots in Real Time", "authors": "Corey Lynch and Ayzaan Wahid and Jonathan Tompson and Tianli Ding and Betheny Bethel and Fei Xia and Michael Lepert and Pete Florence", "year": 2023, "venue": "IEEE Robotics and Automation Letters (RA-L)", "bibtex": "@inproceedings{languagetable2022,\n title={Interactive Language: Talking to Robots in Real Time},\n author={Corey Lynch and Ayzaan Wahid and Jonathan Tompson and Tianli Ding and Betheny Bethel and Fei Xia and Michael Lepert and Pete Florence},\n booktitle={IEEE Robotics and Automation Letters (RA-L)},\n year={2023},\n howpublished={\\url{https://arxiv.org/abs/2210.06407}},\n}" } }, { "id": "humanoideveryday2025", "name": "Humanoid Everyday", "symbol": "HUM", "year": 2025, "venue": "arXiv", "url": "https://arxiv.org/abs/2510.08807", "family": "Embodied & Agents", "task_family": "Manipulation", "domain": "Robotics", "metrics": "TaskSuccess", "judge_type": "auto", "protocol_notes": "10k+ teleoperated humanoid trajectories with RGB, depth, LiDAR, force-torque, and language summaries across 260 tasks; remote evaluation server scores task success and skill category performance for submitted policies.", "active_modalities": [ "Text", "Image", "Video", "3D", "Sensor", "Action" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following", "Planning", "Embodiment" ], "cap_count": 6, "cap_score": 0.6667, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "humanoideveryday2025", "citation": { "title": "Humanoid Everyday: A Comprehensive Robotic Dataset for Open-World Humanoid Manipulation", "authors": "Zhao, Zhenyu and Jing, Hongyi and others", "year": 2025, "venue": "arXiv preprint arXiv:2510.08807", "bibtex": "@article{humanoideveryday2025,\n title={Humanoid Everyday: A Comprehensive Robotic Dataset for Open-World Humanoid Manipulation},\n author={Zhao, Zhenyu and Jing, Hongyi and others},\n journal={arXiv preprint arXiv:2510.08807},\n year={2025},\n url={https://arxiv.org/abs/2510.08807}\n}" } }, { "id": "rlbench2019", "name": "RLBench", "symbol": "RLB", "year": 2019, "venue": "arXiv", "url": "https://arxiv.org/abs/1909.12271", "family": "Embodied & Agents", "task_family": "Manipulation", "domain": "Robotics", "metrics": "SuccessRate", "judge_type": "auto", "protocol_notes": "Simulated robot-learning suite with 100+ language-conditioned tool-use tasks; agents observe RGB-D and proprioceptive sensors and success is measured automatically per episode.", "active_modalities": [ "Text", "Image", "Sensor", "Action" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following", "Planning", "Tool Use", "Embodiment" ], "cap_count": 7, "cap_score": 0.7778, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "rlbench2019", "citation": { "title": "RLBench: The Robot Learning Benchmark \\& Learning Environment", "authors": "James, Stephen and Ma, Michael and others", "year": 2019, "venue": "arXiv preprint arXiv:1909.12271", "bibtex": "@article{rlbench2019,\n title={RLBench: The Robot Learning Benchmark \\& Learning Environment},\n author={James, Stephen and Ma, Michael and others},\n journal={arXiv preprint arXiv:1909.12271},\n year={2019}\n}" } }, { "id": "robomimic2021", "name": "RoboMimic", "symbol": "RO2", "year": 2021, "venue": "arXiv", "url": "https://arxiv.org/abs/2108.03298", "family": "Embodied & Agents", "task_family": "Manipulation", "domain": "Robotics", "metrics": "NormalizedScore", "judge_type": "auto", "protocol_notes": "Large-scale manipulation benchmark of human demos (Can, Lift, Square, Transport) with RGB-D and proprioception and high-DOF actions; evaluation via normalized success scores under offline RL protocols.", "active_modalities": [ "Image", "3D", "Sensor", "Action" ], "active_capabilities": [ "Perception", "Reasoning", "Planning", "Tool Use", "Embodiment" ], "cap_count": 5, "cap_score": 0.5556, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "robomimic2021", "citation": { "title": "RoboMimic: A Framework for Robot Learning from Demonstrations", "authors": "Mandlekar, Ajay and others", "year": 2021, "venue": "arXiv preprint arXiv:2108.03298", "bibtex": "@article{robomimic2021,\n title={RoboMimic: A Framework for Robot Learning from Demonstrations},\n author={Mandlekar, Ajay and others},\n journal={arXiv preprint arXiv:2108.03298},\n year={2021}\n}" } }, { "id": "calvin2021", "name": "CALVIN", "symbol": "CAL", "year": 2021, "venue": "arXiv", "url": "https://arxiv.org/abs/2112.03227", "family": "Embodied & Agents", "task_family": "Manipulation", "domain": "Robotics", "metrics": "SuccessRate", "judge_type": "auto", "protocol_notes": "Language-conditioned long-horizon manipulation benchmark with RGB-D observations and proprioception and tool/object interactions; agents execute textual instructions and are scored via automatic success.", "active_modalities": [ "Text", "Image", "3D", "Sensor", "Action" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following", "Planning", "Tool Use", "Embodiment" ], "cap_count": 7, "cap_score": 0.7778, "cost_tier": 4, "cost_tier_label": "Tier 4: Extreme (Physical / Robotics)", "paper_citation_key": "calvin2021", "citation": { "title": "CALVIN: A Benchmark for Language-Conditioned Policy Learning for Long-Horizon Robot Manipulation Tasks", "authors": "Mees, Oier and others", "year": 2021, "venue": "arXiv preprint arXiv:2112.03227", "bibtex": "@article{calvin2021,\n title={CALVIN: A Benchmark for Language-Conditioned Policy Learning for Long-Horizon Robot Manipulation Tasks},\n author={Mees, Oier and others},\n journal={arXiv preprint arXiv:2112.03227},\n year={2021}\n}" } }, { "id": "ego4dfhp2022", "name": "Ego4D-FutureHands", "symbol": "EG4", "year": 2022, "venue": "CVPR", "url": "https://ego4d-data.org/tasks/future-hand-prediction/", "family": "Embodied & Agents", "task_family": "ActionForecast", "domain": "Egocentric", "metrics": "Top1;Top5", "judge_type": "auto", "protocol_notes": "Ego4D challenge where models observe egocentric video and forecast future hand-object interactions/actions, evaluated via Top-K anticipation accuracy.", "active_modalities": [ "Video", "Action" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Long-Horizon Memory", "Planning" ], "cap_count": 5, "cap_score": 0.5556, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "ego4dfhp2022", "citation": { "title": "Ego4D Future Hand Prediction Challenge", "authors": "Grauman, Kristen and others", "year": 2022, "venue": "CVPR Ego4D Workshop", "bibtex": "@inproceedings{ego4dfhp2022,\n title={Ego4D Future Hand Prediction Challenge},\n author={Grauman, Kristen and others},\n booktitle={CVPR Ego4D Workshop},\n year={2022},\n url={https://ego4d-data.org/tasks/future-hand-prediction/}\n}" } }, { "id": "epicltackitchens2020", "name": "EPIC-KITCHENS-100 Action Anticipation", "symbol": "EPI", "year": 2020, "venue": "arXiv", "url": "https://arxiv.org/abs/2006.13256", "family": "Video & Temporal", "task_family": "ActionAnticipation", "domain": "Egocentric", "metrics": "Top1;Top5", "judge_type": "auto", "protocol_notes": "Egocentric video benchmark for long-term action anticipation; models predict future actions from recent frames and are scored via Top-1/Top-5 accuracy.", "active_modalities": [ "Video" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Long-Horizon Memory", "Planning" ], "cap_count": 5, "cap_score": 0.5556, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "epicltackitchens2020", "citation": { "title": "EPIC-KITCHENS-100 Multi-Task Challenges", "authors": "Damen, Dima and others", "year": 2020, "venue": "arXiv preprint arXiv:2006.13256", "bibtex": "@article{epicltackitchens2020,\n title={EPIC-KITCHENS-100 Multi-Task Challenges},\n author={Damen, Dima and others},\n journal={arXiv preprint arXiv:2006.13256},\n year={2020}\n}" } }, { "id": "metaworld2020", "name": "Meta-World", "symbol": "MET", "year": 2020, "venue": "CoRL", "url": "https://arxiv.org/abs/1910.10897", "family": "Embodied & Agents", "task_family": "Manipulation", "domain": "Robotics", "metrics": "SuccessRate", "judge_type": "auto", "protocol_notes": "Benchmark of 50+ robot manipulation tasks (e.g., hammer / peg-insert) with RGB and proprioception; evaluates multi-task policies and tool-use success.", "active_modalities": [ "Image", "3D", "Sensor", "Action" ], "active_capabilities": [ "Perception", "Reasoning", "Planning", "Tool Use", "Embodiment" ], "cap_count": 5, "cap_score": 0.5556, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "metaworld2020", "citation": { "title": "Meta-World: A Benchmark and Evaluation for Multi-Task and Meta Reinforcement Learning", "authors": "Yu, Tianhe and others", "year": 2020, "venue": "CORL", "bibtex": "@inproceedings{metaworld2020,\n title={Meta-World: A Benchmark and Evaluation for Multi-Task and Meta Reinforcement Learning},\n author={Yu, Tianhe and others},\n booktitle={CORL},\n year={2020}\n}" } }, { "id": "d4rladroit2020", "name": "D4RL-Adroit", "symbol": "D4R", "year": 2020, "venue": "NeurIPS", "url": "https://arxiv.org/abs/2004.07219", "family": "Embodied & Agents", "task_family": "Manipulation", "domain": "Robotics", "metrics": "NormalizedScore", "judge_type": "auto", "protocol_notes": "Offline RL benchmark with dexterous hand tasks (pen, door, hammer); sensor-rich control with standardized normalized scores across datasets.", "active_modalities": [ "3D", "Sensor", "Action" ], "active_capabilities": [ "Perception", "Reasoning", "Planning", "Tool Use", "Embodiment" ], "cap_count": 5, "cap_score": 0.5556, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "d4rladroit2020", "citation": { "title": "D4RL: Datasets for Deep Data-Driven Reinforcement Learning", "authors": "Fu, Justin and others", "year": 2020, "venue": "arXiv preprint arXiv:2004.07219", "bibtex": "@article{d4rladroit2020,\n title={D4RL: Datasets for Deep Data-Driven Reinforcement Learning},\n author={Fu, Justin and others},\n journal={arXiv preprint arXiv:2004.07219},\n year={2020}\n}" } }, { "id": "ego4dqa2022", "name": "Ego4D-LTA", "symbol": "EG5", "year": 2022, "venue": "CVPR", "url": "https://arxiv.org/abs/2110.07058", "family": "Video & Temporal", "task_family": "ActionAnticipation", "domain": "Egocentric", "metrics": "Top1;Top5", "judge_type": "auto", "protocol_notes": "Long-term action anticipation challenge in the Ego4D suite; models observe egocentric video and predict future actions over extended horizons.", "active_modalities": [ "Video" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Long-Horizon Memory", "Planning" ], "cap_count": 5, "cap_score": 0.5556, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "ego4dqa2022", "citation": { "title": "Ego4D: Egocentric Video Dataset and Benchmark Suite", "authors": "Grauman, Kristen and others", "year": 2022, "venue": "CVPR", "bibtex": "@inproceedings{ego4dqa2022,\n title={Ego4D: Egocentric Video Dataset and Benchmark Suite},\n author={Grauman, Kristen and others},\n booktitle={CVPR},\n year={2022},\n url={https://arxiv.org/abs/2110.07058}\n}" } }, { "id": "realwebassist2025", "name": "RealWebAssist", "symbol": "REA", "year": 2025, "venue": "AAAI", "url": "https://arxiv.org/abs/2504.10445", "family": "Embodied & Agents", "task_family": "WebAgent", "domain": "Web", "metrics": "TaskSuccessRate;AverageProgress", "judge_type": "auto", "protocol_notes": "Speech-driven web assistance dataset of 1.8k voice instructions over live websites with audio, transcripts, screenshots, and DOM action traces; evaluation uses task success and average progress vs ground-truth action sequences.", "active_modalities": [ "Text", "Image", "Audio", "Action" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following", "Long-Horizon Memory", "Planning", "Tool Use" ], "cap_count": 7, "cap_score": 0.7778, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "realwebassist2025", "citation": { "title": "RealWebAssist: A Benchmark for Spoken Web Assistance with Multimodal Foundation Models", "authors": "Ye, Suyu and Shi, Haojun and Shih, Darren and Yun, Hyokun and Roosta, Tanya and Shu, Tianmin", "year": 2026, "venue": "Proceedings of the AAAI Conference on Artificial Intelligence", "bibtex": "@inproceedings{realwebassist2025,\n title={RealWebAssist: A Benchmark for Spoken Web Assistance with Multimodal Foundation Models},\n author={Ye, Suyu and Shi, Haojun and Shih, Darren and Yun, Hyokun and Roosta, Tanya and Shu, Tianmin},\n booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},\n year={2026},\n url={https://arxiv.org/abs/2504.10445}\n}" } }, { "id": "bfclaudio2025", "name": "BFCL-Audio", "symbol": "BFC", "year": 2025, "venue": "Salesforce", "url": "https://www.salesforce.com/blog/bfcl-audio-benchmark/", "family": "Audio & Speech", "task_family": "ToolUse", "domain": "Web", "metrics": "ASTAccuracy;DialogSuccess", "judge_type": "auto", "protocol_notes": "Speech-native function-calling benchmark with audio waveforms, transcripts, and multi-tool workflows; evaluates AST-based tool-call accuracy and multi-turn state checks.", "active_modalities": [ "Text", "Audio" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following", "Tool Use" ], "cap_count": 5, "cap_score": 0.5556, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "bfclaudio2025", "citation": { "title": "BFCL-Audio: Benchmarking Function-Calling for Voice Assistants", "authors": "Salesforce AI Research", "year": 2025, "venue": "", "bibtex": "@misc{bfclaudio2025,\n title={BFCL-Audio: Benchmarking Function-Calling for Voice Assistants},\n author = {{Salesforce AI Research}},\n year={2025},\n howpublished={\\url{https://www.salesforce.com/blog/bfcl-audio-benchmark/}},\n note={Accessed 2025-02-17}\n}" } }, { "id": "droid2024", "name": "DROID", "symbol": "DRO", "year": 2024, "venue": "Dataset", "url": "https://droid-dataset.github.io/", "family": "Embodied & Agents", "task_family": "Manipulation", "domain": "Robotics", "metrics": "PolicySuccess", "judge_type": "auto", "protocol_notes": "76k real-world teleoperation episodes with egocentric video and action logs across 84 tasks; used to train generalizable real-robot policies with success metrics on downstream tasks.", "active_modalities": [ "Text", "Image", "Video", "Action" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Long-Horizon Memory", "Planning", "Embodiment" ], "cap_count": 6, "cap_score": 0.6667, "cost_tier": 4, "cost_tier_label": "Tier 4: Extreme (Physical / Robotics)", "paper_citation_key": "droid2024", "citation": { "title": "DROID", "authors": "Alexander Khazatsky and Karl Pertsch and Suraj Nair and Ashwin Balakrishna and Sudeep Dasari and Siddharth Karamcheti and Soroush Nasiriany and Mohan Kumar Srirama and Lawrence Yunliang Chen and Kirsty Ellis and Peter David Fagan and others", "year": 2024, "venue": "Robotics: Science and Systems (RSS)", "bibtex": "@inproceedings{droid2024,\n title={{DROID}: A Large-Scale In-The-Wild Robot Manipulation Dataset},\n author={Alexander Khazatsky and Karl Pertsch and Suraj Nair and Ashwin Balakrishna and Sudeep Dasari and Siddharth Karamcheti and Soroush Nasiriany and Mohan Kumar Srirama and Lawrence Yunliang Chen and Kirsty Ellis and Peter David Fagan and others},\n booktitle={Robotics: Science and Systems (RSS)},\n year={2024},\n howpublished={\\url{https://droid-dataset.github.io/}},\n}" } }, { "id": "i2navrobot2025", "name": "i2Nav-Robot", "symbol": "I2N", "year": 2025, "venue": "arXiv", "url": "https://arxiv.org/abs/2508.11485", "family": "3D & Spatial", "task_family": "Navigation", "domain": "Robotics", "metrics": "LocalizationError;Robustness", "judge_type": "auto", "protocol_notes": "Large-scale indoor-outdoor navigation dataset logging synchronized LiDAR, radar, stereo, odometry, GNSS, and IMUs over 17 km trajectories with centimeter-accurate ground truth for benchmarking multi-sensor fusion.", "active_modalities": [ "Image", "Video", "3D", "Sensor" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Long-Horizon Memory", "Embodiment" ], "cap_count": 5, "cap_score": 0.5556, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "i2navrobot2025", "citation": { "title": "i2Nav-Robot: A Large-Scale Indoor-Outdoor Robot Dataset for Multi-Sensor Fusion Navigation and Mapping", "authors": "Tang, Hailiang and Zhang, Tisheng and Wang, Liqiang and Ding, Xin and others", "year": 2025, "venue": "arXiv preprint arXiv:2508.11485", "bibtex": "@article{i2navrobot2025,\n title={i2Nav-Robot: A Large-Scale Indoor-Outdoor Robot Dataset for Multi-Sensor Fusion Navigation and Mapping},\n author={Tang, Hailiang and Zhang, Tisheng and Wang, Liqiang and Ding, Xin and others},\n journal={arXiv preprint arXiv:2508.11485},\n year={2025}\n}" } }, { "id": "roboturk2019", "name": "RoboTurk-Real", "symbol": "RO3", "year": 2019, "venue": "Dataset", "url": "https://roboturk.stanford.edu/dataset_real.html", "family": "Embodied & Agents", "task_family": "Manipulation", "domain": "Robotics", "metrics": "TaskSuccess;VideoPrediction", "judge_type": "auto", "protocol_notes": "Remote human teleoperation of real robots with multi-view video and control traces for long-horizon tasks (laundry, block stacking); evaluation via imitation success and video/action prediction accuracy.", "active_modalities": [ "Image", "Video", "Action" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Long-Horizon Memory", "Planning", "Embodiment" ], "cap_count": 6, "cap_score": 0.6667, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "roboturk2019", "citation": { "title": "RoboTurk: A Crowdsourcing Platform for Robotic Teleoperation", "authors": "Mandlekar, Ajay and others", "year": 2019, "venue": "", "bibtex": "@misc{roboturk2019,\n title={RoboTurk: A Crowdsourcing Platform for Robotic Teleoperation},\n author={Mandlekar, Ajay and others},\n year={2019},\n howpublished={\\url{https://roboturk.stanford.edu/dataset_real.html}},\n note={Accessed 2019-10-01}\n}" } }, { "id": "habitat2hab2021", "name": "Habitat 2.0 HAB", "symbol": "HAB", "year": 2021, "venue": "ICCV", "url": "https://arxiv.org/abs/2106.14405", "family": "Embodied & Agents", "task_family": "Rearrangement", "domain": "Household", "metrics": "SuccessRate", "judge_type": "auto", "protocol_notes": "Physics-enabled home rearrangement tasks with RGB-D and proprio sensors; success requires placing objects at goal poses without force violations; Habitat server also tracks collisions and efficiency.", "active_modalities": [ "Image", "3D", "Sensor", "Action" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Planning", "Embodiment" ], "cap_count": 5, "cap_score": 0.5556, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "habitat2hab2021", "citation": { "title": "Habitat 2.0: Training Home Assistants to Rearrange Their Worlds", "authors": "Andrew Szot and Alex Clegg and Eric Undersander and Erik Wijmans and Yili Zhao and John Turner and Noah Maestre and Mustafa Mukadam and Devendra Singh Chaplot and Oleksandr Maksymets and Aaron Gokaslan and Vladimir Vondrus and Sameer Dharur and Franziska Meier and Wojciech Galuba and Angel Chang and Zsolt Kira and Vladlen Koltun and Jitendra Malik and Manolis Savva and Dhruv Batra", "year": 2021, "venue": "Advances in Neural Information Processing Systems (NeurIPS)", "bibtex": "@inproceedings{habitat2hab2021,\n title={Habitat 2.0: Training Home Assistants to Rearrange Their Worlds},\n author={Andrew Szot and Alex Clegg and Eric Undersander and Erik Wijmans and Yili Zhao and John Turner and Noah Maestre and Mustafa Mukadam and Devendra Singh Chaplot and Oleksandr Maksymets and Aaron Gokaslan and Vladimir Vondrus and Sameer Dharur and Franziska Meier and Wojciech Galuba and Angel Chang and Zsolt Kira and Vladlen Koltun and Jitendra Malik and Manolis Savva and Dhruv Batra},\n booktitle={Advances in Neural Information Processing Systems (NeurIPS)},\n year={2021},\n howpublished={\\url{https://arxiv.org/abs/2106.14405}},\n}" } }, { "id": "maniskill2021", "name": "ManiSkill", "symbol": "MA2", "year": 2021, "venue": "NeurIPS", "url": "https://openreview.net/pdf?id=SdWOpNTVJZc", "family": "Embodied & Agents", "task_family": "Manipulation", "domain": "Robotics", "metrics": "SuccessRate", "judge_type": "auto", "protocol_notes": "Manipulation challenge with RGB-D or point clouds plus proprioception; offline demonstrations and online RL tracks scored by success on seen and novel object instances.", "active_modalities": [ "Image", "3D", "Sensor", "Action" ], "active_capabilities": [ "Perception", "Reasoning", "Planning", "Embodiment" ], "cap_count": 4, "cap_score": 0.4444, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "maniskill2021", "citation": { "title": "ManiSkill", "authors": "Jiayuan Gu and Fanbo Xiang and Xuanlin Li and Zhan Ling and Xiqiang Liu and Tongzhou Mu and Yihe Tang and Stone Tao and Xinyue Wei and Yunchao Yao and Xiaodi Yuan and Pengwei Xie and Zhiao Huang and Rui Chen and Hao Su", "year": 2021, "venue": "Advances in Neural Information Processing Systems (NeurIPS)", "bibtex": "@inproceedings{maniskill2021,\n title={{ManiSkill}: Generalizable Manipulation Skill Benchmark with Large-Scale Demonstrations},\n author={Jiayuan Gu and Fanbo Xiang and Xuanlin Li and Zhan Ling and Xiqiang Liu and Tongzhou Mu and Yihe Tang and Stone Tao and Xinyue Wei and Yunchao Yao and Xiaodi Yuan and Pengwei Xie and Zhiao Huang and Rui Chen and Hao Su},\n booktitle={Advances in Neural Information Processing Systems (NeurIPS)},\n year={2021},\n howpublished={\\url{https://openreview.net/pdf?id=SdWOpNTVJZc}},\n}" } }, { "id": "dairv2x2022", "name": "DAIR-V2X", "symbol": "DAI", "year": 2022, "venue": "CVPR", "url": "https://arxiv.org/abs/2204.05575", "family": "3D & Spatial", "task_family": "Detection", "domain": "AutonomousDriving", "metrics": "mAP", "judge_type": "auto", "protocol_notes": "Vehicle-infrastructure cooperative perception benchmark with paired LiDAR and camera streams over 71k frames; evaluates 3D detection mAP with communication-aware settings.", "active_modalities": [ "Image", "3D", "Sensor" ], "active_capabilities": [ "Perception", "Grounding" ], "cap_count": 2, "cap_score": 0.2222, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "dairv2x2022", "citation": { "title": "DAIR-V2X", "authors": "Yu, Yang and Liu, Yuxuan and Gao, Peng and Zhang, Peng and others", "year": 2022, "venue": "CVPR", "bibtex": "@inproceedings{dairv2x2022,\n title={DAIR-{V2X}: A Large-Scale Dataset for Vehicle-Infrastructure Cooperative {3D} Object Detection},\n author={Yu, Yang and Liu, Yuxuan and Gao, Peng and Zhang, Peng and others},\n booktitle={CVPR},\n year={2022},\n eprint={2204.05575},\n archivePrefix={arXiv},\n url={https://arxiv.org/abs/2204.05575}\n}" } }, { "id": "v2xreal2024", "name": "V2X-Real", "symbol": "V2X", "year": 2024, "venue": "ECCV", "url": "https://mobility-lab.seas.ucla.edu/v2x-real/", "family": "3D & Spatial", "task_family": "Detection", "domain": "AutonomousDriving", "metrics": "mAP", "judge_type": "auto", "protocol_notes": "Four-way cooperative perception dataset with two cars and two roadside units; reports per-class AP and overall mAP across collaboration modes using the provided devkit.", "active_modalities": [ "Image", "3D", "Sensor" ], "active_capabilities": [ "Perception", "Grounding" ], "cap_count": 2, "cap_score": 0.2222, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "v2xreal2024", "citation": { "title": "V2X", "authors": "Ma, Jiaqi and Li, Zhi and Chen, Ziran and others", "year": 2024, "venue": "ECCV", "bibtex": "@inproceedings{v2xreal2024,\n title={{V2X}-Real: A Comprehensive Multi-Agent Dataset for Vehicle-to-Everything Cooperative Perception},\n author={Ma, Jiaqi and Li, Zhi and Chen, Ziran and others},\n booktitle={ECCV},\n year={2024},\n url={https://mobility-lab.seas.ucla.edu/v2x-real/}\n}" } }, { "id": "tumtrafv2x2024", "name": "TUMTraf-V2X", "symbol": "TUM", "year": 2024, "venue": "arXiv", "url": "https://arxiv.org/abs/2403.01316", "family": "3D & Spatial", "task_family": "Detection", "domain": "AutonomousDriving", "metrics": "mAP", "judge_type": "auto", "protocol_notes": "German urban V2X dataset combining vehicle and infrastructure LiDAR plus cameras for 3D detection and tracking; open-source toolkit computes mAP over challenging occlusion-heavy scenes.", "active_modalities": [ "Image", "3D", "Sensor" ], "active_capabilities": [ "Perception", "Grounding" ], "cap_count": 2, "cap_score": 0.2222, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "tumtrafv2x2024", "citation": { "title": "TUMTraf-V2X", "authors": "Paul, Martin and B\\", "year": 2024, "venue": "arXiv preprint arXiv:2403.01316", "bibtex": "@article{tumtrafv2x2024,\n title={TUMTraf-{V2X} Cooperative Perception Dataset},\n author={Paul, Martin and B{\\\"o}ttger, Steffen and others},\n journal={arXiv preprint arXiv:2403.01316},\n year={2024}\n}" } }, { "id": "webnav2025", "name": "WebNav", "symbol": "WE3", "year": 2025, "venue": "arXiv", "url": "https://arxiv.org/abs/2503.13843", "family": "Embodied & Agents", "task_family": "WebAgent", "domain": "Web", "metrics": "SuccessRate;Time", "judge_type": "human", "protocol_notes": "Voice-controlled web agent that labels DOM elements and uses ReAct LLM planning to issue clicks and typing actions; small study with visually impaired users reports higher success and faster completion than screen readers.", "active_modalities": [ "Text", "Audio", "Action" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following", "Planning", "Tool Use" ], "cap_count": 6, "cap_score": 0.6667, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "webnav2025", "citation": { "title": "WebNav: Evaluating Multimodal LLMs in Web-Scale Navigation Tasks", "authors": "Srinivasan, Trisanth", "year": 2025, "venue": "arXiv preprint arXiv:2503.13843", "bibtex": "@article{webnav2025,\n title={WebNav: Evaluating Multimodal LLMs in Web-Scale Navigation Tasks},\n author={Srinivasan, Trisanth},\n journal={arXiv preprint arXiv:2503.13843},\n year={2025},\n url={https://arxiv.org/abs/2503.13843}\n}" } }, { "id": "voicify2023", "name": "Voicify Your UI", "symbol": "VO4", "year": 2023, "venue": "IMWUT", "url": "https://arxiv.org/abs/2305.05198", "family": "Embodied & Agents", "task_family": "AppControl", "domain": "Mobile", "metrics": "SuccessRate;Time;TLX", "judge_type": "human", "protocol_notes": "Voice assistant for Android app control via UI hierarchy parsing and speech commands; evaluated on four multi-step tasks versus Google Voice Access using success time and NASA-TLX load.", "active_modalities": [ "Text", "Audio", "Action" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following", "Planning", "Tool Use" ], "cap_count": 6, "cap_score": 0.6667, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "voicify2023", "citation": { "title": "Voicify: Context-Aware Voice-Driven UI Automation for Mobile Applications", "authors": "Vu, Minh Duc and Wang, Han and Li, Zhuang and Haffari, Gholamreza and Xing, Zhenchang and Chen, Chunyang", "year": 2023, "venue": "Proceedings of the ACM on Interactive, Mobile, Wearable and Ubiquitous Technologies (IMWUT)", "bibtex": "@article{voicify2023,\n title={Voicify: Context-Aware Voice-Driven UI Automation for Mobile Applications},\n author={Vu, Minh Duc and Wang, Han and Li, Zhuang and Haffari, Gholamreza and Xing, Zhenchang and Chen, Chunyang},\n journal={Proceedings of the ACM on Interactive, Mobile, Wearable and Ubiquitous Technologies (IMWUT)},\n year={2023},\n url={https://arxiv.org/abs/2305.05198}\n}" } }, { "id": "gptvoicetasker2024", "name": "GPTVoiceTasker", "symbol": "GPT", "year": 2024, "venue": "UIST", "url": "https://arxiv.org/abs/2401.14268", "family": "Embodied & Agents", "task_family": "AppControl", "domain": "Mobile", "metrics": "SuccessRate;Time;SUS", "judge_type": "human", "protocol_notes": "LLM-driven voice automation that loops over ASR text and UI context to emit actions and stores reusable scripts; user study on six smartphone tasks measures success time and System Usability Scale.", "active_modalities": [ "Text", "Image", "Audio", "Action" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following", "Long-Horizon Memory", "Planning", "Tool Use" ], "cap_count": 7, "cap_score": 0.7778, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "gptvoicetasker2024", "citation": { "title": "GPTVoiceTasker: Advancing Multi-step Mobile Task Efficiency Through Dynamic Interface Exploration and Learning", "authors": "Vu, Minh Duc and Wang, Han and Chen, Jieshan and Li, Zhuang and Zhao, Shengdong and Xing, Zhenchang and Chen, Chunyang", "year": 2024, "venue": "Proceedings of the ACM Symposium on User Interface Software and Technology (UIST)", "bibtex": "@inproceedings{gptvoicetasker2024,\n title={GPTVoiceTasker: Advancing Multi-step Mobile Task Efficiency Through Dynamic Interface Exploration and Learning},\n author={Vu, Minh Duc and Wang, Han and Chen, Jieshan and Li, Zhuang and Zhao, Shengdong and Xing, Zhenchang and Chen, Chunyang},\n booktitle={Proceedings of the ACM Symposium on User Interface Software and Technology (UIST)},\n year={2024},\n url={https://arxiv.org/abs/2401.14268}\n}" } }, { "id": "multivox2025", "name": "MultiVox", "symbol": "MU2", "year": 2025, "venue": "EMNLP", "url": "https://arxiv.org/abs/2507.10859", "family": "Audio & Speech", "task_family": "AVQA", "domain": "General", "metrics": "Accuracy", "judge_type": "auto", "protocol_notes": "Spoken questions paired with images or videos including confounder versions; accuracy scored on 1k human-recorded multimodal dialogues that stress audio cues like emotion and background noise.", "active_modalities": [ "Text", "Image", "Audio", "Video" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning" ], "cap_count": 3, "cap_score": 0.3333, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "multivox2025", "citation": { "title": "MULTIVOX: A Benchmark for Evaluating Voice Assistants for Multimodal Interactions", "authors": "Selvakumar, Ramaneswaran and Seth, Ashish and Anand, Nishit and others", "year": 2025, "venue": "Proceedings of the Conference on Empirical Methods in Natural Language Processing (EMNLP)", "bibtex": "@inproceedings{multivox2025,\n title={MULTIVOX: A Benchmark for Evaluating Voice Assistants for Multimodal Interactions},\n author={Selvakumar, Ramaneswaran and Seth, Ashish and Anand, Nishit and others},\n booktitle={Proceedings of the Conference on Empirical Methods in Natural Language Processing (EMNLP)},\n year={2025},\n url={https://arxiv.org/abs/2507.10859}\n}" } }, { "id": "voxdialogue2025", "name": "VoxDialogue", "symbol": "VO5", "year": 2025, "venue": "ICLR", "url": "https://openreview.net/forum?id=vbmSSIhKAM", "family": "Audio & Speech", "task_family": "Dialog", "domain": "General", "metrics": "Accuracy;HumanRating", "judge_type": "hybrid", "protocol_notes": "4.5k multi-turn spoken dialogues where meaning depends on paralinguistic cues; responses judged by humans or LLM rubrics for cue awareness and appropriateness.", "active_modalities": [ "Text", "Audio" ], "active_capabilities": [ "Perception", "Reasoning", "Long-Horizon Memory" ], "cap_count": 3, "cap_score": 0.3333, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "voxdialogue2025", "citation": { "title": "VoxDialogue: Can Spoken Dialogue Systems Understand Information Beyond Words?", "authors": "Cheng, Xize and Hu, Ruofan and Yang, Xiaoda and Lu, Jingyu and others", "year": 2025, "venue": "International Conference on Learning Representations (ICLR)", "bibtex": "@inproceedings{voxdialogue2025,\n title={VoxDialogue: Can Spoken Dialogue Systems Understand Information Beyond Words?},\n author={Cheng, Xize and Hu, Ruofan and Yang, Xiaoda and Lu, Jingyu and others},\n booktitle={International Conference on Learning Representations (ICLR)},\n year={2025},\n url={https://openreview.net/forum?id=vbmSSIhKAM}\n}" } }, { "id": "s2sarena2025", "name": "S2S-Arena", "symbol": "S2S", "year": 2025, "venue": "ACL", "url": "https://arxiv.org/abs/2503.05085", "family": "Audio & Speech", "task_family": "Speech2Speech", "domain": "Assistant", "metrics": "HumanWinRate", "judge_type": "human", "protocol_notes": "154 spoken instructions across 21 tasks where models return speech; arena-style head-to-head human evaluation scores content correctness and prosody.", "active_modalities": [ "Audio" ], "active_capabilities": [ "Perception", "Reasoning", "Instruction Following" ], "cap_count": 3, "cap_score": 0.3333, "cost_tier": 2, "cost_tier_label": "Tier 2: $50-$150 (Multi-Turn / API)", "paper_citation_key": "s2sarena2025", "citation": { "title": "S2S-Arena: Evaluating Paralinguistic Instruction Following in Speech-to-Speech Models", "authors": "Jiang, Feng and Lin, Zhiyu and Liu, Yiyang and Xue, Liumeng and others", "year": 2026, "venue": "Proceedings of the Association for Computational Linguistics (ACL)", "bibtex": "@inproceedings{s2sarena2025,\n title={S2S-Arena: Evaluating Paralinguistic Instruction Following in Speech-to-Speech Models},\n author={Jiang, Feng and Lin, Zhiyu and Liu, Yiyang and Xue, Liumeng and others},\n booktitle={Proceedings of the Association for Computational Linguistics (ACL)},\n year={2026},\n url={https://arxiv.org/abs/2503.05085}\n}" } }, { "id": "timerssuch2021", "name": "Timers and Such", "symbol": "TIM", "year": 2021, "venue": "NeurIPSDatasets", "url": "https://openreview.net/forum?id=HrhaC-bLC5U", "family": "Audio & Speech", "task_family": "SpokenNLU", "domain": "Assistant", "metrics": "IntentAcc;SlotER", "judge_type": "auto", "protocol_notes": "Spoken timer and conversion commands with numeric SLU challenges; reports intent accuracy and slot error rate on human and synthetic audio.", "active_modalities": [ "Text", "Audio" ], "active_capabilities": [ "Perception", "Reasoning", "Instruction Following" ], "cap_count": 3, "cap_score": 0.3333, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "timerssuch2021", "citation": { "title": "Timers and Such: A Benchmark for Spoken Numeracy in Assistants", "authors": "Chung, Yu-An and others", "year": 2021, "venue": "NeurIPS Datasets and Benchmarks", "bibtex": "@inproceedings{timerssuch2021,\n title={Timers and Such: A Benchmark for Spoken Numeracy in Assistants},\n author={Chung, Yu-An and others},\n booktitle={NeurIPS Datasets and Benchmarks},\n year={2021},\n url={https://openreview.net/forum?id=HrhaC-bLC5U}\n}" } }, { "id": "pope2023", "name": "POPE", "symbol": "POP", "year": 2023, "venue": "EMNLP", "url": "https://arxiv.org/abs/2305.10355", "family": "Vision-Language", "task_family": "Diagnostic", "domain": "Hallucination", "metrics": "Accuracy;F1;Precision", "judge_type": "auto", "protocol_notes": "Polling-based object hallucination evaluation measuring whether models hallucinate non-existent objects under random, popular, and adversarial sampling.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following" ], "cap_count": 4, "cap_score": 0.4444, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "pope2023", "citation": { "title": "Evaluating Object Hallucination in Large Vision-Language Models", "authors": "Li, Yifan and Du, Yifan and Zhou, Kun and Wang, Jinpeng and Zhao, Wayne Xin and Wen, Ji-Rong", "year": 2023, "venue": "Proceedings of the Conference on Empirical Methods in Natural Language Processing (EMNLP)", "bibtex": "@inproceedings{pope2023,\n title={Evaluating Object Hallucination in Large Vision-Language Models},\n author={Li, Yifan and Du, Yifan and Zhou, Kun and Wang, Jinpeng and Zhao, Wayne Xin and Wen, Ji-Rong},\n booktitle={Proceedings of the Conference on Empirical Methods in Natural Language Processing (EMNLP)},\n year={2023},\n url={https://arxiv.org/abs/2305.10355}\n}" } }, { "id": "hallusionbench2024", "name": "HallusionBench", "symbol": "HAL", "year": 2024, "venue": "CVPR", "url": "https://arxiv.org/abs/2310.14566", "family": "Vision-Language", "task_family": "Diagnostic", "domain": "Hallucination", "metrics": "Accuracy;QuestionPairAcc", "judge_type": "auto", "protocol_notes": "Fine-grained diagnostic benchmark for visual illusion and language hallucination, measuring question-pair consistency across original and perturbed images.", "active_modalities": [ "Text", "Image" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following", "Domain Knowledge" ], "cap_count": 5, "cap_score": 0.5556, "cost_tier": 1, "cost_tier_label": "Tier 1: <$5 (Static / Smoke-Test)", "paper_citation_key": "hallusionbench2024", "citation": { "title": "HallusionBench: An Advanced Diagnostic Suite for Entangled Language Hallucination and Visual Illusion in Large Vision-Language Models", "authors": "Guan, Tianrui and Liu, Fuxiao and Wu, Xiyang and Xian, Rui and Li, Zongxia and Liu, Xiaoyu and Wang, Lingfeng and Chen, Chen and Huang, Furong and others", "year": 2024, "venue": "Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)", "bibtex": "@inproceedings{hallusionbench2024,\n title={HallusionBench: An Advanced Diagnostic Suite for Entangled Language Hallucination and Visual Illusion in Large Vision-Language Models},\n author={Guan, Tianrui and Liu, Fuxiao and Wu, Xiyang and Xian, Rui and Li, Zongxia and Liu, Xiaoyu and Wang, Lingfeng and Chen, Chen and Huang, Furong and others},\n booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},\n year={2024},\n url={https://arxiv.org/abs/2310.14566}\n}" } }, { "id": "osworld2024", "name": "OSWorld", "symbol": "OSW", "year": 2024, "venue": "NeurIPS", "url": "https://arxiv.org/abs/2404.07972", "family": "Embodied & Agents", "task_family": "WebAgent", "domain": "OSAutomation", "metrics": "SuccessRate", "judge_type": "auto", "protocol_notes": "Real-world multi-modal operating system agent benchmark across Ubuntu desktops; evaluates complex task completion using GUI screenshots, bash execution, and OS state change verification.", "active_modalities": [ "Text", "Image", "Action" ], "active_capabilities": [ "Perception", "Grounding", "Reasoning", "Instruction Following", "Long-Horizon Memory", "Planning", "Tool Use", "Embodiment" ], "cap_count": 8, "cap_score": 0.8889, "cost_tier": 3, "cost_tier_label": "Tier 3: >$400 (Sandbox / Agents)", "paper_citation_key": "osworld2024", "citation": { "title": "OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Operating Systems", "authors": "Xie, Tianbao and Zhang, Danyang and Chen, Jixuan and Li, Xiaochuan and Zhao, Siheng and Cao, Ruisheng and Zhou, Toh Jing and Wang, Lu and others", "year": 2024, "venue": "Advances in Neural Information Processing Systems (NeurIPS)", "bibtex": "@inproceedings{osworld2024,\n title={OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Operating Systems},\n author={Xie, Tianbao and Zhang, Danyang and Chen, Jixuan and Li, Xiaochuan and Zhao, Siheng and Cao, Ruisheng and Zhou, Toh Jing and Wang, Lu and others},\n booktitle={Advances in Neural Information Processing Systems (NeurIPS)},\n year={2024},\n url={https://arxiv.org/abs/2404.07972}\n}" } } ], "dark_matter": [ { "symbol": "Dm1", "name": "Real-Time Audio-Visual Interruption", "desc": "Continuous duplex voice-vision interaction with <200ms latency and dynamic turn-taking.", "category": "Temporal & Speech" }, { "symbol": "Dm2", "name": "Video-Grounded Tool & API Synthesis", "desc": "Zero benchmarks exist (mod_video + cap_tool = 0). Calling external calculators/web APIs over long video.", "category": "Tool & Video" }, { "symbol": "Dm3", "name": "Multi-Sensor Tactile-Proprioceptive Closed Loop", "desc": "Continuous high-frequency tactile arrays and torque sensing rather than visual approximations.", "category": "Embodied & Sensor" }, { "symbol": "Dm4", "name": "Non-Stationary Long-Horizon 3D Memory", "desc": "Continual spatial mapping and semantic object tracking over weeks-long temporal horizons in morphing physical scenes.", "category": "3D & Memory" }, { "symbol": "Dm5", "name": "Extreme Ultra-Long Omni-Modal Needle Retrieval", "desc": "Needle-in-a-haystack retrieval across 10^6+ tokens of interleaved waveforms, 4K frames, LiDAR, and text.", "category": "Omni-Modal Scale" }, { "symbol": "Dm6", "name": "Autonomous Multimodal Causal Counterfactuals", "desc": "Probing physical intervention hypotheses (what happens if X is removed before pushing Y) rather than observational co-occurrence.", "category": "Causality & Physics" }, { "symbol": "Dm7", "name": "Contamination-Immune Dynamic Canary Probes", "desc": "Procedurally generated, auto-mutating benchmarks with cryptographic canary watermarks to prevent pre-training data leakage.", "category": "Scientific Integrity" } ] }