Babillage is a multimodal benchmark dataset introduced along with MoshiVis (Project Page | arXiv), containing three common vision-language benchmarks converted in spoken form, for the evaluation of Vision Speech Models. For each benchmark (COCO-Captions, OCR-VQA, VQAv2), we first reformat the text question-answer pairs into a more conversational dialogue, and then convert them using a text-to-speech pipeline, using a consistent synthetic voice for the answer (assistant) and diverse voices for the question (user). We provide the resulting spoken samples in the dataset. The dataset is designed for research in conversational AI, spoken VQA, and multimodal dialogue systems.
Babillage contains the following dataset config and splits:
25010 samples, 5000 unique images) following Karpathy splits (including spoken answers)25010 samples, 5000 unique images) following Karpathy splits (including spoken answers)100032 samples, 20730 unique images) (including spoken answers)100424 samples, 20795 unique images) (including spoken answers)214354 samples, 40504 unique images)Each dataset sample consists of:
sample_id: Index of the corresponding sample in the base datasetimage_id: Image ID (only for CoOCR-VQA and CoCOCO)question_audio: The question in audio form.question_transcript: A list of words constituting the text transcript of question_audioquestion_alignment: A list of time spans tuples (start, end) consituting the alignment of each word in question_transcript to the audioanswer_audio: The answer in audio formanswer_transcript: A list of words constituting the text transcript of answer_audioanswer_alignment: A list of time spans tuple (start, end) consituting the alignment of each word in answer_transcript to the audioIn the following code snippets, we showcase how to merge Babillage splits with their corresponding
original vision-language dataset, to map the audio samples with the corresponding image and ground-truth.
Note that all audio files in this dataset are stored as ogg. If you want to obtain them as uncompressed
wav, you can use the following snippet:
import io
import soundfile as sf
import marimo as mo # example display with marimo notebook
def audio_widget(decoded_audio) -> Any:
"""A small snippet to convert to uncompressed audio (wav) and display the sample with marimo"""
byte_io = io.BytesIO(bytes())
sf.write(byte_io, decoded_audio["array"], samplerate=decoded_audio["sampling_rate"], format="wav")
return mo.audio(byte_io)
Below you'll also find a convenient code snippet to display one data sample from the Babillage dataset
from random import randint
import numpy as np
def display_one(dataset):
ditem = dataset[randint(0, len(dataset) - 1)]
print("Ground-truth:", ditem["gt"])
print("Question alignment:", [(w, ts) for w, ts in zip(ditem["question_transcript"],
ditem["question_alignment"])])
if "answer_audio" in ditem:
print("Answer alignment:", [(w, ts) for w, ts in zip(ditem["answer_transcript"],
ditem["answer_alignment"])])
return (mo.image(np.array(ditem["image"])),
audio_widget(ditem["question_audio"]),
audio_widget(ditem["answer_audio"]))
else:
return (mo.image(np.array(ditem["image"])),
audio_widget(ditem["question_audio"]))
import datasets
split = "validation"
coCOCO = datasets.load_dataset("kyutai/Babillage", "coco", split=split)
# Merge with COCO original information
original_columns = (
datasets.load_dataset("HuggingFaceM4/COCO", "2014", split=split)
.flatten()
.select_columns(["sentences.raw", "image", "cocoid"])
.cast_column('image', datasets.Image(decode=False))
)
coCOCO = coCOCO.add_column("og_image_id", original_columns["cocoid"])
coCOCO = coCOCO.add_column("gt", original_columns["sentences.raw"])
coCOCO = coCOCO.add_column("image", original_columns["image"])
coCOCO = coCOCO.cast_column('image', datasets.Image(decode=True))
del original_columns
# Display one sample
display_one(coCOCO)
import datasets
split = "validation"
coOCRVQA = datasets.load_dataset("kyutai/Babillage", "ocrvqa", split=split)
original_columns = (
datasets.load_dataset("howard-hou/OCR-VQA", split=split)
.flatten()
.select_columns(["image_id", "questions", "answers", "image"])
.cast_column('image', datasets.Image(decode=False))
)
num_samples_per_image = [len(x) for x in original_columns["questions"]]
coOCRVQA = coOCRVQA.add_column("og_image_id", [
id for n, id in zip(num_samples_per_image, original_columns["image_id"])
for _ in range(n)])
coOCRVQA = coOCRVQA.add_column(
"gt", [[quest, ans]
for qlst, alst in zip(original_columns["questions"], original_columns["answers"])
for quest, ans in zip(qlst, alst)])
coOCRVQA = coOCRVQA.add_column("image", [
img for n, img in zip(num_samples_per_image, original_columns["image"])
for _ in range(n)])
coOCRVQA = coOCRVQA.cast_column('image', datasets.Image(decode=True))
# Display one sample
display_one(coOCRVQA)
import datasets
split = "validation"
coVQA = datasets.load_dataset("kyutai/Babillage", "vqav2", split=split)
original_columns = (
datasets.load_dataset("lmms-lab/VQAv2", split=split)
.flatten()
.select_columns(["image_id", "question", "answers", "image"])
.cast_column('image', datasets.Image(decode=False)))
coVQA = coVQA.add_column("og_image_id", original_columns["image_id"])
coVQA = coVQA.add_column("gt", original_columns["answers"])
coVQA = coVQA.add_column("gt_question", original_columns["question"])
coVQA = coVQA.add_column("image", original_columns["image"])
coVQA = coVQA.cast_column('image', datasets.Image(decode=True))
# Display one sample
display_one(coVQA)
Babillage is licensed under the CC-BY 4.0 license.
If you use this dataset, please cite:
@article{kyutai2025moshivis,
author = {Amélie Royer and Moritz Böhle and Gabriel de Marmiesse and
Laurent Mazaré and Alexandre Défossez and Neil Zeghidour and Patrick Pérez},
year = {2025},
title = {Vision-Speech Models: Teaching Speech Models to Converse about Images},
journal = {ArXiv},
url = {https://arxiv.org/abs/2503.15633}
}
@misc{babillage2025,
title={{Babillage: A Conversational and Spoken VQA Dataset}},
author = {Amélie Royer, Moritz Böhle, Gabriel de Marmiesse, Laurent Mazaré, Alexandre Défossez, Neil Zeghidour, Patrick Pérez},
year={2025},
url={https://huggingface.co/datasets/kyutai/babillage}
}
Babillage is a multimodal benchmark dataset introduced along with MoshiVis (Project Page | arXiv), containing three common vision-language benchmarks converted in spoken form, for the evaluation of Vision Speech Models. For each benchmark (COCO-Captions, OCR-VQA, VQAv2), we first reformat the text question-answer pairs into a more conversational dialogue, and then convert them using a text-to-speech pipeline, using a consistent synthetic voice for the answer (assistant) and diverse voices for the question (user). We provide the resulting spoken samples in the dataset. The dataset is designed for research in conversational AI, spoken VQA, and multimodal dialogue systems.
Babillage contains the following dataset config and splits:
25010 samples, 5000 unique images) following Karpathy splits (including spoken answers)25010 samples, 5000 unique images) following Karpathy splits (including spoken answers)100032 samples, 20730 unique images) (including spoken answers)100424 samples, 20795 unique images) (including spoken answers)214354 samples, 40504 unique images)Each dataset sample consists of:
sample_id: Index of the corresponding sample in the base datasetimage_id: Image ID (only for CoOCR-VQA and CoCOCO)question_audio: The question in audio form.question_transcript: A list of words constituting the text transcript of question_audioquestion_alignment: A list of time spans tuples (start, end) consituting the alignment of each word in question_transcript to the audioanswer_audio: The answer in audio formanswer_transcript: A list of words constituting the text transcript of answer_audioanswer_alignment: A list of time spans tuple (start, end) consituting the alignment of each word in answer_transcript to the audioIn the following code snippets, we showcase how to merge Babillage splits with their corresponding
original vision-language dataset, to map the audio samples with the corresponding image and ground-truth.
Note that all audio files in this dataset are stored as ogg. If you want to obtain them as uncompressed
wav, you can use the following snippet:
import io
import soundfile as sf
import marimo as mo # example display with marimo notebook
def audio_widget(decoded_audio) -> Any:
"""A small snippet to convert to uncompressed audio (wav) and display the sample with marimo"""
byte_io = io.BytesIO(bytes())
sf.write(byte_io, decoded_audio["array"], samplerate=decoded_audio["sampling_rate"], format="wav")
return mo.audio(byte_io)
Below you'll also find a convenient code snippet to display one data sample from the Babillage dataset
from random import randint
import numpy as np
def display_one(dataset):
ditem = dataset[randint(0, len(dataset) - 1)]
print("Ground-truth:", ditem["gt"])
print("Question alignment:", [(w, ts) for w, ts in zip(ditem["question_transcript"],
ditem["question_alignment"])])
if "answer_audio" in ditem:
print("Answer alignment:", [(w, ts) for w, ts in zip(ditem["answer_transcript"],
ditem["answer_alignment"])])
return (mo.image(np.array(ditem["image"])),
audio_widget(ditem["question_audio"]),
audio_widget(ditem["answer_audio"]))
else:
return (mo.image(np.array(ditem["image"])),
audio_widget(ditem["question_audio"]))
import datasets
split = "validation"
coCOCO = datasets.load_dataset("kyutai/Babillage", "coco", split=split)
# Merge with COCO original information
original_columns = (
datasets.load_dataset("HuggingFaceM4/COCO", "2014", split=split)
.flatten()
.select_columns(["sentences.raw", "image", "cocoid"])
.cast_column('image', datasets.Image(decode=False))
)
coCOCO = coCOCO.add_column("og_image_id", original_columns["cocoid"])
coCOCO = coCOCO.add_column("gt", original_columns["sentences.raw"])
coCOCO = coCOCO.add_column("image", original_columns["image"])
coCOCO = coCOCO.cast_column('image', datasets.Image(decode=True))
del original_columns
# Display one sample
display_one(coCOCO)
import datasets
split = "validation"
coOCRVQA = datasets.load_dataset("kyutai/Babillage", "ocrvqa", split=split)
original_columns = (
datasets.load_dataset("howard-hou/OCR-VQA", split=split)
.flatten()
.select_columns(["image_id", "questions", "answers", "image"])
.cast_column('image', datasets.Image(decode=False))
)
num_samples_per_image = [len(x) for x in original_columns["questions"]]
coOCRVQA = coOCRVQA.add_column("og_image_id", [
id for n, id in zip(num_samples_per_image, original_columns["image_id"])
for _ in range(n)])
coOCRVQA = coOCRVQA.add_column(
"gt", [[quest, ans]
for qlst, alst in zip(original_columns["questions"], original_columns["answers"])
for quest, ans in zip(qlst, alst)])
coOCRVQA = coOCRVQA.add_column("image", [
img for n, img in zip(num_samples_per_image, original_columns["image"])
for _ in range(n)])
coOCRVQA = coOCRVQA.cast_column('image', datasets.Image(decode=True))
# Display one sample
display_one(coOCRVQA)
import datasets
split = "validation"
coVQA = datasets.load_dataset("kyutai/Babillage", "vqav2", split=split)
original_columns = (
datasets.load_dataset("lmms-lab/VQAv2", split=split)
.flatten()
.select_columns(["image_id", "question", "answers", "image"])
.cast_column('image', datasets.Image(decode=False)))
coVQA = coVQA.add_column("og_image_id", original_columns["image_id"])
coVQA = coVQA.add_column("gt", original_columns["answers"])
coVQA = coVQA.add_column("gt_question", original_columns["question"])
coVQA = coVQA.add_column("image", original_columns["image"])
coVQA = coVQA.cast_column('image', datasets.Image(decode=True))
# Display one sample
display_one(coVQA)
Babillage is licensed under the CC-BY 4.0 license.
If you use this dataset, please cite:
@article{kyutai2025moshivis,
author = {Amélie Royer and Moritz Böhle and Gabriel de Marmiesse and
Laurent Mazaré and Alexandre Défossez and Neil Zeghidour and Patrick Pérez},
year = {2025},
title = {Vision-Speech Models: Teaching Speech Models to Converse about Images},
journal = {ArXiv},
url = {https://arxiv.org/abs/2503.15633}
}
@misc{babillage2025,
title={{Babillage: A Conversational and Spoken VQA Dataset}},
author = {Amélie Royer, Moritz Böhle, Gabriel de Marmiesse, Laurent Mazaré, Alexandre Défossez, Neil Zeghidour, Patrick Pérez},
year={2025},
url={https://huggingface.co/datasets/kyutai/babillage}
}