WorldBench residency environment.

environments/worldbench/README.md

@@ -0,0 +1,51 @@
+# WorldBench
+
+WorldBench is a visually diverse multimodal reasoning benchmark with 2,000
+image-based multiple-choice questions spanning living things, objects, scenes, the
+digital world, academics, documents, and agents. See the
+[paper](https://arxiv.org/abs/2606.06538) and
+[official dataset](https://huggingface.co/datasets/zlab-princeton/WorldBench).
+
+## Implementation
+
+This environment provides WorldBench as a Verifiers taskset:
+
+- It streams a pinned revision of the Hugging Face dataset.
+- It implements the benchmark prompt, answer parser, and metadata.
+- It scores exact-match accuracy and reports answer parsing.
+- It preserves source images within a 10 MB encoded-payload and 8000-pixel
+  dimension limit, resizing only when necessary.
+- It supports direct evaluation with `NullHarness` and agent evaluation with harnesses
+  such as Codex.
+
+## Results
+
+The following direct evaluations used all 2,000 examples, one rollout per example,
+`NullHarness`, and a 32,768-token output cap:
+
+### General results
+
+| Model | Average accuracy | Living Things | Objects | Scenes | Digital World | Academics | Documents, Charts, and Tables | Agents |
+| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
+| Kimi K3 | 59.95% | 51.84% | 55.73% | 53.79% | 74.60% | 64.47% | 67.44% | 65.06% |
+| Kimi-K2.5 | 53.60% | 47.76% | 51.58% | 50.57% | 58.47% | 53.07% | 59.88% | 63.25% |
+| Qwen3.5-VL-35B-A3B | 48.30% | 41.22% | 46.64% | 49.43% | 50.00% | 51.32% | 50.58% | 51.81% |
+
+### Paper comparison
+
+| Model | Paper-reported | This environment | Difference |
+| --- | ---: | ---: | ---: |
+| Qwen3.5-VL-35B-A3B | 52.9% | 48.30% | -4.60 pp |
+| Kimi-K2.5 | 52.5% | 53.60% | +1.10 pp |
+
+#### Per-domain accuracy
+
+| Domain | Qwen paper | Qwen this environment | Kimi paper | Kimi this environment |
+| --- | ---: | ---: | ---: | ---: |
+| Living Things | 48.6% | 41.22% | 49.0% | 47.76% |
+| Objects | 52.6% | 46.64% | 52.0% | 51.58% |
+| Scenes | 52.2% | 49.43% | 49.7% | 50.57% |
+| Digital World | 50.4% | 50.00% | 56.9% | 58.47% |
+| Academics | 57.0% | 51.32% | 51.3% | 53.07% |
+| Documents, Charts, and Tables | 59.3% | 50.58% | 59.3% | 59.88% |
+| Agents | 54.2% | 51.81% | 54.8% | 63.25% |

environments/worldbench/pyproject.toml

@@ -0,0 +1,18 @@
+[project]
+name = "worldbench"
+version = "0.1.0"
+description = "WorldBench, a visually diverse multimodal reasoning benchmark, as a Verifiers taskset."
+readme = "README.md"
+requires-python = ">=3.11"
+dependencies = [
+    "datasets>=3.3,<6",
+    "pillow>=10",
+    "verifiers>=0.2.1",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["worldbench"]

environments/worldbench/worldbench/__init__.py

@@ -0,0 +1,3 @@
+from worldbench.taskset import WorldBenchTaskset
+
+__all__ = ["WorldBenchTaskset"]

environments/worldbench/worldbench/formatting.py

@@ -0,0 +1,36 @@
+"""Prompt formatting and answer parsing compatible with WorldBench."""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Mapping
+
+_ANSWER = re.compile(r"(?:^|\n)\s*ANSWER:\s*([A-D])\s*\Z", re.IGNORECASE)
+
+
+def format_prompt(question: str, options: Mapping[str, str]) -> str:
+    """Format one WorldBench multiple-choice question."""
+    return (
+        f"Question: {question}\n\n"
+        "Options:\n"
+        f"A: {options['A']}\n"
+        f"B: {options['B']}\n"
+        f"C: {options['C']}\n"
+        f"D: {options['D']}\n\n"
+        "Please answer the question and select the most appropriate answer from the given options.\n"
+        "First provide a brief explanation of your reasoning, then state your answer.\n\n"
+        "Your response must end with the answer line and contain nothing after it.\n"
+        "Format your response as:\n"
+        "Explanation: [Brief explanation of your reasoning]\n"
+        "Answer: [LETTER]"
+    )
+
+
+def parse_answer(raw_text: object) -> str:
+    """Extract an A-D answer from a model response."""
+    if not isinstance(raw_text, str) or not raw_text:
+        return ""
+
+    text = re.sub(r"[*_`]", " ", raw_text.strip().upper())
+    match = _ANSWER.search(text)
+    return match.group(1).upper() if match else ""

environments/worldbench/worldbench/images.py

@@ -0,0 +1,70 @@
+"""Image encoding for WorldBench."""
+
+from __future__ import annotations
+
+import base64
+from io import BytesIO
+
+from PIL import Image
+
+MAX_BASE64_BYTES = 10_000_000
+MAX_DIMENSION = 8000
+SCALES = (1.0, 0.75, 0.5, 0.25, 0.125)
+PASSTHROUGH_MEDIA_TYPES = frozenset({"image/jpeg", "image/png", "image/webp"})
+PIL_FORMATS = {
+    "image/jpeg": "JPEG",
+    "image/png": "PNG",
+    "image/webp": "WEBP",
+}
+LOSSY_QUALITIES = (90, 75, 60, 45, 30)
+
+
+def bounded_image_data_url(image_bytes: bytes) -> str:
+    """Preserve supported images and normalize other formats to bounded PNG."""
+    with Image.open(BytesIO(image_bytes)) as source:
+        media_type = Image.MIME[source.format]
+        encoded = base64.b64encode(image_bytes).decode("ascii")
+        if (
+            media_type in PASSTHROUGH_MEDIA_TYPES
+            and max(source.size) <= MAX_DIMENSION
+            and len(encoded) <= MAX_BASE64_BYTES
+        ):
+            return f"data:{media_type};base64,{encoded}"
+        target_media_type = media_type if media_type in PASSTHROUGH_MEDIA_TYPES else "image/png"
+        image = (
+            source.copy() if target_media_type == "image/png" and media_type == "image/png" else source.convert("RGB")
+        )
+
+    width, height = image.size
+    largest_dimension = max(width, height)
+    if largest_dimension > MAX_DIMENSION:
+        scale = MAX_DIMENSION / largest_dimension
+        image = image.resize(
+            (max(1, int(width * scale)), max(1, int(height * scale))),
+            Image.Resampling.LANCZOS,
+        )
+
+    for scale in SCALES:
+        candidate = image
+        if scale < 1.0:
+            width, height = image.size
+            candidate = image.resize(
+                (max(1, int(width * scale)), max(1, int(height * scale))),
+                Image.Resampling.LANCZOS,
+            )
+        qualities: tuple[int | None, ...] = (
+            LOSSY_QUALITIES if target_media_type in {"image/jpeg", "image/webp"} else (None,)
+        )
+        for quality in qualities:
+            output = BytesIO()
+            save_kwargs = {} if quality is None else {"quality": quality}
+            candidate.save(
+                output,
+                format=PIL_FORMATS[target_media_type],
+                **save_kwargs,
+            )
+            encoded = base64.b64encode(output.getvalue())
+            if len(encoded) <= MAX_BASE64_BYTES:
+                return f"data:{target_media_type};base64,{encoded.decode('ascii')}"
+
+    raise ValueError(f"image cannot be encoded under the {MAX_BASE64_BYTES}-byte limit")

environments/worldbench/worldbench/taskset.py

@@ -0,0 +1,75 @@
+"""WorldBench task data, deterministic scoring, and Hugging Face loader."""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+from typing import Literal, TypeAlias, cast
+
+import verifiers.v1 as vf
+from datasets import load_dataset
+
+from worldbench.formatting import format_prompt, parse_answer
+from worldbench.images import bounded_image_data_url
+
+DATASET_ID = "zlab-princeton/WorldBench"
+DATASET_REVISION = "c2d372ee38b4c2d486e0db8b9a35d24c7200998b"
+DATASET_SPLIT = "train"
+
+AnswerLabel: TypeAlias = Literal["A", "B", "C", "D"]
+
+
+class WorldBenchData(vf.TaskData):
+    question: str
+    options: dict[AnswerLabel, str]
+    answer: AnswerLabel
+    category: str
+    domain: str
+
+
+class WorldBenchTask(vf.Task[WorldBenchData]):
+    @vf.metric
+    async def answer_parsed(self, trace: vf.Trace) -> float:
+        return float(bool(parse_answer(trace.last_reply)))
+
+    @vf.reward
+    async def correct_answer(self, trace: vf.Trace) -> float:
+        return float(parse_answer(trace.last_reply) == self.data.answer)
+
+
+class WorldBenchTaskset(vf.Taskset[WorldBenchTask, vf.TasksetConfig]):
+    def load(self) -> Iterator[WorldBenchTask]:
+        dataset = load_dataset(
+            DATASET_ID,
+            revision=DATASET_REVISION,
+            split=DATASET_SPLIT,
+            streaming=True,
+        )
+        for idx, row in enumerate(dataset):
+            options: dict[AnswerLabel, str] = {
+                "A": row["option_a"],
+                "B": row["option_b"],
+                "C": row["option_c"],
+                "D": row["option_d"],
+            }
+            image_bytes = row["image"]["bytes"]
+            image_url = bounded_image_data_url(image_bytes)
+
+            prompt = [
+                vf.UserMessage(
+                    content=[
+                        vf.ImageUrlContentPart(image_url=vf.ImageUrlSource(url=image_url)),
+                        vf.TextContentPart(text=format_prompt(row["question"], options)),
+                    ]
+                )
+            ]
+            data = WorldBenchData(
+                idx=idx,
+                name=f"worldbench-{idx}",
+                prompt=prompt,
+                question=row["question"],
+                options=options,
+                answer=cast(AnswerLabel, row["answer"].strip().upper()),
+                category=row["category"],
+                domain=row["domain"],
+            )
+            yield WorldBenchTask(data, self.config.task)