From aea8a6c5c66137eee010326a778234d20016d699 Mon Sep 17 00:00:00 2001 From: ydnysh Date: Thu, 1 Oct 2026 12:01:45 +0530 Subject: [PATCH 1/4] Add Synthetic Workspace Gym taskset --- environments/synthetic_workspace_gym/LICENSE | 201 +++++ .../synthetic_workspace_gym/README.md | 73 ++ .../synthetic_workspace_gym/pyproject.toml | 21 + .../synthetic_workspace_gym/__init__.py | 17 + .../synthetic_workspace_gym/artifacts.json | 36 + .../evaluators/__init__.py | 3 + .../evaluators/base.py | 12 + .../evaluators/capabilities.py | 45 ++ .../evaluators/composite_workspace.py | 7 + .../evaluators/metrics.py | 160 ++++ .../evaluators/pipeline.py | 237 ++++++ .../evaluators/pipeline_profile.py | 155 ++++ .../evaluators/registry.py | 65 ++ .../evaluators/retrieval_workspace.py | 131 +++ .../evaluators/script_repair.py | 134 ++++ .../evaluators/tabular.py | 178 +++++ .../schemas/__init__.py | 33 + .../synthetic_workspace_gym/schemas/common.py | 287 +++++++ .../synthetic_workspace_gym/taskset.py | 749 ++++++++++++++++++ .../synthetic_workspace_gym/trusted_grader.py | 56 ++ .../synthetic_workspace_gym/utils/__init__.py | 1 + .../synthetic_workspace_gym/utils/io.py | 54 ++ .../synthetic_workspace_gym/utils/paths.py | 37 + .../synthetic_workspace_gym/utils/scratch.py | 18 + 24 files changed, 2710 insertions(+) create mode 100644 environments/synthetic_workspace_gym/LICENSE create mode 100644 environments/synthetic_workspace_gym/README.md create mode 100644 environments/synthetic_workspace_gym/pyproject.toml create mode 100644 environments/synthetic_workspace_gym/synthetic_workspace_gym/__init__.py create mode 100644 environments/synthetic_workspace_gym/synthetic_workspace_gym/artifacts.json create mode 100644 environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/__init__.py create mode 100644 environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/base.py create mode 100644 environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/capabilities.py create mode 100644 environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/composite_workspace.py create mode 100644 environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/metrics.py create mode 100644 environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/pipeline.py create mode 100644 environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/pipeline_profile.py create mode 100644 environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/registry.py create mode 100644 environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/retrieval_workspace.py create mode 100644 environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/script_repair.py create mode 100644 environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/tabular.py create mode 100644 environments/synthetic_workspace_gym/synthetic_workspace_gym/schemas/__init__.py create mode 100644 environments/synthetic_workspace_gym/synthetic_workspace_gym/schemas/common.py create mode 100644 environments/synthetic_workspace_gym/synthetic_workspace_gym/taskset.py create mode 100644 environments/synthetic_workspace_gym/synthetic_workspace_gym/trusted_grader.py create mode 100644 environments/synthetic_workspace_gym/synthetic_workspace_gym/utils/__init__.py create mode 100644 environments/synthetic_workspace_gym/synthetic_workspace_gym/utils/io.py create mode 100644 environments/synthetic_workspace_gym/synthetic_workspace_gym/utils/paths.py create mode 100644 environments/synthetic_workspace_gym/synthetic_workspace_gym/utils/scratch.py diff --git a/environments/synthetic_workspace_gym/LICENSE b/environments/synthetic_workspace_gym/LICENSE new file mode 100644 index 0000000..a708599 --- /dev/null +++ b/environments/synthetic_workspace_gym/LICENSE @@ -0,0 +1,201 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, +and distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by +the copyright owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all +other entities that control, are controlled by, or are under common +control with that entity. For the purposes of this definition, +"control" means (i) the power, direct or indirect, to cause the +direction or management of such entity, whether by contract or +otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity +exercising permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, +including but not limited to software source code, documentation +source, and configuration files. + +"Object" form shall mean any form resulting from mechanical +transformation or translation of a Source form, including but +not limited to compiled object code, generated documentation, +and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or +Object form, made available under the License, as indicated by a +copyright notice that is included in or attached to the work +(an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object +form, that is based on (or derived from) the Work and for which the +editorial revisions, annotations, elaborations, or other modifications +represent, as a whole, an original work of authorship. For the purposes +of this License, Derivative Works shall not include works that remain +separable from, or merely link (or bind by name) to the interfaces of, +the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including +the original version of the Work and any modifications or additions +to that Work or Derivative Works thereof, that is intentionally +submitted to Licensor for inclusion in the Work by the copyright owner +or by an individual or Legal Entity authorized to submit on behalf of +the copyright owner. For the purposes of this definition, "submitted" +means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, +and issue tracking systems that are managed by, or on behalf of, the +Licensor for the purpose of discussing and improving the Work, but +excluding communication that is conspicuously marked or otherwise +designated in writing by the copyright owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity +on behalf of whom a Contribution has been received by Licensor and +subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of +this License, each Contributor hereby grants to You a perpetual, +worldwide, non-exclusive, no-charge, royalty-free, irrevocable +copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the +Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of +this License, each Contributor hereby grants to You a perpetual, +worldwide, non-exclusive, no-charge, royalty-free, irrevocable +(except as stated in this section) patent license to make, have made, +use, offer to sell, sell, import, and otherwise transfer the Work, +where such license applies only to those patent claims licensable +by such Contributor that are necessarily infringed by their +Contribution(s) alone or by combination of their Contribution(s) +with the Work to which such Contribution(s) was submitted. If You +institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work +or a Contribution incorporated within the Work constitutes direct +or contributory patent infringement, then any patent licenses +granted to You under this License for that Work shall terminate +as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the +Work or Derivative Works thereof in any medium, with or without +modifications, and in Source or Object form, provided that You +meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + +You may add Your own copyright statement to Your modifications and +may provide additional or different license terms and conditions +for use, reproduction, or distribution of Your modifications, or +for any such Derivative Works as a whole, provided Your use, +reproduction, and distribution of the Work otherwise complies with +the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, +any Contribution intentionally submitted for inclusion in the Work +by You to the Licensor shall be under the terms and conditions of +this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify +the terms of any separate license agreement you may have executed +with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade +names, trademarks, service marks, or product names of the Licensor, +except as required for reasonable and customary use in describing the +origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or +agreed to in writing, Licensor provides the Work (and each +Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +implied, including, without limitation, any warranties or conditions +of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A +PARTICULAR PURPOSE. You are solely responsible for determining the +appropriateness of using or redistributing the Work and assume any +risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, +whether in tort (including negligence), contract, or otherwise, +unless required by applicable law (such as deliberate and grossly +negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, +incidental, or consequential damages of any character arising as a +result of this License or out of the use or inability to use the +Work (including but not limited to damages for loss of goodwill, +work stoppage, computer failure or malfunction, or any and all +other commercial damages or losses), even if such Contributor +has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing +the Work or Derivative Works thereof, You may choose to offer, +and charge a fee for, acceptance of support, warranty, indemnity, +or other liability obligations and/or rights consistent with this +License. However, in accepting such obligations, You may act only +on Your own behalf and on Your sole responsibility, not on behalf +of any other Contributor, and only if You agree to indemnify, +defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason +of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + +To apply the Apache License to your work, attach the following +boilerplate notice, with the fields enclosed by brackets "[]" +replaced with your own identifying information. (Don't include +the brackets!) The text should be enclosed in the appropriate +comment syntax for the file format. We also recommend that a +file or class name and description of purpose be included on the +same "printed page" as the copyright notice for easier +identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/environments/synthetic_workspace_gym/README.md b/environments/synthetic_workspace_gym/README.md new file mode 100644 index 0000000..3cf3f5c --- /dev/null +++ b/environments/synthetic_workspace_gym/README.md @@ -0,0 +1,73 @@ +# Synthetic Workspace Gym V1 + +Synthetic Workspace Gym (SWG) is a native Verifiers V1 taskset for training terminal agents on generated, executable workspaces. Tasks cover tabular transformations, script repair, data pipelines, local-document retrieval, and composite workflows. Each task has a deterministic assignment, an agent-visible workspace, and a trusted evaluator with partial-credit scoring. + +## Design + +`SyntheticWorkspaceTaskset` loads prompts and assignments from a Hugging Face dataset pinned to an immutable commit. Each assignment becomes a typed `SyntheticWorkspaceTask` with typed `SyntheticWorkspaceData` and `SyntheticWorkspaceState`. Generation and calibration code lives separately under `generators/synthetic_workspace_gym`; it is not installed with this environment. + +The task declares `NEEDS_CONTAINER = True` and a working directory of `/workspace`. Verifiers and the selected standard harness own model interaction, terminal access, runtime provisioning, rollout timeouts, and trace capture. SWG does not wrap a harness or implement a rollout loop; it adds a shorter timeout only around candidate programs executed during trusted finalization. + +Visible workspaces are baked into six digest-pinned images, one per curriculum or evaluation panel. Setup selects one workspace and removes the rest of the split payload before the agent starts. Hidden assets never enter the agent runtime or task state. + +Finalization uses three trust domains: + +1. The completed workspace is inventoried by an absolute trusted interpreter and read through the runtime file API under file-count, directory-count, per-file, and aggregate-byte limits. Symlinks and non-regular entries are rejected. +2. A fresh runtime from the same split image executes candidate code with test inputs but no expected answers. Candidate commands run in a timeout-managed process group; timeout sends `TERM`, escalates to `KILL`, records return code `124`, skips an unavailable determinism rerun, and continues to grading. Concrete observations and profile outputs travel in a host-controlled payload outside `/workspace`; stdout never carries score-bearing data. +3. A second fresh runtime from the dedicated grader image receives those artifacts and the grader-only dataset payload. It computes reward without executing agent-controlled code. + +Both temporary runtimes use `vf.provision_runtime`, are network-restricted, and are torn down automatically. The weighted `workspace_score` reward is the evaluator's normalized score in `[0, 1]`; metrics report binary success, changed/final file counts, and evaluator subscores. + +The task requests framework-only network access. A fresh runtime is used for every rollout, and generated paths are validated before being written or extracted. + +## Repository layout + +The paired generator contribution keeps reusable generation and validation libraries in `generators/synthetic_workspace_gym/`, image definitions and build helpers in `scripts/synthetic_workspace_gym/`, and source curriculum configurations in `configs/synthetic_workspace_gym/`. Generated manifests, hidden trees, and visible workspaces are not committed to `environments/`. + +## Install + +From the repository root: + +```bash +uv pip install -e environments/synthetic_workspace_gym +``` + +## Smoke evaluation + +Run the required small integration smoke with the standard harness/runtime: + +```bash +uv run eval --taskset.id synthetic-workspace-gym -n 1 -r 2 --max-turns 4 +``` + +Use the default `sft-easy-v4` mid-training manifest or select another immutable dataset manifest and filters: + +```bash +uv run eval --taskset.id synthetic-workspace-gym \ + --env.taskset.manifest rl-eval-v3 \ + --env.taskset.families '["script_repair","pipeline"]' \ + --env.taskset.difficulties '[4,5]' \ + -n 20 -r 3 --rich false --no-push +``` + +The purpose-specific manifests are `sft-easy-v4` (family-calibrated D1-D3 teacher traces), `sft-validation-v4` (scenario- and seed-held-out D1-D3), `rl-hard-v4` (family-calibrated D4-D5 atomic and composite training), and `rl-eval-v4` (family-stratified held-out D4-D5 scenarios, seeds, fixtures, and compositions). V4 records the exact generator commit, uses platform-independent task bytes, and measures untouched pipeline baselines through the same evaluator paths used at grading time. Verifiers' normal `-n`, `-r`, `--shuffle`, harness, model, sampling, and runtime options apply without environment-specific rollout arguments. + +For difficulty calibration, use `eval-d1-d4-paired-panel-48-v3` and `eval-d5-family-calibration-40-v3`. The first holds family, scenario, and seed fixed while difficulty changes; the second reports D5 separately by family and retrieval profile. + +## Configuration + +| Field | Default | Purpose | +| ------------------------------------------ | ------------- | -------------------------------------------------------------- | +| `manifest` | `sft-easy-v4` | Immutable curriculum or evaluation panel | +| `families` | all | Optional family filter | +| `difficulties` | all | Optional difficulty filter (`1` through `5`) | +| `tasks` | all | Optional exact task-ID filter | +| `task.max_workspace_files` | `1024` | Maximum number of regular files transferred | +| `task.max_workspace_directories` | `1024` | Maximum number of workspace directories | +| `task.max_file_bytes` | `8388608` | Maximum workspace file or execution artifact transfer | +| `task.max_workspace_bytes` | `67108864` | Maximum aggregate workspace transfer | +| `task.max_result_bytes` | `2097152` | Maximum structured grading result returned from the runtime | +| `task.candidate_execution_timeout_seconds` | `120` | Per-run candidate execution limit inside finalization | +| `task.candidate_kill_grace_seconds` | `5` | Grace period between candidate process-group termination signals | + +The package pins the dataset commit and all seven runtime images in `synthetic_workspace_gym/artifacts.json`. Changing task data, task behavior, or reward semantics requires publishing new immutable artifacts and bumping the package version. diff --git a/environments/synthetic_workspace_gym/pyproject.toml b/environments/synthetic_workspace_gym/pyproject.toml new file mode 100644 index 0000000..6819f7b --- /dev/null +++ b/environments/synthetic_workspace_gym/pyproject.toml @@ -0,0 +1,21 @@ +[project] +name = "synthetic-workspace-gym" +version = "0.2.1" +description = "Native Verifiers V1 workspace tasks for training terminal agents." +readme = "README.md" +requires-python = ">=3.11" +license = "Apache-2.0" +authors = [{ name = "Yadnyesh Chakane" }] +tags = ["multi-turn", "tool-use", "workspace", "code", "v1"] +dependencies = ["huggingface-hub>=0.28,<2", "verifiers>=0.3.0"] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["synthetic_workspace_gym"] + +[tool.verifiers.eval] +num_examples = 50 +rollouts_per_example = 1 diff --git a/environments/synthetic_workspace_gym/synthetic_workspace_gym/__init__.py b/environments/synthetic_workspace_gym/synthetic_workspace_gym/__init__.py new file mode 100644 index 0000000..c153aaa --- /dev/null +++ b/environments/synthetic_workspace_gym/synthetic_workspace_gym/__init__.py @@ -0,0 +1,17 @@ +from synthetic_workspace_gym.taskset import ( + SyntheticWorkspaceData, + SyntheticWorkspaceState, + SyntheticWorkspaceTask, + SyntheticWorkspaceTaskConfig, + SyntheticWorkspaceTaskset, + SyntheticWorkspaceTasksetConfig, +) + +__all__ = [ + "SyntheticWorkspaceData", + "SyntheticWorkspaceState", + "SyntheticWorkspaceTask", + "SyntheticWorkspaceTaskConfig", + "SyntheticWorkspaceTaskset", + "SyntheticWorkspaceTasksetConfig", +] diff --git a/environments/synthetic_workspace_gym/synthetic_workspace_gym/artifacts.json b/environments/synthetic_workspace_gym/synthetic_workspace_gym/artifacts.json new file mode 100644 index 0000000..d720fdc --- /dev/null +++ b/environments/synthetic_workspace_gym/synthetic_workspace_gym/artifacts.json @@ -0,0 +1,36 @@ +{ + "dataset_repo": "ydnysh/synthetic-workspace-gym", + "dataset_revision": "90893216b0cf7a2f46a077ab44ab9d404c06a686", + "generator_revision": "533c4d345a6a17ba22eea8a68ffdc2f62c3bc0a2", + "artifact_integrity_sha256": "2eb246b24fbbd905e763afa48428e833744748d827e2c5d7fc02cf41194c8c27", + "images": { + "eval-d1-d4-paired-panel-48-v4": { + "runtime": "prime/yadnyesh/synthetic-workspace-gym-eval-d1-d4-paired-panel-48-v4:v4-2eb246b2", + "container": "cmee2dees0002x3lw5g0cbf8p/synthetic-workspace-gym-eval-d1-d4-paired-panel-48-v4@sha256:dcf86a3ca713ac26fc2c2abf4981e03e3fa9097eeca5f0b55499415df7d49a97" + }, + "eval-d5-family-calibration-40-v4": { + "runtime": "prime/yadnyesh/synthetic-workspace-gym-eval-d5-family-calibration-40-v4:v4-2eb246b2", + "container": "cmee2dees0002x3lw5g0cbf8p/synthetic-workspace-gym-eval-d5-family-calibration-40-v4@sha256:4a1d1edca62098d694d1783488defa937eb8dd6e241341c38920a183024ea2aa" + }, + "grader": { + "runtime": "prime/yadnyesh/synthetic-workspace-gym-grader:v4-2eb246b2", + "container": "cmee2dees0002x3lw5g0cbf8p/synthetic-workspace-gym-grader@sha256:60f131885799072b6c1fa36534bc7a33696620caa759392dd182ac5d282748d4" + }, + "rl-eval-v4": { + "runtime": "prime/yadnyesh/synthetic-workspace-gym-rl-eval-v4:v4-2eb246b2", + "container": "cmee2dees0002x3lw5g0cbf8p/synthetic-workspace-gym-rl-eval-v4@sha256:a297f0b462a94bb98375c8926a682d0d8142e14592b1d3782218a7e15b6ebe37" + }, + "rl-hard-v4": { + "runtime": "prime/yadnyesh/synthetic-workspace-gym-rl-hard-v4:v4-2eb246b2", + "container": "cmee2dees0002x3lw5g0cbf8p/synthetic-workspace-gym-rl-hard-v4@sha256:bb5d996631237fe8164f6c529158e27e399fd390309d40271216a442033d01ff" + }, + "sft-easy-v4": { + "runtime": "prime/yadnyesh/synthetic-workspace-gym-sft-easy-v4:v4-2eb246b2", + "container": "cmee2dees0002x3lw5g0cbf8p/synthetic-workspace-gym-sft-easy-v4@sha256:1ccada8a717ae8663dcdc6cba91c902a4c111d94b14897d314692579d0b171b1" + }, + "sft-validation-v4": { + "runtime": "prime/yadnyesh/synthetic-workspace-gym-sft-validation-v4:v4-2eb246b2", + "container": "cmee2dees0002x3lw5g0cbf8p/synthetic-workspace-gym-sft-validation-v4@sha256:7eeb9dabe9b32e85a1f88da92859fe3e7ef098f9c7b03ca44ce6ef7824707378" + } + } +} diff --git a/environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/__init__.py b/environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/__init__.py new file mode 100644 index 0000000..125e453 --- /dev/null +++ b/environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/__init__.py @@ -0,0 +1,3 @@ +from .registry import get_evaluator, list_evaluators + +__all__ = ["get_evaluator", "list_evaluators"] diff --git a/environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/base.py b/environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/base.py new file mode 100644 index 0000000..834b99b --- /dev/null +++ b/environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/base.py @@ -0,0 +1,12 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from pathlib import Path + +from synthetic_workspace_gym.schemas import EnvironmentManifest, EvaluatorResult + + +class BaseEvaluator(ABC): + @abstractmethod + def evaluate(self, workspace_path: Path, manifest: EnvironmentManifest, hidden_root: Path) -> EvaluatorResult: + raise NotImplementedError diff --git a/environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/capabilities.py b/environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/capabilities.py new file mode 100644 index 0000000..ecddca0 --- /dev/null +++ b/environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/capabilities.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class CapabilityScore: + """One independently observable evaluator capability.""" + + name: str + value: float + weight: float + diagnostic: str | None = None + + def __post_init__(self) -> None: + if not self.name.strip(): + raise ValueError("Capability names must not be empty.") + if self.weight < 0: + raise ValueError("Capability weights must be non-negative.") + + @property + def clamped_value(self) -> float: + return max(0.0, min(1.0, float(self.value))) + + +def weighted_capability_score(capabilities: list[CapabilityScore]) -> float: + """Return a normalized weighted score without semantic floors or caps.""" + + total_weight = sum(item.weight for item in capabilities) + if total_weight <= 0: + raise ValueError("Capability weights must sum to a positive value.") + score = sum(item.clamped_value * item.weight for item in capabilities) + return round(max(0.0, min(1.0, score / total_weight)), 6) + + +def capability_subscores( + capabilities: list[CapabilityScore], +) -> dict[str, float]: + return {f"capability_{item.name}": item.clamped_value for item in capabilities} + + +def capability_diagnostics( + capabilities: list[CapabilityScore], +) -> dict[str, str]: + return {item.name: item.diagnostic for item in capabilities if item.diagnostic is not None} diff --git a/environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/composite_workspace.py b/environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/composite_workspace.py new file mode 100644 index 0000000..b5678f0 --- /dev/null +++ b/environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/composite_workspace.py @@ -0,0 +1,7 @@ +from __future__ import annotations + +from .retrieval_workspace import RetrievalWorkspaceEvaluator + + +class CompositeWorkspaceEvaluator(RetrievalWorkspaceEvaluator): + """Capability-scored evaluator for retrieval-guided pipeline tasks.""" diff --git a/environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/metrics.py b/environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/metrics.py new file mode 100644 index 0000000..a431754 --- /dev/null +++ b/environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/metrics.py @@ -0,0 +1,160 @@ +from __future__ import annotations + +import json +from collections import Counter +from typing import Any + + +def row_overlap_metrics(expected: Any, actual: Any) -> dict[str, float]: + if not isinstance(expected, list) or not isinstance(actual, list): + return { + "row_precision": 0.0, + "row_recall": 0.0, + "row_f1": 0.0, + "exact_match": 1.0 if actual == expected else 0.0, + } + expected_rows = Counter(normalize_row(row) for row in expected) + actual_rows = Counter(normalize_row(row) for row in actual) + matches = sum((expected_rows & actual_rows).values()) + precision = matches / len(actual) if actual else (1.0 if not expected else 0.0) + recall = matches / len(expected) if expected else (1.0 if not actual else 0.0) + if precision + recall == 0: + f1 = 0.0 + else: + f1 = (2 * precision * recall) / (precision + recall) + return { + "row_precision": round(precision, 6), + "row_recall": round(recall, 6), + "row_f1": round(f1, 6), + "exact_match": 1.0 if actual == expected else 0.0, + } + + +def weighted_match_score(*, output_exists: float, valid_structure: float, metrics: dict[str, float]) -> float: + if metrics["exact_match"] == 1.0 and output_exists == 1.0 and valid_structure == 1.0: + return 1.0 + score = ( + 0.2 * output_exists + + 0.2 * valid_structure + + 0.3 * metrics["row_precision"] + + 0.2 * metrics["row_recall"] + + 0.1 * metrics["exact_match"] + ) + return round(min(1.0, max(0.0, score)), 6) + + +def baseline_normalized_score(score: float, baseline: float) -> float: + """Measure semantic progress above the artifact shipped to the agent.""" + + score = min(1.0, max(0.0, float(score))) + baseline = min(1.0, max(0.0, float(baseline))) + if score >= 1.0: + return 1.0 + if score <= baseline or baseline >= 1.0: + return 0.0 + return round((score - baseline) / (1.0 - baseline), 6) + + +def normalize_row(row: Any) -> str: + return json.dumps(row, sort_keys=True, separators=(",", ":")) + + +def row_diff_diagnostics(expected: Any, actual: Any, *, limit: int = 3) -> dict[str, Any]: + """Return compact, artifact-safe diagnostics for list-of-row mismatches.""" + + if not isinstance(expected, list) or not isinstance(actual, list): + return {"expected_type": type(expected).__name__, "actual_type": type(actual).__name__} + + diagnostics: dict[str, Any] = { + "expected_schema_keys": sorted(_row_keys(expected)), + "actual_schema_keys": sorted(_row_keys(actual)), + } + first_difference = _first_differing_row(expected, actual) + if first_difference is not None: + index, expected_row, actual_row = first_difference + diagnostics["first_differing_row"] = { + "index": index, + "expected": expected_row, + "actual": actual_row, + } + + expected_rows = Counter(normalize_row(row) for row in expected) + actual_rows = Counter(normalize_row(row) for row in actual) + diagnostics["missing_rows_preview"] = _counter_preview(expected_rows - actual_rows, limit=limit) + diagnostics["unexpected_rows_preview"] = _counter_preview(actual_rows - expected_rows, limit=limit) + return diagnostics + + +def json_field_diff_diagnostics(expected: Any, actual: Any, *, limit: int = 5) -> dict[str, Any]: + """Return path-level diagnostics for nested JSON contract mismatches.""" + + expected_fields = {str(row["path"]): row["value"] for row in flatten_json(expected)} + actual_fields = {str(row["path"]): row["value"] for row in flatten_json(actual)} + missing_paths = sorted(set(expected_fields) - set(actual_fields)) + unexpected_paths = sorted(set(actual_fields) - set(expected_fields)) + mismatches = [] + for path in sorted(set(expected_fields) & set(actual_fields)): + if expected_fields[path] != actual_fields[path]: + mismatches.append( + { + "path": path, + "expected": expected_fields[path], + "actual": actual_fields[path], + } + ) + if len(mismatches) >= limit: + break + return { + "missing_paths": missing_paths[:limit], + "unexpected_paths": unexpected_paths[:limit], + "value_mismatches": mismatches, + } + + +def flatten_json(value: Any, *, prefix: str = "$") -> list[dict[str, object]]: + if isinstance(value, dict): + if not value: + return [{"path": prefix, "value": ""}] + flattened: list[dict[str, object]] = [] + for key in sorted(value): + flattened.extend(flatten_json(value[key], prefix=f"{prefix}.{key}")) + return flattened + if isinstance(value, list): + if not value: + return [{"path": prefix, "value": ""}] + flattened = [] + for index, item in enumerate(value): + flattened.extend(flatten_json(item, prefix=f"{prefix}[{index}]")) + return flattened + return [{"path": prefix, "value": value}] + + +def _row_keys(rows: list[Any]) -> set[str]: + keys: set[str] = set() + for row in rows: + if isinstance(row, dict): + keys.update(str(key) for key in row) + return keys + + +def _first_differing_row(expected: list[Any], actual: list[Any]) -> tuple[int, Any, Any] | None: + for index in range(max(len(expected), len(actual))): + expected_row = expected[index] if index < len(expected) else None + actual_row = actual[index] if index < len(actual) else None + if expected_row != actual_row: + return index, expected_row, actual_row + return None + + +def _counter_preview(rows: Counter[str], *, limit: int) -> list[Any]: + preview: list[Any] = [] + for row, count in rows.most_common(limit): + try: + parsed = json.loads(row) + except json.JSONDecodeError: + parsed = row + if count > 1: + preview.append({"row": parsed, "count": count}) + else: + preview.append(parsed) + return preview diff --git a/environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/pipeline.py b/environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/pipeline.py new file mode 100644 index 0000000..06c9288 --- /dev/null +++ b/environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/pipeline.py @@ -0,0 +1,237 @@ +from __future__ import annotations + +import json +import time +from pathlib import Path +from types import SimpleNamespace + +from synthetic_workspace_gym.evaluators.base import BaseEvaluator +from synthetic_workspace_gym.evaluators.metrics import ( + baseline_normalized_score, + row_overlap_metrics, + weighted_match_score, +) +from synthetic_workspace_gym.schemas import EnvironmentManifest, EvaluatorResult +from synthetic_workspace_gym.utils.io import read_json + + +class PipelineEvaluator(BaseEvaluator): + CAPABILITY_WEIGHTS = { + "execution": 0.05, + "top_level_shape": 0.10, + "row_schema": 0.15, + "normalization": 0.15, + "filtering": 0.15, + "aggregation": 0.25, + "ordering": 0.05, + "determinism": 0.10, + } + + def capability_scores( + self, + expected: object, + actual: object, + *, + deterministic_rerun: float, + ) -> dict[str, float]: + expected_rows = expected if isinstance(expected, list) else [] + top_level_shape = float(isinstance(actual, list)) + raw_rows = actual if isinstance(actual, list) else actual.get("rows", []) if isinstance(actual, dict) else [] + actual_rows = [row for row in raw_rows if isinstance(row, dict)] + expected_key_set = {"team", "job_count", "total_hours"} + row_schema = float( + bool(actual_rows) + and len(actual_rows) == len(raw_rows) + and all(set(row) == expected_key_set for row in actual_rows) + ) + legacy_schema = bool(actual_rows) and all(set(row) == {"team", "jobs", "hours"} for row in actual_rows) + semantic_factor = 0.75 if legacy_schema or not top_level_shape else 1.0 + canonical_rows = [ + { + "team": row.get("team"), + "job_count": row.get("job_count", row.get("jobs")), + "total_hours": row.get("total_hours", row.get("hours")), + } + for row in actual_rows + ] + expected_teams = [str(row.get("team")) for row in expected_rows if isinstance(row, dict)] + actual_teams = [str(row.get("team")) for row in canonical_rows] + normalization = semantic_factor * float(sorted(actual_teams) == sorted(expected_teams) and bool(expected_teams)) + expected_count = sum(int(row.get("job_count", 0)) for row in expected_rows if isinstance(row, dict)) + actual_count = sum(int(row.get("job_count") or 0) for row in canonical_rows) + filtering = semantic_factor * float(actual_count == expected_count and expected_count > 0) + expected_hours = { + str(row.get("team")): row.get("total_hours") for row in expected_rows if isinstance(row, dict) + } + actual_hours = {str(row.get("team")): row.get("total_hours") for row in canonical_rows} + aggregation = semantic_factor * float(actual_hours == expected_hours and bool(expected_hours)) + ordering = semantic_factor * float(actual_teams == expected_teams and bool(actual_teams)) + return { + "execution": 1.0, + "top_level_shape": top_level_shape, + "row_schema": row_schema, + "normalization": normalization, + "filtering": filtering, + "aggregation": aggregation, + "ordering": ordering, + "determinism": deterministic_rerun, + } + + def weighted_capability_score(self, scores: dict[str, float]) -> float: + return round( + sum(self.CAPABILITY_WEIGHTS[name] * scores.get(name, 0.0) for name in self.CAPABILITY_WEIGHTS), + 6, + ) + + def required_json_artifact_scores( + self, + workspace_path: Path, + hidden_root: Path, + config: dict[str, object], + ) -> tuple[dict[str, float], list[str]]: + scores: dict[str, float] = {} + failures: list[str] = [] + for raw_item in config.get("required_json_artifacts", []): + item = dict(raw_item) + capability = str(item.get("capability", "required_artifact")) + try: + actual = json.loads((workspace_path / str(item["path"])).read_text(encoding="utf-8")) + expected = read_json(hidden_root / str(item["expected_path"])) + score = float(actual == expected) + except (FileNotFoundError, json.JSONDecodeError, OSError): + score = 0.0 + scores[capability] = score + if score == 0.0: + failures.append(f"{capability}_failed") + return scores, failures + + def evaluate(self, workspace_path: Path, manifest: EnvironmentManifest, hidden_root: Path) -> EvaluatorResult: + started = time.perf_counter() + config = read_json(hidden_root / "evaluator_config.json") + precomputed_path = hidden_root / "precomputed_execution.json" + if not precomputed_path.is_file(): + raise ValueError("pipeline execution must be collected in the isolated runner runtime") + run = dict(read_json(precomputed_path).get("visible", {})) + completed = SimpleNamespace( + returncode=int(run.get("returncode", 1)), + stdout=str(run.get("stdout", "")), + stderr=str(run.get("stderr", "")), + ) + if completed.returncode != 0: + return EvaluatorResult( + success=False, + score=0.0, + subscores={ + "execution": 0.0, + "valid_json": 0.0, + "row_precision": 0.0, + "row_recall": 0.0, + "row_f1": 0.0, + "exact_match": 0.0, + }, + failure_labels=["execution_failed"], + diagnostics={ + "stdout": completed.stdout, + "stderr": completed.stderr, + "returncode": completed.returncode, + }, + runtime_seconds=time.perf_counter() - started, + ) + + output_path = workspace_path / config["required_output_path"] + if not output_path.exists(): + raw_score = 0.05 + score = ( + baseline_normalized_score(raw_score, float(config["initial_score"])) + if config.get("initial_score") is not None + else raw_score + ) + return EvaluatorResult( + success=False, + score=score, + subscores={ + "execution": 1.0, + "valid_json": 0.0, + "row_precision": 0.0, + "row_recall": 0.0, + "row_f1": 0.0, + "exact_match": 0.0, + }, + failure_labels=["output_missing"], + diagnostics={"required_output_path": config["required_output_path"]}, + runtime_seconds=time.perf_counter() - started, + ) + + try: + actual = json.loads(output_path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raw_score = 0.10 + score = ( + baseline_normalized_score(raw_score, float(config["initial_score"])) + if config.get("initial_score") is not None + else raw_score + ) + return EvaluatorResult( + success=False, + score=score, + subscores={ + "execution": 1.0, + "valid_json": 0.0, + "row_precision": 0.0, + "row_recall": 0.0, + "row_f1": 0.0, + "exact_match": 0.0, + }, + failure_labels=["invalid_json"], + diagnostics={"error": str(exc)}, + runtime_seconds=time.perf_counter() - started, + ) + + expected = read_json(hidden_root / "expected_output.json") + rerun = SimpleNamespace(returncode=int(run.get("rerun_returncode", 1))) + deterministic_rerun = float(bool(run.get("deterministic", False))) + artifact_scores, artifact_failures = self.required_json_artifact_scores(workspace_path, hidden_root, config) + artifacts_valid = all(score == 1.0 for score in artifact_scores.values()) + success = actual == expected and deterministic_rerun == 1.0 and artifacts_valid + metrics = row_overlap_metrics(expected, actual) + score = weighted_match_score(output_exists=1.0, valid_structure=1.0, metrics=metrics) + capability_scores: dict[str, float] = {} + if config.get("capability_scoring"): + capability_scores = self.capability_scores(expected, actual, deterministic_rerun=deterministic_rerun) + capability_scores.update(artifact_scores) + score = self.weighted_capability_score(capability_scores) + if artifact_failures: + score = min(score, float(config.get("required_artifact_failure_cap", 0.30))) + raw_score = score + if config.get("initial_score") is not None: + score = baseline_normalized_score(raw_score, float(config["initial_score"])) + if success: + score = 1.0 + diagnostics = { + "required_output_path": config["required_output_path"], + "stdout": completed.stdout, + "stderr": completed.stderr, + "returncode": completed.returncode, + "rerun_returncode": rerun.returncode, + "deterministic_rerun": bool(deterministic_rerun), + "artifact_failures": artifact_failures, + "raw_score": raw_score, + "initial_score": config.get("initial_score"), + } + if not success: + diagnostics["expected_preview"] = expected[:2] if isinstance(expected, list) else expected + diagnostics["actual_preview"] = actual[:2] if isinstance(actual, list) else actual + return EvaluatorResult( + success=success, + score=score, + subscores={ + "execution": 1.0, + "valid_json": 1.0, + "deterministic_rerun": deterministic_rerun, + **metrics, + **{f"capability_{name}": value for name, value in capability_scores.items()}, + }, + failure_labels=([] if success else list(dict.fromkeys(["output_mismatch", *artifact_failures]))), + diagnostics=diagnostics, + runtime_seconds=time.perf_counter() - started, + ) diff --git a/environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/pipeline_profile.py b/environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/pipeline_profile.py new file mode 100644 index 0000000..c808e1b --- /dev/null +++ b/environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/pipeline_profile.py @@ -0,0 +1,155 @@ +from __future__ import annotations + +import json +import time +from pathlib import Path + +from synthetic_workspace_gym.evaluators.base import BaseEvaluator +from synthetic_workspace_gym.evaluators.capabilities import ( + CapabilityScore, + capability_diagnostics, + capability_subscores, + weighted_capability_score, +) +from synthetic_workspace_gym.schemas import EnvironmentManifest, EvaluatorResult +from synthetic_workspace_gym.utils.io import read_json + + +class ProfiledPipelineEvaluator(BaseEvaluator): + WEIGHTS = { + "execution": 0.05, + "top_level_shape": 0.05, + "row_schema": 0.10, + "normalization": 0.15, + "deduplication": 0.15, + "filtering": 0.10, + "aggregation": 0.25, + "ordering": 0.05, + "determinism": 0.10, + } + + def evaluate(self, workspace_path: Path, manifest: EnvironmentManifest, hidden_root: Path) -> EvaluatorResult: + del manifest + started = time.perf_counter() + config = read_json(hidden_root / "evaluator_config.json") + execution_path = hidden_root / "precomputed_execution.json" + if not execution_path.is_file(): + raise ValueError("profiled pipeline execution must run in the isolated runner runtime") + execution = read_json(execution_path) + visible = self._precomputed_result( + hidden_root / "precomputed_outputs" / "visible-output.json", + dict(execution.get("visible", {})), + ) + hidden = self._precomputed_result( + hidden_root / "precomputed_outputs" / "hidden-output.json", + dict(execution.get("hidden", {})), + ) + if visible["returncode"] != 0: + return self._hard_failure(started, "execution_failed", visible) + if not visible["exists"]: + return self._hard_failure(started, "output_missing", visible) + if visible["value"] is None: + return self._hard_failure(started, "invalid_json", visible, score=0.10) + + visible_expected = read_json(hidden_root / "expected_output.json") + hidden_expected = read_json(hidden_root / str(config["hidden_expected_path"])) + visible_scores = self._semantic_scores(visible_expected, visible["value"]) + hidden_scores = self._semantic_scores(hidden_expected, hidden["value"]) + values = { + "execution": float(hidden["returncode"] == 0), + **{name: round((visible_scores[name] + hidden_scores[name]) / 2, 6) for name in visible_scores}, + "determinism": min(float(visible["deterministic"]), float(hidden["deterministic"])), + } + capabilities = [ + CapabilityScore( + name=name, + value=values.get(name, 0.0), + weight=weight, + diagnostic="passed" if values.get(name, 0.0) == 1.0 else "partial or failed", + ) + for name, weight in self.WEIGHTS.items() + ] + success = bool( + visible["value"] == visible_expected and hidden["value"] == hidden_expected and values["determinism"] == 1.0 + ) + base_score = weighted_capability_score(capabilities) + semantic_names = {"normalization", "deduplication", "filtering", "aggregation", "ordering"} + semantic_completion = sum(values[name] for name in semantic_names) / len(semantic_names) + score = 1.0 if success else round(base_score * semantic_completion**6, 6) + return EvaluatorResult( + success=success, + score=score, + subscores=capability_subscores(capabilities), + failure_labels=[] if success else ["pipeline_capabilities_incomplete"], + diagnostics={ + "capability_diagnostics": capability_diagnostics(capabilities), + "visible_exact": visible["value"] == visible_expected, + "hidden_exact": hidden["value"] == hidden_expected, + "base_weighted_score": base_score, + "semantic_completion": round(semantic_completion, 6), + "visible_stderr": visible["stderr"], + "hidden_stderr": hidden["stderr"], + }, + runtime_seconds=time.perf_counter() - started, + ) + + def _semantic_scores(self, expected: object, actual: object) -> dict[str, float]: + expected_rows = expected if isinstance(expected, list) else [] + actual_rows = actual if isinstance(actual, list) else [] + shape = float(isinstance(actual, list)) + keys = {"team", "job_count", "total_hours"} + schema = float(bool(actual_rows) and all(isinstance(row, dict) and set(row) == keys for row in actual_rows)) + expected_by_team = {str(row["team"]): row for row in expected_rows if isinstance(row, dict)} + actual_by_team = {str(row["team"]): row for row in actual_rows if isinstance(row, dict)} + expected_teams = set(expected_by_team) + actual_teams = set(actual_by_team) + team_union = expected_teams | actual_teams + normalization = len(expected_teams & actual_teams) / len(team_union) if team_union else 1.0 + expected_count = sum(int(row["job_count"]) for row in expected_by_team.values()) + actual_count = sum(int(row.get("job_count", 0)) for row in actual_by_team.values()) + count_score = 1.0 - min(1.0, abs(expected_count - actual_count) / max(1, expected_count)) + aggregation = sum( + actual_by_team.get(team, {}).get("total_hours") == row.get("total_hours") + for team, row in expected_by_team.items() + ) / max(1, len(expected_by_team)) + actual_order = [str(row.get("team")) for row in actual_rows if isinstance(row, dict)] + return { + "top_level_shape": shape, + "row_schema": schema, + "normalization": round(normalization, 6), + "deduplication": round(count_score, 6), + "filtering": round(count_score, 6), + "aggregation": round(aggregation, 6), + "ordering": float(actual_order == sorted(actual_order) and bool(actual_order)), + } + + def _precomputed_result(self, output: Path, run: dict[str, object]) -> dict[str, object]: + raw = output.read_bytes() if output.is_file() else b"" + try: + value = json.loads(raw) if raw else None + except json.JSONDecodeError: + value = None + return { + "returncode": int(run.get("returncode", 1)), + "exists": bool(raw), + "value": value, + "deterministic": bool(run.get("deterministic", False)), + "stderr": str(run.get("stderr", "")), + } + + def _hard_failure( + self, + started: float, + label: str, + outcome: dict[str, object], + *, + score: float = 0.0, + ) -> EvaluatorResult: + return EvaluatorResult( + success=False, + score=score, + subscores={name: 0.0 for name in self.WEIGHTS}, + failure_labels=[label], + diagnostics={"returncode": outcome["returncode"], "stderr": outcome["stderr"]}, + runtime_seconds=time.perf_counter() - started, + ) diff --git a/environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/registry.py b/environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/registry.py new file mode 100644 index 0000000..42c79da --- /dev/null +++ b/environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/registry.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +from importlib import import_module + +from synthetic_workspace_gym.evaluators.base import BaseEvaluator +from synthetic_workspace_gym.schemas import EnvironmentFamily + +from .composite_workspace import CompositeWorkspaceEvaluator +from .pipeline import PipelineEvaluator +from .retrieval_workspace import RetrievalWorkspaceEvaluator +from .script_repair import ScriptRepairEvaluator +from .tabular import TabularEvaluator + +EVALUATORS = { + EnvironmentFamily.TABULAR: TabularEvaluator(), + EnvironmentFamily.SCRIPT_REPAIR: ScriptRepairEvaluator(), + EnvironmentFamily.PIPELINE: PipelineEvaluator(), + EnvironmentFamily.RETRIEVAL_WORKSPACE: RetrievalWorkspaceEvaluator(), + EnvironmentFamily.COMPOSITE_WORKSPACE: CompositeWorkspaceEvaluator(), +} + + +def get_evaluator(family: EnvironmentFamily | str, evaluator_entrypoint: str | None = None): + if evaluator_entrypoint: + return load_evaluator_from_entrypoint(evaluator_entrypoint) + return D5CalibratedEvaluator(EVALUATORS[EnvironmentFamily(family)]) + + +def list_evaluators() -> list[str]: + return [family.value for family in EVALUATORS] + + +def load_evaluator_from_entrypoint(entrypoint: str) -> BaseEvaluator: + module_name, separator, attr_name = entrypoint.partition(":") + if not separator or not module_name or not attr_name: + raise ValueError(f"Invalid evaluator entrypoint: {entrypoint}") + module = import_module(module_name) + loaded = getattr(module, attr_name) + if isinstance(loaded, BaseEvaluator): + return D5CalibratedEvaluator(loaded) + instance = loaded() + if not isinstance(instance, BaseEvaluator): + raise TypeError(f"Evaluator entrypoint did not resolve to a BaseEvaluator: {entrypoint}") + return D5CalibratedEvaluator(instance) + + +class D5CalibratedEvaluator(BaseEvaluator): + """Compress incomplete D5 rewards while preserving evaluator diagnostics.""" + + def __init__(self, evaluator: BaseEvaluator) -> None: + self.evaluator = evaluator + + def evaluate(self, workspace_path, manifest, hidden_root): + result = self.evaluator.evaluate(workspace_path, manifest, hidden_root) + realization = dict(manifest.metadata.get("difficulty_realization", {})) + if int(realization.get("level", 0)) != 5 or result.success: + return result + raw_score = float(result.score) + result.score = round(raw_score**3, 6) + result.diagnostics = { + **result.diagnostics, + "d5_raw_partial_score": raw_score, + "d5_partial_score_exponent": 3, + } + return result diff --git a/environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/retrieval_workspace.py b/environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/retrieval_workspace.py new file mode 100644 index 0000000..f71969d --- /dev/null +++ b/environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/retrieval_workspace.py @@ -0,0 +1,131 @@ +from __future__ import annotations + +import json +import time +from pathlib import Path +from typing import Any + +from synthetic_workspace_gym.evaluators.base import BaseEvaluator +from synthetic_workspace_gym.evaluators.metrics import ( + baseline_normalized_score, + flatten_json, + json_field_diff_diagnostics, + row_overlap_metrics, + weighted_match_score, +) +from synthetic_workspace_gym.schemas import EnvironmentManifest, EvaluatorResult +from synthetic_workspace_gym.utils.io import read_json + +from .script_repair import ScriptRepairEvaluator + + +class RetrievalWorkspaceEvaluator(BaseEvaluator): + def evaluate(self, workspace_path: Path, manifest: EnvironmentManifest, hidden_root: Path) -> EvaluatorResult: + workspace_path = workspace_path.resolve() + hidden_root = hidden_root.resolve() + config = read_json(hidden_root / "evaluator_config.json") + if config.get("secure_protocol") == "observations-v1": + return ScriptRepairEvaluator().evaluate_observations( + workspace_path, + hidden_root, + config, + time.perf_counter(), + ) + if str(config.get("mode", "exact_json")) != "exact_json": + raise ValueError("retrieval hidden tests require the secure observations protocol") + return self.evaluate_exact_json(workspace_path, manifest, hidden_root, config) + + def evaluate_exact_json( + self, + workspace_path: Path, + manifest: EnvironmentManifest, + hidden_root: Path, + config: dict[str, object], + ) -> EvaluatorResult: + del manifest + started = time.perf_counter() + output_path = workspace_path / str(config["output_path"]) + if not output_path.exists(): + return EvaluatorResult( + success=False, + score=0.0 if config.get("score_normalization") else 0.2, + subscores={ + "output_exists": 0.0, + "valid_json": 0.0, + "field_precision": 0.0, + "field_recall": 0.0, + "field_f1": 0.0, + "exact_match": 0.0, + }, + failure_labels=["output_missing"], + diagnostics={"required_output_path": str(config["output_path"])}, + runtime_seconds=time.perf_counter() - started, + ) + try: + actual = json.loads(output_path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + return EvaluatorResult( + success=False, + score=0.0 if config.get("score_normalization") else 0.2, + subscores={ + "output_exists": 1.0, + "valid_json": 0.0, + "field_precision": 0.0, + "field_recall": 0.0, + "field_f1": 0.0, + "exact_match": 0.0, + }, + failure_labels=["invalid_json"], + diagnostics={"error": str(exc), "required_output_path": str(config["output_path"])}, + runtime_seconds=time.perf_counter() - started, + ) + + expected = read_json(hidden_root / str(config.get("expected_path", "expected_output.json"))) + metrics = self.field_overlap_metrics(expected, actual) + raw_score = weighted_match_score( + output_exists=1.0, + valid_structure=1.0, + metrics={ + "row_precision": metrics["field_precision"], + "row_recall": metrics["field_recall"], + "row_f1": metrics["field_f1"], + "exact_match": metrics["exact_match"], + }, + ) + baseline_score = config.get("initial_score") + score = baseline_normalized_score(raw_score, float(baseline_score)) if baseline_score is not None else raw_score + success = actual == expected + diagnostics = { + "required_output_path": str(config["output_path"]), + "expected_preview": self.preview(expected), + "actual_preview": self.preview(actual), + "raw_score": raw_score, + "initial_score": baseline_score, + } + if not success: + diagnostics.update(json_field_diff_diagnostics(expected, actual)) + return EvaluatorResult( + success=success, + score=score, + subscores={"output_exists": 1.0, "valid_json": 1.0, **metrics}, + failure_labels=[] if success else ["output_mismatch"], + diagnostics=diagnostics, + runtime_seconds=time.perf_counter() - started, + ) + + def field_overlap_metrics(self, expected: Any, actual: Any) -> dict[str, float]: + row_metrics = row_overlap_metrics(flatten_json(expected), flatten_json(actual)) + return { + "field_precision": row_metrics["row_precision"], + "field_recall": row_metrics["row_recall"], + "field_f1": row_metrics["row_f1"], + "exact_match": row_metrics["exact_match"], + } + + def preview(self, value: Any) -> Any: + if isinstance(value, list): + return value[:2] + if isinstance(value, dict): + keys = sorted(value)[:4] + return {key: value[key] for key in keys} + return value diff --git a/environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/script_repair.py b/environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/script_repair.py new file mode 100644 index 0000000..c6bf16c --- /dev/null +++ b/environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/script_repair.py @@ -0,0 +1,134 @@ +from __future__ import annotations + +import json +import time +from pathlib import Path + +from synthetic_workspace_gym.evaluators.base import BaseEvaluator +from synthetic_workspace_gym.evaluators.metrics import baseline_normalized_score +from synthetic_workspace_gym.schemas import EnvironmentManifest, EvaluatorResult +from synthetic_workspace_gym.utils.io import read_json + + +class ScriptRepairEvaluator(BaseEvaluator): + def evaluate(self, workspace_path: Path, manifest: EnvironmentManifest, hidden_root: Path) -> EvaluatorResult: + del manifest + started = time.perf_counter() + config = read_json(hidden_root / "evaluator_config.json") + if config.get("secure_protocol") != "observations-v1": + raise ValueError("script-repair tasks require the secure observations protocol") + return self.evaluate_observations(workspace_path, hidden_root, config, started) + + def evaluate_observations( + self, + workspace_path: Path, + hidden_root: Path, + config: dict[str, object], + started: float, + ) -> EvaluatorResult: + execution = read_json(hidden_root / "precomputed_execution.json") + expected = read_json(hidden_root / str(config["expected_observations_path"])) + visible_execution = dict(execution.get("visible", {})) + observations_path = hidden_root / "precomputed_outputs" / "observations.json" + actual = None + if int(visible_execution.get("returncode", 1)) == 0 and bool(execution.get("observations_present", False)): + try: + actual = json.loads(observations_path.read_text(encoding="utf-8")) + except (FileNotFoundError, json.JSONDecodeError, OSError): + pass + + outcomes = self._compare_observations(expected, actual) + tests_total = len(outcomes) + tests_passed = sum(outcomes.values()) + ratio = tests_passed / tests_total if tests_total else 0.0 + artifact_scores, artifact_failures = self.required_json_artifact_scores(workspace_path, hidden_root, config) + capability_scores = self._observation_capabilities(outcomes, config) + if capability_scores: + capability_scores.update(artifact_scores) + success = bool(tests_total and tests_passed == tests_total and not artifact_failures) + score = 1.0 if success else ratio + if capability_scores: + score = sum(capability_scores.values()) / len(capability_scores) + if artifact_failures: + score = min(score, float(config.get("required_artifact_failure_cap", 0.30))) + raw_score = score + if config.get("initial_score") is not None: + score = baseline_normalized_score(raw_score, float(config["initial_score"])) + if success: + score = 1.0 + return EvaluatorResult( + success=success, + score=round(score, 6), + subscores={ + "tests_passed": tests_passed, + "tests_total": float(tests_total), + "tests_passed_ratio": round(ratio, 6), + **{f"capability_{name}": round(value, 6) for name, value in capability_scores.items()}, + }, + failure_labels=[] if success else ["hidden_observations_mismatch", *artifact_failures], + diagnostics={ + "failed_observations": [name for name, value in outcomes.items() if value == 0.0], + "execution": execution, + "artifact_failures": artifact_failures, + "raw_score": raw_score, + "initial_score": config.get("initial_score"), + }, + runtime_seconds=time.perf_counter() - started, + ) + + def _compare_observations(self, expected: object, actual: object) -> dict[str, float]: + if isinstance(expected, list): + actual_by_name = { + str(item.get("name")): item.get("value") for item in actual or [] if isinstance(item, dict) + } + return { + str(item["name"]): float(actual_by_name.get(str(item["name"])) == item.get("value")) + for item in expected + if isinstance(item, dict) and "name" in item + } + if isinstance(expected, dict): + actual_values = actual if isinstance(actual, dict) else {} + return {name: float(actual_values.get(name) == value) for name, value in expected.items()} + raise TypeError("expected observations must be a list or mapping") + + def required_json_artifact_scores( + self, + workspace_path: Path, + hidden_root: Path, + config: dict[str, object], + ) -> tuple[dict[str, float], list[str]]: + scores: dict[str, float] = {} + failures: list[str] = [] + for raw_item in config.get("required_json_artifacts", []): + item = dict(raw_item) + capability = str(item.get("capability", "required_artifact")) + try: + actual = json.loads((workspace_path / str(item["path"])).read_text(encoding="utf-8")) + expected = read_json(hidden_root / str(item["expected_path"])) + score = float(actual == expected) + except (FileNotFoundError, json.JSONDecodeError, OSError): + score = 0.0 + scores[capability] = score + if score == 0.0: + failures.append(f"{capability}_failed") + return scores, failures + + def _observation_capabilities( + self, + outcomes: dict[str, float], + config: dict[str, object], + ) -> dict[str, float]: + groups = dict(config.get("capability_groups", {})) + if not groups: + return {} + scores: dict[str, float] = {} + for capability, methods_value in groups.items(): + methods = [str(method) for method in methods_value] + relevant = [ + value + for name, value in outcomes.items() + if any(name.startswith(f"test_{method}:") or name == method for method in methods) + ] + if relevant: + scores[str(capability)] = sum(relevant) / len(relevant) + return scores diff --git a/environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/tabular.py b/environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/tabular.py new file mode 100644 index 0000000..becaab3 --- /dev/null +++ b/environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/tabular.py @@ -0,0 +1,178 @@ +from __future__ import annotations + +import json +import time +from pathlib import Path + +from synthetic_workspace_gym.evaluators.base import BaseEvaluator +from synthetic_workspace_gym.evaluators.metrics import ( + baseline_normalized_score, + row_diff_diagnostics, + row_overlap_metrics, + weighted_match_score, +) +from synthetic_workspace_gym.schemas import EnvironmentManifest, EvaluatorResult +from synthetic_workspace_gym.utils.io import read_json + + +class TabularEvaluator(BaseEvaluator): + def capability_scores(self, expected: object, actual: object) -> dict[str, float]: + expected_rows = expected if isinstance(expected, list) else [] + actual_rows = actual if isinstance(actual, list) else [] + expected_keys = {frozenset(row) for row in expected_rows if isinstance(row, dict)} + actual_keys = {frozenset(row) for row in actual_rows if isinstance(row, dict)} + expected_ids = [str(row.get("account_id")) for row in expected_rows if isinstance(row, dict)] + actual_ids = [str(row.get("account_id")) for row in actual_rows if isinstance(row, dict)] + expected_counts = { + str(row.get("account_id")): row.get("event_count") for row in expected_rows if isinstance(row, dict) + } + actual_counts = { + str(row.get("account_id")): row.get("event_count") for row in actual_rows if isinstance(row, dict) + } + expected_amounts = { + str(row.get("account_id")): row.get("total_amount") for row in expected_rows if isinstance(row, dict) + } + actual_amounts = { + str(row.get("account_id")): row.get("total_amount") for row in actual_rows if isinstance(row, dict) + } + return { + "schema_contract": float(bool(actual_rows) and actual_keys == expected_keys), + "identity_resolution": float(sorted(actual_ids) == sorted(expected_ids)), + "filtering_and_deduplication": float(actual_counts == expected_counts), + "numeric_aggregation": float(actual_amounts == expected_amounts), + "deterministic_ordering": float(actual_ids == sorted(actual_ids) and bool(actual_ids)), + } + + def required_json_artifact_scores( + self, + workspace_path: Path, + hidden_root: Path, + config: dict[str, object], + ) -> tuple[dict[str, float], list[str]]: + scores: dict[str, float] = {} + failures: list[str] = [] + for raw_item in config.get("required_json_artifacts", []): + item = dict(raw_item) + capability = str(item.get("capability", "required_artifact")) + try: + actual = json.loads((workspace_path / str(item["path"])).read_text(encoding="utf-8")) + expected = read_json(hidden_root / str(item["expected_path"])) + score = float(actual == expected) + except (FileNotFoundError, json.JSONDecodeError, OSError): + score = 0.0 + scores[capability] = score + if score == 0.0: + failures.append(f"{capability}_failed") + return scores, failures + + def evaluate(self, workspace_path: Path, manifest: EnvironmentManifest, hidden_root: Path) -> EvaluatorResult: + started = time.perf_counter() + config = read_json(hidden_root / "evaluator_config.json") + expected = read_json(hidden_root / "expected_output.json") + execution = 1.0 + if config.get("entrypoint"): + precomputed_path = hidden_root / "precomputed_execution.json" + if not precomputed_path.is_file(): + raise ValueError("tabular execution must be collected in the isolated runner runtime") + completed = dict(read_json(precomputed_path).get("visible", {})) + execution = float(int(completed.get("returncode", 1)) == 0) + if not execution: + return EvaluatorResult( + success=False, + score=0.0, + subscores={"execution": 0.0, "output_exists": 0.0}, + failure_labels=["execution_failed"], + diagnostics={ + "stdout": completed.get("stdout", ""), + "stderr": completed.get("stderr", ""), + "returncode": completed.get("returncode", 1), + }, + runtime_seconds=time.perf_counter() - started, + ) + output_path = workspace_path / config["output_path"] + if not output_path.exists(): + return EvaluatorResult( + success=False, + score=0.0 if config.get("score_normalization") else 0.2, + subscores={ + "output_exists": 0.0, + "valid_json": 0.0, + "row_precision": 0.0, + "row_recall": 0.0, + "row_f1": 0.0, + "exact_match": 0.0, + }, + failure_labels=["output_missing"], + diagnostics={"required_output_path": config["output_path"]}, + runtime_seconds=time.perf_counter() - started, + ) + try: + actual = json.loads(output_path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + return EvaluatorResult( + success=False, + score=0.0 if config.get("score_normalization") else 0.2, + subscores={ + "output_exists": 1.0, + "valid_json": 0.0, + "row_precision": 0.0, + "row_recall": 0.0, + "row_f1": 0.0, + "exact_match": 0.0, + }, + failure_labels=["invalid_json"], + diagnostics={ + "error": str(exc), + "required_output_path": config["output_path"], + }, + runtime_seconds=time.perf_counter() - started, + ) + + artifact_scores, artifact_failures = self.required_json_artifact_scores(workspace_path, hidden_root, config) + artifacts_valid = all(score == 1.0 for score in artifact_scores.values()) + success = actual == expected and artifacts_valid + metrics = row_overlap_metrics(expected, actual) + score = weighted_match_score(output_exists=1.0, valid_structure=1.0, metrics=metrics) + capability_scores: dict[str, float] = {} + if config.get("capability_scoring"): + capability_scores = self.capability_scores(expected, actual) + score = sum(capability_scores.values()) / len(capability_scores) + if ( + capability_scores["filtering_and_deduplication"] == 0.0 + and capability_scores["numeric_aggregation"] == 0.0 + ): + score = min(score, 0.15) + elif capability_scores["numeric_aggregation"] == 0.0: + score = min(score, 0.40) + if artifact_failures: + score = min(score, float(config.get("required_artifact_failure_cap", 0.30))) + raw_score = score + if config.get("initial_score") is not None: + score = baseline_normalized_score(raw_score, float(config["initial_score"])) + if success: + score = 1.0 + diagnostics = { + "required_output_path": config["output_path"], + "expected_rows": len(expected), + "actual_rows": len(actual) if isinstance(actual, list) else None, + "raw_score": raw_score, + "initial_score": config.get("initial_score"), + } + if not success: + diagnostics["expected_preview"] = expected[:2] if isinstance(expected, list) else expected + diagnostics["actual_preview"] = actual[:2] if isinstance(actual, list) else actual + diagnostics.update(row_diff_diagnostics(expected, actual)) + return EvaluatorResult( + success=success, + score=score, + subscores={ + "execution": execution, + "output_exists": 1.0, + "valid_json": 1.0, + **metrics, + **{f"capability_{name}": value for name, value in {**capability_scores, **artifact_scores}.items()}, + }, + failure_labels=([] if success else list(dict.fromkeys(["output_mismatch", *artifact_failures]))), + diagnostics=diagnostics, + runtime_seconds=time.perf_counter() - started, + ) diff --git a/environments/synthetic_workspace_gym/synthetic_workspace_gym/schemas/__init__.py b/environments/synthetic_workspace_gym/synthetic_workspace_gym/schemas/__init__.py new file mode 100644 index 0000000..7c50696 --- /dev/null +++ b/environments/synthetic_workspace_gym/synthetic_workspace_gym/schemas/__init__.py @@ -0,0 +1,33 @@ +from .common import ( + Action, + ActionType, + ComplexityProfile, + EnvironmentFamily, + EnvironmentManifest, + EnvironmentSpec, + EpisodeSummary, + EvaluatorResult, + ObservabilitySettings, + ToolObservation, + ToolPermissions, + ToolState, + TrajectoryEvent, + utc_timestamp, +) + +__all__ = [ + "Action", + "ActionType", + "ComplexityProfile", + "EnvironmentFamily", + "EnvironmentManifest", + "EnvironmentSpec", + "EpisodeSummary", + "EvaluatorResult", + "ObservabilitySettings", + "ToolObservation", + "ToolPermissions", + "ToolState", + "TrajectoryEvent", + "utc_timestamp", +] diff --git a/environments/synthetic_workspace_gym/synthetic_workspace_gym/schemas/common.py b/environments/synthetic_workspace_gym/synthetic_workspace_gym/schemas/common.py new file mode 100644 index 0000000..689f08f --- /dev/null +++ b/environments/synthetic_workspace_gym/synthetic_workspace_gym/schemas/common.py @@ -0,0 +1,287 @@ +from __future__ import annotations + +from dataclasses import dataclass, field, fields +from datetime import datetime, timezone +from enum import Enum +from typing import Any + +JSONValue = Any + + +class SerializableDataclass: + def to_dict(self) -> dict[str, Any]: + payload: dict[str, Any] = {} + for item in fields(self): + payload[item.name] = serialize_value(getattr(self, item.name)) + return payload + + +def serialize_value(value: Any) -> Any: + if isinstance(value, Enum): + return value.value + if isinstance(value, SerializableDataclass): + return value.to_dict() + if isinstance(value, list): + return [serialize_value(item) for item in value] + if isinstance(value, dict): + return {str(key): serialize_value(item) for key, item in value.items()} + return value + + +class EnvironmentFamily(str, Enum): + TABULAR = "tabular" + SCRIPT_REPAIR = "script_repair" + PIPELINE = "pipeline" + RETRIEVAL_WORKSPACE = "retrieval_workspace" + COMPOSITE_WORKSPACE = "composite_workspace" + + +class ActionType(str, Enum): + READ_FILE = "read_file" + WRITE_FILE = "write_file" + APPEND_FILE = "append_file" + LIST_DIRECTORY = "list_directory" + RUN_SHELL = "run_shell" + RUN_PYTHON = "run_python" + SUBMIT = "submit" + + +@dataclass(slots=True) +class ToolPermissions(SerializableDataclass): + read_file: bool = True + write_file: bool = True + append_file: bool = True + list_directory: bool = True + run_shell: bool = True + run_python: bool = True + submit: bool = True + shell_timeout_seconds: int = 10 + python_timeout_seconds: int = 10 + + def enabled_tools(self) -> list[str]: + enabled: list[str] = [] + if self.read_file: + enabled.append(ActionType.READ_FILE.value) + if self.write_file: + enabled.append(ActionType.WRITE_FILE.value) + if self.append_file: + enabled.append(ActionType.APPEND_FILE.value) + if self.list_directory: + enabled.append(ActionType.LIST_DIRECTORY.value) + if self.run_shell: + enabled.append(ActionType.RUN_SHELL.value) + if self.run_python: + enabled.append(ActionType.RUN_PYTHON.value) + if self.submit: + enabled.append(ActionType.SUBMIT.value) + return enabled + + @classmethod + def from_dict(cls, payload: dict[str, Any]) -> "ToolPermissions": + return cls(**payload) + + +@dataclass(slots=True) +class ObservabilitySettings(SerializableDataclass): + show_instruction: bool = True + include_workspace_digest: bool = True + redact_hidden_paths: bool = True + + @classmethod + def from_dict(cls, payload: dict[str, Any]) -> "ObservabilitySettings": + return cls(**payload) + + +@dataclass(slots=True) +class ComplexityProfile(SerializableDataclass): + file_count: int + distractor_count: int + dependency_depth: int + reasoning_hops: int + transformation_count: int + bug_subtlety: int + execution_required: bool + output_constraint_strength: int + + @classmethod + def from_dict(cls, payload: dict[str, Any]) -> "ComplexityProfile": + return cls(**payload) + + +@dataclass(slots=True) +class EnvironmentSpec(SerializableDataclass): + env_family: EnvironmentFamily + difficulty: int + seed: int + scenario_id: str | None = None + max_steps: int = 12 + time_limit_seconds: int = 60 + tool_permissions: ToolPermissions = field(default_factory=ToolPermissions) + observability: ObservabilitySettings = field(default_factory=ObservabilitySettings) + task_params: dict[str, JSONValue] = field(default_factory=dict) + evaluator_params: dict[str, JSONValue] = field(default_factory=dict) + generation_params: dict[str, JSONValue] = field(default_factory=dict) + complexity_profile: ComplexityProfile | None = None + + def __post_init__(self) -> None: + self.env_family = EnvironmentFamily(self.env_family) + if self.scenario_id == "": + self.scenario_id = None + if not 1 <= self.difficulty <= 5: + raise ValueError("difficulty must be between 1 and 5") + if self.max_steps <= 0: + raise ValueError("max_steps must be positive") + if self.time_limit_seconds <= 0: + raise ValueError("time_limit_seconds must be positive") + + @classmethod + def from_dict(cls, payload: dict[str, Any]) -> "EnvironmentSpec": + return cls( + env_family=EnvironmentFamily(payload["env_family"]), + difficulty=payload["difficulty"], + seed=payload["seed"], + scenario_id=payload.get("scenario_id"), + max_steps=payload.get("max_steps", 12), + time_limit_seconds=payload.get("time_limit_seconds", 60), + tool_permissions=ToolPermissions.from_dict(payload.get("tool_permissions", {})), + observability=ObservabilitySettings.from_dict(payload.get("observability", {})), + task_params=payload.get("task_params", {}), + evaluator_params=payload.get("evaluator_params", {}), + generation_params=payload.get("generation_params", {}), + complexity_profile=( + ComplexityProfile.from_dict(payload["complexity_profile"]) + if payload.get("complexity_profile") + else None + ), + ) + + +@dataclass(slots=True) +class EnvironmentManifest(SerializableDataclass): + env_id: str + family: EnvironmentFamily + difficulty: int + seed: int + instruction: str + workspace_root: str + visible_files: list[str] + hidden_root: str + hidden_files: list[str] + tool_permissions: ToolPermissions + max_steps: int + time_limit_seconds: int + metadata: dict[str, JSONValue] + evaluator_entrypoint: str + reference_solution: dict[str, JSONValue] = field(default_factory=dict) + + def __post_init__(self) -> None: + self.family = EnvironmentFamily(self.family) + + @classmethod + def from_dict(cls, payload: dict[str, Any]) -> "EnvironmentManifest": + return cls( + env_id=payload["env_id"], + family=EnvironmentFamily(payload["family"]), + difficulty=payload["difficulty"], + seed=payload["seed"], + instruction=payload["instruction"], + workspace_root=payload["workspace_root"], + visible_files=list(payload.get("visible_files", [])), + hidden_root=payload.get("hidden_root", "hidden"), + hidden_files=list(payload.get("hidden_files", [])), + tool_permissions=ToolPermissions.from_dict(payload.get("tool_permissions", {})), + max_steps=payload["max_steps"], + time_limit_seconds=payload.get("time_limit_seconds", 60), + metadata=payload.get("metadata", {}), + evaluator_entrypoint=payload["evaluator_entrypoint"], + reference_solution=payload.get("reference_solution", {}), + ) + + +@dataclass(slots=True) +class EvaluatorResult(SerializableDataclass): + success: bool + score: float + subscores: dict[str, float] = field(default_factory=dict) + failure_labels: list[str] = field(default_factory=list) + diagnostics: dict[str, JSONValue] = field(default_factory=dict) + runtime_seconds: float = 0.0 + + @classmethod + def from_dict(cls, payload: dict[str, Any]) -> "EvaluatorResult": + return cls( + success=payload["success"], + score=payload["score"], + subscores=payload.get("subscores", {}), + failure_labels=payload.get("failure_labels", []), + diagnostics=payload.get("diagnostics", {}), + runtime_seconds=payload.get("runtime_seconds", 0.0), + ) + + +@dataclass(slots=True) +class Action(SerializableDataclass): + action_type: ActionType + arguments: dict[str, JSONValue] = field(default_factory=dict) + + def __post_init__(self) -> None: + self.action_type = ActionType(self.action_type) + + +@dataclass(slots=True) +class ToolObservation(SerializableDataclass): + success: bool + message: str + content: str | None = None + stdout: str = "" + stderr: str = "" + exit_code: int | None = None + listing: list[str] = field(default_factory=list) + error: str | None = None + touched_files: list[str] = field(default_factory=list) + workspace_digest: str | None = None + + +@dataclass(slots=True) +class TrajectoryEvent(SerializableDataclass): + step_index: int + timestamp: str + action_type: ActionType + action_arguments: dict[str, JSONValue] + observation_summary: str + stdout: str = "" + stderr: str = "" + exit_code: int | None = None + files_touched: list[str] = field(default_factory=list) + workspace_digest: str | None = None + success: bool = True + + def __post_init__(self) -> None: + self.action_type = ActionType(self.action_type) + + +@dataclass(slots=True) +class ToolState(SerializableDataclass): + step_index: int + remaining_steps: int + available_tools: list[str] + recent_files: list[str] = field(default_factory=list) + last_exit_code: int | None = None + submitted: bool = False + + +@dataclass(slots=True) +class EpisodeSummary(SerializableDataclass): + episode_id: str + env_id: str + agent_name: str + step_count: int + submitted: bool + duration_seconds: float + files_touched: list[str] + evaluation: EvaluatorResult + artifact_root: str + + +def utc_timestamp() -> str: + return datetime.now(tz=timezone.utc).isoformat() diff --git a/environments/synthetic_workspace_gym/synthetic_workspace_gym/taskset.py b/environments/synthetic_workspace_gym/synthetic_workspace_gym/taskset.py new file mode 100644 index 0000000..986a064 --- /dev/null +++ b/environments/synthetic_workspace_gym/synthetic_workspace_gym/taskset.py @@ -0,0 +1,749 @@ +from __future__ import annotations + +import asyncio +import base64 +import gzip +import json +import posixpath +import textwrap +from collections.abc import Iterable +from dataclasses import dataclass +from importlib.resources import files +from pathlib import Path, PurePosixPath +from typing import Any, Literal + +import verifiers.v1 as vf +from huggingface_hub import hf_hub_download +from pydantic import Field, field_validator +from verifiers.v1.runtimes import provision_runtime + +Family = Literal[ + "tabular", + "script_repair", + "pipeline", + "retrieval_workspace", + "composite_workspace", +] + +DEFAULT_MANIFEST = "sft-easy-v4" +WORKDIR = "/workspace" +EXECUTION_ROOT = "/opt/swg-exec" +GRADER_EXECUTION_ROOT = "/opt/swg-grade/execution" +SYSTEM_PROMPT = "Work in /workspace and complete the task using the provided terminal tools." + +WORKSPACE_INVENTORY_SCRIPT = textwrap.dedent( + """ + import json + import os + import stat + import sys + + root = sys.argv[1] + max_files = int(sys.argv[2]) + max_directories = int(sys.argv[3]) + max_file_bytes = int(sys.argv[4]) + max_total_bytes = int(sys.argv[5]) + directories = [] + files = [] + total_bytes = 0 + + if not os.path.isdir(root): + raise SystemExit("workspace root is missing") + for current, dirnames, filenames in os.walk(root, topdown=True, followlinks=False): + dirnames.sort() + filenames.sort() + for name in dirnames: + path = os.path.join(current, name) + mode = os.lstat(path).st_mode + relative = os.path.relpath(path, root) + if stat.S_ISLNK(mode) or not stat.S_ISDIR(mode): + raise SystemExit(f"unsupported workspace directory entry: {relative}") + directories.append(relative) + if len(directories) > max_directories: + raise SystemExit("workspace directory-count limit exceeded") + for name in filenames: + path = os.path.join(current, name) + info = os.lstat(path) + relative = os.path.relpath(path, root) + if stat.S_ISLNK(info.st_mode) or not stat.S_ISREG(info.st_mode): + raise SystemExit(f"unsupported workspace file entry: {relative}") + if info.st_size > max_file_bytes: + raise SystemExit(f"workspace per-file byte limit exceeded: {relative}") + files.append(relative) + total_bytes += info.st_size + if len(files) > max_files: + raise SystemExit("workspace file-count limit exceeded") + if total_bytes > max_total_bytes: + raise SystemExit("workspace total-byte limit exceeded") + print(json.dumps({"directories": directories, "files": files}, sort_keys=True)) + """ +).strip() + +WORKSPACE_CLEAR_SCRIPT = textwrap.dedent( + """ + import os + import shutil + import sys + + root = sys.argv[1] + for entry in os.scandir(root): + if entry.is_dir(follow_symlinks=False): + shutil.rmtree(entry.path) + else: + os.unlink(entry.path) + """ +).strip() + + +class SyntheticWorkspaceData(vf.TaskData): + family: Family + scenario: str + difficulty: int + seed: int + split: str + generation_fingerprint: str + task_blob: str + visible_files: list[str] + initial_digests: dict[str, str] + grader_image: str + dataset_repo: str + dataset_revision: str + + +class SyntheticWorkspaceState(vf.State): + evaluator_result: dict[str, Any] = Field(default_factory=dict) + changed_file_count: int = 0 + final_file_count: int = 0 + + +class SyntheticWorkspaceTaskConfig(vf.TaskConfig): + max_workspace_files: int = Field(default=1024, gt=0) + max_workspace_directories: int = Field(default=1024, gt=0) + max_file_bytes: int = Field(default=8 * 1024 * 1024, gt=0) + max_workspace_bytes: int = Field(default=64 * 1024 * 1024, gt=0) + max_result_bytes: int = Field(default=2 * 1024 * 1024, gt=0) + candidate_execution_timeout_seconds: float = Field(default=120.0, gt=0) + candidate_kill_grace_seconds: float = Field(default=5.0, gt=0) + + +@dataclass(frozen=True) +class WorkspaceSnapshot: + directories: tuple[str, ...] + files: dict[str, bytes] + + +class SyntheticWorkspaceTasksetConfig(vf.TasksetConfig): + manifest: str = DEFAULT_MANIFEST + families: list[Family] = Field(default_factory=list) + difficulties: list[int] = Field(default_factory=list) + tasks: list[str] = Field(default_factory=list) + task: SyntheticWorkspaceTaskConfig = SyntheticWorkspaceTaskConfig() + + @field_validator("manifest") + @classmethod + def validate_manifest_name(cls, value: str) -> str: + if not value or any(token in value for token in ("/", "\\", "..")): + raise ValueError("manifest must be a configured SWG curriculum name") + return value + + @field_validator("difficulties") + @classmethod + def validate_difficulties(cls, values: list[int]) -> list[int]: + if any(value < 1 or value > 5 for value in values): + raise ValueError("difficulties must be between 1 and 5") + return values + + +class SyntheticWorkspaceTask( + vf.Task[ + SyntheticWorkspaceData, + SyntheticWorkspaceState, + SyntheticWorkspaceTaskConfig, + ] +): + NEEDS_CONTAINER = True + + async def setup(self, runtime: vf.Runtime) -> None: + await _restore_baked_workspace(runtime, self.data.name or "swg-task") + + async def finalize( + self, + trace: vf.Trace[SyntheticWorkspaceData, SyntheticWorkspaceState], + runtime: vf.Runtime, + ) -> None: + task_blob = await asyncio.to_thread( + _load_task_blob, + self.data.dataset_repo, + self.data.dataset_revision, + self.data.task_blob, + ) + if task_blob.get("task_id") != self.data.name: + raise ValueError("downloaded SWG task blob does not match the selected task") + manifest = dict(task_blob["manifest"]) + hidden_files = _decode_tree(task_blob["hidden_files"]) + execution_files = _decode_tree(task_blob.get("execution_files", {})) + execution_plan = dict(task_blob["execution_plan"]) + snapshot = await _snapshot_workspace( + runtime, + max_files=self.config.max_workspace_files, + max_directories=self.config.max_workspace_directories, + max_file_bytes=self.config.max_file_bytes, + max_total_bytes=self.config.max_workspace_bytes, + ) + + runner_config = _isolated_config(runtime, image=self.data.image or "") + async with provision_runtime(runner_config, name=f"swg-runner-{trace.id}") as runner: + await runner.prepare_setup() + await _restore_baked_workspace(runner, self.data.name or "swg-task") + await _replace_workspace(runner, snapshot) + for relative_path, content in execution_files.items(): + await runner.write(_safe_path(EXECUTION_ROOT, relative_path), content) + await runner.prepare_execution([]) + execution, execution_outputs, executed_snapshot = await _execute_plan( + runner, + execution_plan, + execution_files, + max_files=self.config.max_workspace_files, + max_directories=self.config.max_workspace_directories, + max_file_bytes=self.config.max_file_bytes, + max_total_bytes=self.config.max_workspace_bytes, + timeout_seconds=self.config.candidate_execution_timeout_seconds, + kill_grace_seconds=self.config.candidate_kill_grace_seconds, + ) + + grader_config = _isolated_config(runtime, image=self.data.grader_image) + async with provision_runtime(grader_config, name=f"swg-grader-{trace.id}") as grader: + await grader.prepare_setup() + mkdir = await grader.run( + [ + "/bin/mkdir", + "-p", + WORKDIR, + "/opt/swg-grade/hidden", + "/opt/swg-grade/lib", + GRADER_EXECUTION_ROOT, + ], + {}, + ) + if mkdir.exit_code != 0: + raise vf.SandboxError(f"grader setup failed: {(mkdir.stderr or mkdir.stdout).strip()[-2000:]}") + await _replace_workspace(grader, executed_snapshot) + for relative_path, content in hidden_files.items(): + await grader.write(_safe_path("/opt/swg-grade/hidden", relative_path), content) + for relative_path, content in execution_outputs.items(): + await grader.write(_safe_path(GRADER_EXECUTION_ROOT, relative_path), content) + for relative_path, content in _grader_support_files().items(): + await grader.write(_safe_path("/opt/swg-grade/lib", relative_path), content) + await grader.write( + "/opt/swg-grade/trusted_grader.py", Path(__file__).with_name("trusted_grader.py").read_bytes() + ) + await grader.write("/opt/swg-grade/manifest.json", _json_bytes(manifest)) + await grader.write("/opt/swg-grade/initial-digests.json", _json_bytes(self.data.initial_digests)) + await grader.write("/opt/swg-grade/execution.json", _json_bytes(execution)) + await grader.prepare_execution([]) + result = await grader.run( + [ + "/usr/local/bin/python", + "/opt/swg-grade/trusted_grader.py", + WORKDIR, + "/opt/swg-grade/hidden", + "/opt/swg-grade/manifest.json", + "/opt/swg-grade/initial-digests.json", + "/opt/swg-grade/execution.json", + GRADER_EXECUTION_ROOT, + "/opt/swg-grade/result.json", + ], + {}, + ) + if result.exit_code != 0: + detail = (result.stderr or result.stdout).strip()[-2000:] + raise RuntimeError(f"trusted SWG grader failed: {detail}") + evaluator_result = json.loads( + (await grader.read("/opt/swg-grade/result.json", max_bytes=self.config.max_result_bytes)).decode() + ) + + trace.state.changed_file_count = int(evaluator_result.pop("changed_file_count", 0)) + trace.state.final_file_count = int(evaluator_result.pop("final_file_count", 0)) + trace.state.evaluator_result = evaluator_result + trace.info["synthetic_workspace_gym"] = { + "family": self.data.family, + "scenario": self.data.scenario, + "difficulty": self.data.difficulty, + "seed": self.data.seed, + "split": self.data.split, + "generation_fingerprint": self.data.generation_fingerprint, + "success": bool(evaluator_result.get("success", False)), + "failure_labels": list(evaluator_result.get("failure_labels", [])), + "secure_execution_protocol": execution_plan["kind"], + } + + @vf.reward(weight=1.0) + async def workspace_score(self, trace: vf.Trace[SyntheticWorkspaceData, SyntheticWorkspaceState]) -> float: + return float(trace.state.evaluator_result.get("score", 0.0)) + + @vf.metric + async def workspace_metrics( + self, + trace: vf.Trace[SyntheticWorkspaceData, SyntheticWorkspaceState], + ) -> dict[str, float]: + result = trace.state.evaluator_result + metrics = { + "success": float(bool(result.get("success", False))), + "changed_file_count": float(trace.state.changed_file_count), + "final_file_count": float(trace.state.final_file_count), + } + for name, value in dict(result.get("subscores", {})).items(): + metrics[f"subscore/{name}"] = float(value) + return metrics + + async def apply_gold_solution(self, runtime: vf.Runtime) -> None: + task_blob = await asyncio.to_thread( + _load_task_blob, + self.data.dataset_repo, + self.data.dataset_revision, + self.data.task_blob, + ) + solution_files = dict(task_blob["manifest"].get("reference_solution", {}).get("files", {})) + for relative_path, content in solution_files.items(): + await runtime.write(_safe_path(WORKDIR, relative_path), str(content).encode()) + + +class SyntheticWorkspaceTaskset(vf.Taskset[SyntheticWorkspaceTask, SyntheticWorkspaceTasksetConfig]): + def load(self) -> Iterable[SyntheticWorkspaceTask]: + artifacts = _load_artifacts() + dataset_repo = str(artifacts["dataset_repo"]) + dataset_revision = str(artifacts["dataset_revision"]) + images = dict(artifacts["images"]) + assignments = _load_assignments(dataset_repo, dataset_revision, self.config.manifest) + image = _runtime_image(images.get(self.config.manifest), f"agent image for {self.config.manifest}") + grader_image = _runtime_image(images.get("grader"), "grader image") + + selected_families = set(self.config.families) + selected_difficulties = set(self.config.difficulties) + selected_tasks = set(self.config.tasks) + matched = 0 + for assignment in assignments: + family = str(assignment["family"]) + difficulty = int(assignment["difficulty"]) + task_id = str(assignment["task_id"]) + if selected_families and family not in selected_families: + continue + if selected_difficulties and difficulty not in selected_difficulties: + continue + if selected_tasks and task_id not in selected_tasks: + continue + data = SyntheticWorkspaceData( + idx=matched, + name=task_id, + description=f"SWG {family} workspace task", + prompt=str(assignment["prompt"]), + system_prompt=SYSTEM_PROMPT, + image=image, + workdir=WORKDIR, + network_allow=[], + network_block=["*"], + timeout=vf.TaskTimeout(setup=120, agent=900, finalize=900, scoring=60), + resources=vf.TaskResources(cpu=2, memory=4, disk=4), + family=family, + scenario=str(assignment["scenario"]), + difficulty=difficulty, + seed=int(assignment["seed"]), + split=str(assignment["split"]), + generation_fingerprint=str(assignment["generation_fingerprint"]), + task_blob=str(assignment["task_blob"]), + visible_files=[str(path) for path in assignment["visible_files"]], + initial_digests={str(path): str(digest) for path, digest in assignment["initial_digests"].items()}, + grader_image=grader_image, + dataset_repo=dataset_repo, + dataset_revision=dataset_revision, + ) + matched += 1 + yield SyntheticWorkspaceTask(data, self.config.task) + if matched == 0: + raise ValueError("no SWG tasks matched the requested manifest filters") + + +async def _restore_baked_workspace(runtime: vf.Runtime, task_id: str) -> None: + source = _safe_path("/opt/swg/workspaces", task_id) + result = await runtime.run(["/bin/test", "-d", source], {}) + if result.exit_code != 0: + raise RuntimeError(f"visible workspace is missing from the split image: {task_id}") + result = await runtime.run(["/bin/mkdir", "-p", WORKDIR], {}) + if result.exit_code == 0: + result = await runtime.run(["/bin/cp", "-a", f"{source}/.", WORKDIR], {}) + if result.exit_code == 0: + result = await runtime.run(["/bin/rm", "-rf", "/opt/swg/workspaces", "/opt/swg/task_ids.json"], {}) + if result.exit_code != 0: + raise RuntimeError(f"failed to restore the baked SWG workspace: {result.stderr}") + + +async def _snapshot_workspace( + runtime: vf.Runtime, + *, + max_files: int, + max_directories: int, + max_file_bytes: int, + max_total_bytes: int, +) -> WorkspaceSnapshot: + inventory = await runtime.run( + [ + "/usr/local/bin/python", + "-c", + WORKSPACE_INVENTORY_SCRIPT, + WORKDIR, + str(max_files), + str(max_directories), + str(max_file_bytes), + str(max_total_bytes), + ], + {}, + ) + if inventory.exit_code != 0: + detail = (inventory.stderr or inventory.stdout).strip()[-2000:] + raise vf.SandboxError(f"failed to inventory bounded workspace: {detail}") + try: + payload = json.loads(inventory.stdout) + directories = tuple(str(path) for path in payload["directories"]) + paths = tuple(str(path) for path in payload["files"]) + except (KeyError, TypeError, ValueError, json.JSONDecodeError) as exc: + raise vf.SandboxError("workspace inventory returned an invalid payload") from exc + if len(paths) > max_files or len(directories) > max_directories: + raise vf.SandboxError("workspace inventory exceeded configured entry limits") + for relative_path in (*directories, *paths): + _validate_workspace_path(relative_path) + + semaphore = asyncio.Semaphore(32) + + async def read(relative_path: str) -> tuple[str, bytes]: + async with semaphore: + content = await runtime.read( + _safe_path(WORKDIR, relative_path), + max_bytes=max_file_bytes + 1, + ) + if len(content) > max_file_bytes: + raise vf.SandboxError(f"workspace file exceeds the {max_file_bytes}-byte transfer limit") + return relative_path, content + + values = await asyncio.gather(*(read(path) for path in paths)) + snapshot = WorkspaceSnapshot(directories=directories, files=dict(values)) + total_bytes = sum(len(content) for content in snapshot.files.values()) + if total_bytes > max_total_bytes: + raise vf.SandboxError(f"workspace exceeds the {max_total_bytes}-byte aggregate transfer limit") + return snapshot + + +async def _replace_workspace(runtime: vf.Runtime, snapshot: WorkspaceSnapshot) -> None: + result = await runtime.run(["/usr/local/bin/python", "-c", WORKSPACE_CLEAR_SCRIPT, WORKDIR], {}) + if result.exit_code == 0: + directories = [WORKDIR, *(_safe_path(WORKDIR, path) for path in snapshot.directories)] + result = await runtime.run(["/bin/mkdir", "-p", *directories], {}) + if result.exit_code != 0: + raise vf.SandboxError(f"failed to replace workspace tree: {(result.stderr or result.stdout).strip()[-2000:]}") + for relative_path, content in snapshot.files.items(): + await runtime.write(_safe_path(WORKDIR, relative_path), content) + + +async def _execute_plan( + runtime: vf.Runtime, + plan: dict[str, Any], + execution_files: dict[str, bytes], + *, + max_files: int, + max_directories: int, + max_file_bytes: int, + max_total_bytes: int, + timeout_seconds: float, + kill_grace_seconds: float, +) -> tuple[dict[str, Any], dict[str, bytes], WorkspaceSnapshot]: + kind = str(plan["kind"]) + if kind == "observations": + output = f"{EXECUTION_ROOT}/observations.json" + await _remove_file(runtime, output) + result, timed_out = await _run_candidate( + runtime, + [ + "/usr/local/bin/python", + _safe_path(EXECUTION_ROOT, str(plan["runner"])), + WORKDIR, + output, + ], + {"PYTHONDONTWRITEBYTECODE": "1"}, + timeout_seconds=timeout_seconds, + kill_grace_seconds=kill_grace_seconds, + ) + observations = b"" + if result.exit_code == 0: + try: + observations = await _read_bounded(runtime, output, max_file_bytes) + except (FileNotFoundError, vf.SandboxError): + pass + snapshot = await _snapshot_workspace( + runtime, + max_files=max_files, + max_directories=max_directories, + max_file_bytes=max_file_bytes, + max_total_bytes=max_total_bytes, + ) + execution = { + "kind": kind, + "visible": _program_result(result, timed_out=timed_out), + "observations_present": bool(observations), + } + outputs = {"observations.json": observations} if observations else {} + return execution, outputs, snapshot + + if kind == "profiled-command": + visible, visible_output = await _run_entrypoint( + runtime, + str(plan["entrypoint"]), + str(plan["output"]), + max_file_bytes, + timeout_seconds=timeout_seconds, + kill_grace_seconds=kill_grace_seconds, + ) + snapshot = await _snapshot_workspace( + runtime, + max_files=max_files, + max_directories=max_directories, + max_file_bytes=max_file_bytes, + max_total_bytes=max_total_bytes, + ) + fixture_prefix = f"{plan['fixture_dir']}/" + for relative_path, content in execution_files.items(): + if relative_path.startswith(fixture_prefix): + await runtime.write(_safe_path(WORKDIR, relative_path.removeprefix(fixture_prefix)), content) + if visible["timed_out"]: + hidden = _timeout_program_result("hidden run skipped after visible execution timeout") + hidden_output = b"" + else: + hidden, hidden_output = await _run_entrypoint( + runtime, + str(plan["entrypoint"]), + str(plan["output"]), + max_file_bytes, + timeout_seconds=timeout_seconds, + kill_grace_seconds=kill_grace_seconds, + ) + outputs = {} + if visible_output: + outputs["visible-output.json"] = visible_output + if hidden_output: + outputs["hidden-output.json"] = hidden_output + return {"kind": kind, "visible": visible, "hidden": hidden}, outputs, snapshot + + if kind == "command" and plan.get("entrypoint"): + visible, _ = await _run_entrypoint( + runtime, + str(plan["entrypoint"]), + str(plan["output"]), + max_file_bytes, + timeout_seconds=timeout_seconds, + kill_grace_seconds=kill_grace_seconds, + ) + execution = {"kind": kind, "visible": visible} + else: + execution = {"kind": kind, "visible": {"returncode": 0, "deterministic": True}} + snapshot = await _snapshot_workspace( + runtime, + max_files=max_files, + max_directories=max_directories, + max_file_bytes=max_file_bytes, + max_total_bytes=max_total_bytes, + ) + return execution, {}, snapshot + + +async def _run_entrypoint( + runtime: vf.Runtime, + entrypoint: str, + output_path: str, + max_bytes: int, + *, + timeout_seconds: float, + kill_grace_seconds: float, +) -> tuple[dict[str, Any], bytes]: + command = ["/usr/local/bin/python", _safe_path(WORKDIR, entrypoint)] + output_file = _safe_path(WORKDIR, output_path) + await _remove_file(runtime, output_file) + environment = {"PYTHONDONTWRITEBYTECODE": "1"} + first, first_timed_out = await _run_candidate( + runtime, + command, + environment, + timeout_seconds=timeout_seconds, + kill_grace_seconds=kill_grace_seconds, + ) + output = b"" + if first.exit_code == 0: + try: + output = await _read_bounded(runtime, output_file, max_bytes) + except (FileNotFoundError, vf.SandboxError): + pass + await _remove_file(runtime, output_file) + second = first + second_timed_out = False + if first.exit_code == 0: + second, second_timed_out = await _run_candidate( + runtime, + command, + environment, + timeout_seconds=timeout_seconds, + kill_grace_seconds=kill_grace_seconds, + ) + second_output = b"" + if second.exit_code == 0: + try: + second_output = await _read_bounded(runtime, output_file, max_bytes) + except (FileNotFoundError, vf.SandboxError): + pass + return ( + { + **_program_result(first, timed_out=first_timed_out or second_timed_out), + "rerun_returncode": second.exit_code, + "deterministic": bool( + output and first.exit_code == 0 and second.exit_code == 0 and output == second_output + ), + "output_present": bool(output), + }, + output, + ) + + +async def _run_candidate( + runtime: vf.Runtime, + argv: list[str], + environment: dict[str, str], + *, + timeout_seconds: float, + kill_grace_seconds: float, +) -> tuple[vf.ProgramResult, bool]: + wrapped = [ + "/usr/bin/timeout", + "--signal=TERM", + f"--kill-after={kill_grace_seconds:.3f}s", + f"{timeout_seconds:.3f}s", + *argv, + ] + rpc_timeout = timeout_seconds + kill_grace_seconds + 30.0 + try: + result = await asyncio.wait_for(runtime.run(wrapped, environment), timeout=rpc_timeout) + except TimeoutError: + return vf.ProgramResult(124, "", "candidate execution exceeded the runtime RPC timeout"), True + return result, result.exit_code in {124, 137} + + +async def _remove_file(runtime: vf.Runtime, path: str) -> None: + result = await runtime.run(["/bin/rm", "-f", path], {}) + if result.exit_code != 0: + raise vf.SandboxError(f"failed to clear {path!r}: {(result.stderr or result.stdout).strip()[-2000:]}") + + +def _program_result(result: vf.ProgramResult, *, timed_out: bool = False) -> dict[str, Any]: + return { + "returncode": result.exit_code, + "timed_out": timed_out, + "stdout": result.stdout[-4000:], + "stderr": result.stderr[-4000:], + } + + +def _timeout_program_result(message: str) -> dict[str, Any]: + return {"returncode": 124, "timed_out": True, "stdout": "", "stderr": message} + + +async def _read_bounded(runtime: vf.Runtime, path: str, max_bytes: int) -> bytes: + content = await runtime.read(path) + if len(content) > max_bytes: + raise vf.SandboxError(f"{path!r} exceeds the {max_bytes}-byte transfer limit") + return content + + +def _isolated_config(runtime: vf.Runtime, *, image: str) -> vf.RuntimeConfig: + config = runtime.config + values = config.model_dump() + if "image" not in values: + raise RuntimeError("SWG secure grading requires an image-backed runtime") + values.update({"image": image, "workdir": WORKDIR}) + if "allow" in values: + values.update({"allow": [], "block": ["*"]}) + return type(config).model_validate(values) + + +def _safe_path(root: str, relative_path: str) -> str: + pure = PurePosixPath(relative_path.replace("\\", "/")) + if pure.is_absolute() or ".." in pure.parts or not pure.parts: + raise ValueError(f"unsafe SWG path: {relative_path!r}") + return posixpath.join(root, pure.as_posix()) + + +def _validate_workspace_path(relative_path: str) -> None: + if "\\" in relative_path: + raise vf.SandboxError(f"workspace path contains an unsupported backslash: {relative_path!r}") + try: + relative_path.encode("utf-8", errors="strict") + except UnicodeEncodeError as exc: + raise vf.SandboxError("workspace path is not valid UTF-8") from exc + _safe_path(WORKDIR, relative_path) + + +def _decode_tree(encoded: dict[str, str]) -> dict[str, bytes]: + return {path: base64.b64decode(content, validate=True) for path, content in encoded.items()} + + +def _json_bytes(payload: object) -> bytes: + return (json.dumps(payload, sort_keys=True, separators=(",", ":")) + "\n").encode() + + +def _load_artifacts() -> dict[str, Any]: + payload = json.loads(files("synthetic_workspace_gym").joinpath("artifacts.json").read_text(encoding="utf-8")) + if payload.get("dataset_revision") == "PENDING": + raise RuntimeError("SWG immutable dataset and image artifacts have not been published") + return payload + + +def _load_assignments(repo_id: str, revision: str, manifest: str) -> list[dict[str, Any]]: + path = hf_hub_download(repo_id, f"manifests/{manifest}.jsonl", repo_type="dataset", revision=revision) + return [json.loads(line) for line in Path(path).read_text(encoding="utf-8").splitlines() if line.strip()] + + +def _load_task_blob(repo_id: str, revision: str, blob_path: str) -> dict[str, Any]: + path = hf_hub_download(repo_id, blob_path, repo_type="dataset", revision=revision) + with gzip.open(path, "rt", encoding="utf-8") as source: + return json.load(source) + + +def _require_digest_pin(image: str, label: str) -> None: + marker = "@sha256:" + if marker not in image or len(image.rpartition(marker)[2]) != 64: + raise ValueError(f"{label} must be pinned by digest") + + +def _runtime_image(artifact: object, label: str) -> str: + if not isinstance(artifact, dict): + raise ValueError(f"{label} must declare runtime and container references") + runtime = str(artifact.get("runtime", "")) + container = str(artifact.get("container", "")) + _require_digest_pin(container, f"{label} container") + if not runtime.startswith("prime/") or ":" not in runtime.rpartition("/")[2]: + raise ValueError(f"{label} runtime must be a versioned public Prime image reference") + return runtime + + +def _grader_support_files() -> dict[str, bytes]: + package_root = Path(__file__).parent + payload = {"synthetic_workspace_gym/__init__.py": b""} + for directory_name in ("evaluators", "schemas", "utils"): + for path in sorted((package_root / directory_name).rglob("*.py")): + relative = path.relative_to(package_root).as_posix() + payload[f"synthetic_workspace_gym/{relative}"] = path.read_bytes() + return payload + + +__all__ = [ + "SyntheticWorkspaceData", + "SyntheticWorkspaceState", + "SyntheticWorkspaceTask", + "SyntheticWorkspaceTaskConfig", + "SyntheticWorkspaceTaskset", + "SyntheticWorkspaceTasksetConfig", +] diff --git a/environments/synthetic_workspace_gym/synthetic_workspace_gym/trusted_grader.py b/environments/synthetic_workspace_gym/synthetic_workspace_gym/trusted_grader.py new file mode 100644 index 0000000..4aac7b7 --- /dev/null +++ b/environments/synthetic_workspace_gym/synthetic_workspace_gym/trusted_grader.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +import hashlib +import json +import shutil +import sys +from pathlib import Path + + +def _digest(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _file_digests(root: Path) -> dict[str, str]: + return { + path.relative_to(root).as_posix(): _digest(path) + for path in sorted(root.rglob("*")) + if path.is_file() and not path.is_symlink() + } + + +def main() -> None: + if len(sys.argv) != 8: + raise SystemExit( + "usage: trusted_grader.py WORKSPACE HIDDEN MANIFEST INITIAL_DIGESTS EXECUTION EXECUTION_OUTPUTS OUTPUT" + ) + workspace, hidden, manifest_path, initial_path, execution_path, execution_outputs, output_path = map( + Path, + sys.argv[1:], + ) + sys.path.insert(0, str(Path(__file__).parent / "lib")) + + from synthetic_workspace_gym.evaluators.registry import get_evaluator + from synthetic_workspace_gym.schemas import EnvironmentManifest + + manifest = EnvironmentManifest.from_dict(json.loads(manifest_path.read_text(encoding="utf-8"))) + initial = json.loads(initial_path.read_text(encoding="utf-8")) + (hidden / "precomputed_execution.json").write_bytes(execution_path.read_bytes()) + trusted_outputs = hidden / "precomputed_outputs" + shutil.rmtree(trusted_outputs, ignore_errors=True) + shutil.copytree(execution_outputs, trusted_outputs) + current = _file_digests(workspace) + evaluator = get_evaluator( + manifest.family, + evaluator_entrypoint=manifest.evaluator_entrypoint, + ) + result = evaluator.evaluate(workspace, manifest, hidden).to_dict() + result["changed_file_count"] = sum( + 1 for path in set(initial) | set(current) if initial.get(path) != current.get(path) + ) + result["final_file_count"] = len(current) + output_path.write_text(json.dumps(result, sort_keys=True), encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/environments/synthetic_workspace_gym/synthetic_workspace_gym/utils/__init__.py b/environments/synthetic_workspace_gym/synthetic_workspace_gym/utils/__init__.py new file mode 100644 index 0000000..e849f62 --- /dev/null +++ b/environments/synthetic_workspace_gym/synthetic_workspace_gym/utils/__init__.py @@ -0,0 +1 @@ +"""Utility helpers for Synthetic Workspace Gym.""" diff --git a/environments/synthetic_workspace_gym/synthetic_workspace_gym/utils/io.py b/environments/synthetic_workspace_gym/synthetic_workspace_gym/utils/io.py new file mode 100644 index 0000000..9cd74cb --- /dev/null +++ b/environments/synthetic_workspace_gym/synthetic_workspace_gym/utils/io.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +import json +from dataclasses import asdict, is_dataclass +from datetime import datetime, timezone +from enum import Enum +from pathlib import Path +from typing import Any + + +def convert_to_json_compatible(value: Any) -> Any: + if is_dataclass(value): + return {key: convert_to_json_compatible(item) for key, item in asdict(value).items()} + if isinstance(value, Enum): + return value.value + if isinstance(value, Path): + return str(value) + if isinstance(value, datetime): + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + return value.isoformat() + if isinstance(value, dict): + return {str(key): convert_to_json_compatible(item) for key, item in value.items()} + if isinstance(value, (list, tuple, set)): + return [convert_to_json_compatible(item) for item in value] + return value + + +def to_json_compatible(value: Any) -> Any: + """Convert a value into a JSON-safe structure.""" + + return convert_to_json_compatible(value) + + +def write_json(path: Path, payload: Any, *, indent: int = 2) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(to_json_compatible(payload), indent=indent, sort_keys=True), encoding="utf-8") + + +def read_json(path: Path) -> Any: + return json.loads(path.read_text(encoding="utf-8")) + + +def write_jsonl(path: Path, rows: list[Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as handle: + for row in rows: + handle.write(json.dumps(to_json_compatible(row), sort_keys=True)) + handle.write("\n") + + +def write_text(path: Path, content: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") diff --git a/environments/synthetic_workspace_gym/synthetic_workspace_gym/utils/paths.py b/environments/synthetic_workspace_gym/synthetic_workspace_gym/utils/paths.py new file mode 100644 index 0000000..0f3d17a --- /dev/null +++ b/environments/synthetic_workspace_gym/synthetic_workspace_gym/utils/paths.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +import hashlib +import shutil +from pathlib import Path + + +def ensure_within_root(root: Path, relative_path: str) -> Path: + root = root.resolve() + candidate = (root / relative_path).resolve() + if candidate != root and root not in candidate.parents: + raise ValueError(f"Path '{relative_path}' escapes workspace root '{root}'.") + return candidate + + +def list_relative_files(root: Path) -> list[str]: + if not root.exists(): + return [] + files = [str(path.relative_to(root)).replace("\\", "/") for path in root.rglob("*") if path.is_file()] + return sorted(files) + + +def copy_tree(source: Path, destination: Path) -> None: + if destination.exists(): + shutil.rmtree(destination) + shutil.copytree(source, destination) + + +def file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + while True: + chunk = handle.read(64 * 1024) + if not chunk: + break + digest.update(chunk) + return digest.hexdigest() diff --git a/environments/synthetic_workspace_gym/synthetic_workspace_gym/utils/scratch.py b/environments/synthetic_workspace_gym/synthetic_workspace_gym/utils/scratch.py new file mode 100644 index 0000000..f3f852b --- /dev/null +++ b/environments/synthetic_workspace_gym/synthetic_workspace_gym/utils/scratch.py @@ -0,0 +1,18 @@ +from __future__ import annotations + +import shutil +from contextlib import contextmanager +from pathlib import Path +from uuid import uuid4 + + +@contextmanager +def scratch_directory(base_dir: Path, prefix: str): + base_dir.mkdir(parents=True, exist_ok=True) + path = base_dir / f"{prefix}{uuid4().hex[:10]}" + path.mkdir(parents=True, exist_ok=False) + try: + yield path + finally: + if path.exists(): + shutil.rmtree(path, ignore_errors=True) From 0f9424cfd8aa37a6b4a08a818d216725755cb978 Mon Sep 17 00:00:00 2001 From: ydnysh Date: Sat, 3 Oct 2026 21:44:04 +0530 Subject: [PATCH 2/4] Harden SWG taskset rewards and publish V5 --- .../synthetic_workspace_gym/README.md | 10 ++-- .../synthetic_workspace_gym/pyproject.toml | 4 +- .../synthetic_workspace_gym/artifacts.json | 53 +++++++++++-------- .../evaluators/metrics.py | 2 + .../evaluators/pipeline.py | 16 +++++- .../synthetic_workspace_gym/taskset.py | 30 +++++++++-- 6 files changed, 80 insertions(+), 35 deletions(-) diff --git a/environments/synthetic_workspace_gym/README.md b/environments/synthetic_workspace_gym/README.md index 3cf3f5c..6450498 100644 --- a/environments/synthetic_workspace_gym/README.md +++ b/environments/synthetic_workspace_gym/README.md @@ -8,7 +8,7 @@ Synthetic Workspace Gym (SWG) is a native Verifiers V1 taskset for training term The task declares `NEEDS_CONTAINER = True` and a working directory of `/workspace`. Verifiers and the selected standard harness own model interaction, terminal access, runtime provisioning, rollout timeouts, and trace capture. SWG does not wrap a harness or implement a rollout loop; it adds a shorter timeout only around candidate programs executed during trusted finalization. -Visible workspaces are baked into six digest-pinned images, one per curriculum or evaluation panel. Setup selects one workspace and removes the rest of the split payload before the agent starts. Hidden assets never enter the agent runtime or task state. +Visible workspaces are baked into six immutable Prime VM artifacts, one per curriculum or evaluation panel. Each pin records the public runtime tag together with Prime's artifact ID and CAS path. Setup selects one workspace and removes the rest of the split payload before the agent starts. Hidden assets never enter the agent runtime or task state. Finalization uses three trust domains: @@ -40,7 +40,7 @@ Run the required small integration smoke with the standard harness/runtime: uv run eval --taskset.id synthetic-workspace-gym -n 1 -r 2 --max-turns 4 ``` -Use the default `sft-easy-v4` mid-training manifest or select another immutable dataset manifest and filters: +Use the default `sft-easy-v5` mid-training manifest or select another immutable dataset manifest and filters: ```bash uv run eval --taskset.id synthetic-workspace-gym \ @@ -50,15 +50,15 @@ uv run eval --taskset.id synthetic-workspace-gym \ -n 20 -r 3 --rich false --no-push ``` -The purpose-specific manifests are `sft-easy-v4` (family-calibrated D1-D3 teacher traces), `sft-validation-v4` (scenario- and seed-held-out D1-D3), `rl-hard-v4` (family-calibrated D4-D5 atomic and composite training), and `rl-eval-v4` (family-stratified held-out D4-D5 scenarios, seeds, fixtures, and compositions). V4 records the exact generator commit, uses platform-independent task bytes, and measures untouched pipeline baselines through the same evaluator paths used at grading time. Verifiers' normal `-n`, `-r`, `--shuffle`, harness, model, sampling, and runtime options apply without environment-specific rollout arguments. +The purpose-specific manifests are `sft-easy-v5` (family-calibrated D1-D3 teacher traces), `sft-validation-v5` (scenario- and seed-held-out D1-D3), `rl-hard-v5` (family-calibrated D4-D5 atomic and composite training), and `rl-eval-v5` (family-stratified held-out D4-D5 scenarios, seeds, fixtures, and compositions). V5 records the exact public generator commit, calibrates untouched rewards through the shipped execution protocol, requires deterministic command outputs for reward, and transports observations through a trusted post-process frame. Verifiers' normal `-n`, `-r`, `--shuffle`, harness, model, sampling, and runtime options apply without environment-specific rollout arguments. -For difficulty calibration, use `eval-d1-d4-paired-panel-48-v3` and `eval-d5-family-calibration-40-v3`. The first holds family, scenario, and seed fixed while difficulty changes; the second reports D5 separately by family and retrieval profile. +For difficulty calibration, use `eval-d1-d4-paired-panel-48-v5` and `eval-d5-family-calibration-40-v5`. The first holds family, scenario, and seed fixed while difficulty changes; the second reports D5 separately by family and retrieval profile. ## Configuration | Field | Default | Purpose | | ------------------------------------------ | ------------- | -------------------------------------------------------------- | -| `manifest` | `sft-easy-v4` | Immutable curriculum or evaluation panel | +| `manifest` | `sft-easy-v5` | Immutable curriculum or evaluation panel | | `families` | all | Optional family filter | | `difficulties` | all | Optional difficulty filter (`1` through `5`) | | `tasks` | all | Optional exact task-ID filter | diff --git a/environments/synthetic_workspace_gym/pyproject.toml b/environments/synthetic_workspace_gym/pyproject.toml index 6819f7b..57e84c1 100644 --- a/environments/synthetic_workspace_gym/pyproject.toml +++ b/environments/synthetic_workspace_gym/pyproject.toml @@ -1,13 +1,13 @@ [project] name = "synthetic-workspace-gym" -version = "0.2.1" +version = "0.2.2" description = "Native Verifiers V1 workspace tasks for training terminal agents." readme = "README.md" requires-python = ">=3.11" license = "Apache-2.0" authors = [{ name = "Yadnyesh Chakane" }] tags = ["multi-turn", "tool-use", "workspace", "code", "v1"] -dependencies = ["huggingface-hub>=0.28,<2", "verifiers>=0.3.0"] +dependencies = ["huggingface-hub>=0.28,<2", "verifiers>=0.3.1,<0.4"] [build-system] requires = ["hatchling"] diff --git a/environments/synthetic_workspace_gym/synthetic_workspace_gym/artifacts.json b/environments/synthetic_workspace_gym/synthetic_workspace_gym/artifacts.json index d720fdc..e4421bd 100644 --- a/environments/synthetic_workspace_gym/synthetic_workspace_gym/artifacts.json +++ b/environments/synthetic_workspace_gym/synthetic_workspace_gym/artifacts.json @@ -1,36 +1,43 @@ { "dataset_repo": "ydnysh/synthetic-workspace-gym", - "dataset_revision": "90893216b0cf7a2f46a077ab44ab9d404c06a686", - "generator_revision": "533c4d345a6a17ba22eea8a68ffdc2f62c3bc0a2", - "artifact_integrity_sha256": "2eb246b24fbbd905e763afa48428e833744748d827e2c5d7fc02cf41194c8c27", + "dataset_revision": "3ba83f647b96556be8c5084584a2d86e93225684", + "generator_revision": "3bd13bb949846e7b7393afe81727f1127e984475", + "artifact_integrity_sha256": "128c3c984d91e7d46918e0f2f77cab4543943a25e11083bf18d0ad00dcf9f7f8", "images": { - "eval-d1-d4-paired-panel-48-v4": { - "runtime": "prime/yadnyesh/synthetic-workspace-gym-eval-d1-d4-paired-panel-48-v4:v4-2eb246b2", - "container": "cmee2dees0002x3lw5g0cbf8p/synthetic-workspace-gym-eval-d1-d4-paired-panel-48-v4@sha256:dcf86a3ca713ac26fc2c2abf4981e03e3fa9097eeca5f0b55499415df7d49a97" + "eval-d1-d4-paired-panel-48-v5": { + "prime_artifact_id": "objopl5cxo35njpwui2c81l0", + "prime_artifact_path": "cmee2dees0002x3lw5g0cbf8p/y9eqm22g31pgjf78oqr8hd9c/linux-amd64/rootfs-cas", + "runtime": "prime/yadnyesh/synthetic-workspace-gym-eval-d1-d4-paired-panel-48-v5:v5-128c3c98" }, - "eval-d5-family-calibration-40-v4": { - "runtime": "prime/yadnyesh/synthetic-workspace-gym-eval-d5-family-calibration-40-v4:v4-2eb246b2", - "container": "cmee2dees0002x3lw5g0cbf8p/synthetic-workspace-gym-eval-d5-family-calibration-40-v4@sha256:4a1d1edca62098d694d1783488defa937eb8dd6e241341c38920a183024ea2aa" + "eval-d5-family-calibration-40-v5": { + "prime_artifact_id": "ob985zt0e9wgma8yl7k4o2d6", + "prime_artifact_path": "cmee2dees0002x3lw5g0cbf8p/bm3csqsxl0qemvi65gux53no/linux-amd64/rootfs-cas", + "runtime": "prime/yadnyesh/synthetic-workspace-gym-eval-d5-family-calibration-40-v5:v5-128c3c98" }, "grader": { - "runtime": "prime/yadnyesh/synthetic-workspace-gym-grader:v4-2eb246b2", - "container": "cmee2dees0002x3lw5g0cbf8p/synthetic-workspace-gym-grader@sha256:60f131885799072b6c1fa36534bc7a33696620caa759392dd182ac5d282748d4" + "prime_artifact_id": "zws7hg0o30b3r6429mw3on0b", + "prime_artifact_path": "cmee2dees0002x3lw5g0cbf8p/bj7qh2sruhulqvoq8eak6uc3/linux-amd64/rootfs-cas", + "runtime": "prime/yadnyesh/synthetic-workspace-gym-grader:v5-128c3c98" }, - "rl-eval-v4": { - "runtime": "prime/yadnyesh/synthetic-workspace-gym-rl-eval-v4:v4-2eb246b2", - "container": "cmee2dees0002x3lw5g0cbf8p/synthetic-workspace-gym-rl-eval-v4@sha256:a297f0b462a94bb98375c8926a682d0d8142e14592b1d3782218a7e15b6ebe37" + "rl-eval-v5": { + "prime_artifact_id": "lo7gtb8kwssur90juv827c1t", + "prime_artifact_path": "cmee2dees0002x3lw5g0cbf8p/ct9aitaw6ws9ysej62cpt5fm/linux-amd64/rootfs-cas", + "runtime": "prime/yadnyesh/synthetic-workspace-gym-rl-eval-v5:v5-128c3c98" }, - "rl-hard-v4": { - "runtime": "prime/yadnyesh/synthetic-workspace-gym-rl-hard-v4:v4-2eb246b2", - "container": "cmee2dees0002x3lw5g0cbf8p/synthetic-workspace-gym-rl-hard-v4@sha256:bb5d996631237fe8164f6c529158e27e399fd390309d40271216a442033d01ff" + "rl-hard-v5": { + "prime_artifact_id": "vpjq1d8o9e8yanrdv4s0g877", + "prime_artifact_path": "cmee2dees0002x3lw5g0cbf8p/bi3n9fts4fpo9flqfpqlh5oz/linux-amd64/rootfs-cas", + "runtime": "prime/yadnyesh/synthetic-workspace-gym-rl-hard-v5:v5-128c3c98" }, - "sft-easy-v4": { - "runtime": "prime/yadnyesh/synthetic-workspace-gym-sft-easy-v4:v4-2eb246b2", - "container": "cmee2dees0002x3lw5g0cbf8p/synthetic-workspace-gym-sft-easy-v4@sha256:1ccada8a717ae8663dcdc6cba91c902a4c111d94b14897d314692579d0b171b1" + "sft-easy-v5": { + "prime_artifact_id": "i3upbkdq7eqbqae0pp8pl1qk", + "prime_artifact_path": "cmee2dees0002x3lw5g0cbf8p/od75fuujjdl5kpsl0uwb88k8/linux-amd64/rootfs-cas", + "runtime": "prime/yadnyesh/synthetic-workspace-gym-sft-easy-v5:v5-128c3c98" }, - "sft-validation-v4": { - "runtime": "prime/yadnyesh/synthetic-workspace-gym-sft-validation-v4:v4-2eb246b2", - "container": "cmee2dees0002x3lw5g0cbf8p/synthetic-workspace-gym-sft-validation-v4@sha256:7eeb9dabe9b32e85a1f88da92859fe3e7ef098f9c7b03ca44ce6ef7824707378" + "sft-validation-v5": { + "prime_artifact_id": "dxn6sxwuocuaar0k5thud9jw", + "prime_artifact_path": "cmee2dees0002x3lw5g0cbf8p/esuz0w5sjnj4zmcj5uafe2wm/linux-amd64/rootfs-cas", + "runtime": "prime/yadnyesh/synthetic-workspace-gym-sft-validation-v5:v5-128c3c98" } } } diff --git a/environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/metrics.py b/environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/metrics.py index a431754..657f43b 100644 --- a/environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/metrics.py +++ b/environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/metrics.py @@ -40,6 +40,8 @@ def weighted_match_score(*, output_exists: float, valid_structure: float, metric + 0.2 * metrics["row_recall"] + 0.1 * metrics["exact_match"] ) + if metrics["row_precision"] == 0.0 and metrics["row_recall"] == 0.0: + score = min(score, 0.2) return round(min(1.0, max(0.0, score)), 6) diff --git a/environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/pipeline.py b/environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/pipeline.py index 06c9288..73efc6d 100644 --- a/environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/pipeline.py +++ b/environments/synthetic_workspace_gym/synthetic_workspace_gym/evaluators/pipeline.py @@ -205,6 +205,8 @@ def evaluate(self, workspace_path: Path, manifest: EnvironmentManifest, hidden_r raw_score = score if config.get("initial_score") is not None: score = baseline_normalized_score(raw_score, float(config["initial_score"])) + if deterministic_rerun != 1.0: + score = 0.0 if success: score = 1.0 diagnostics = { @@ -231,7 +233,19 @@ def evaluate(self, workspace_path: Path, manifest: EnvironmentManifest, hidden_r **metrics, **{f"capability_{name}": value for name, value in capability_scores.items()}, }, - failure_labels=([] if success else list(dict.fromkeys(["output_mismatch", *artifact_failures]))), + failure_labels=( + [] + if success + else list( + dict.fromkeys( + [ + "output_mismatch", + *(("nondeterministic_output",) if deterministic_rerun != 1.0 else ()), + *artifact_failures, + ] + ) + ) + ), diagnostics=diagnostics, runtime_seconds=time.perf_counter() - started, ) diff --git a/environments/synthetic_workspace_gym/synthetic_workspace_gym/taskset.py b/environments/synthetic_workspace_gym/synthetic_workspace_gym/taskset.py index 986a064..d031d36 100644 --- a/environments/synthetic_workspace_gym/synthetic_workspace_gym/taskset.py +++ b/environments/synthetic_workspace_gym/synthetic_workspace_gym/taskset.py @@ -7,7 +7,7 @@ import posixpath import textwrap from collections.abc import Iterable -from dataclasses import dataclass +from dataclasses import dataclass, field from importlib.resources import files from pathlib import Path, PurePosixPath from typing import Any, Literal @@ -25,7 +25,7 @@ "composite_workspace", ] -DEFAULT_MANIFEST = "sft-easy-v4" +DEFAULT_MANIFEST = "sft-easy-v5" WORKDIR = "/workspace" EXECUTION_ROOT = "/opt/swg-exec" GRADER_EXECUTION_ROOT = "/opt/swg-grade/execution" @@ -307,6 +307,23 @@ async def apply_gold_solution(self, runtime: vf.Runtime) -> None: for relative_path, content in solution_files.items(): await runtime.write(_safe_path(WORKDIR, relative_path), str(content).encode()) + async def validate(self, runtime: vf.Runtime) -> bool: + await self.apply_gold_solution(runtime) + trace = _ValidationTrace( + id=f"validate-{self.data.name or 'swg-task'}", + state=SyntheticWorkspaceState(), + ) + await self.finalize(trace, runtime) # type: ignore[arg-type] + reward = await self.workspace_score(trace) # type: ignore[arg-type] + return bool(trace.state.evaluator_result.get("success")) and reward == 1.0 + + +@dataclass +class _ValidationTrace: + id: str + state: SyntheticWorkspaceState + info: dict[str, Any] = field(default_factory=dict) + class SyntheticWorkspaceTaskset(vf.Taskset[SyntheticWorkspaceTask, SyntheticWorkspaceTasksetConfig]): def load(self) -> Iterable[SyntheticWorkspaceTask]: @@ -720,10 +737,15 @@ def _require_digest_pin(image: str, label: str) -> None: def _runtime_image(artifact: object, label: str) -> str: if not isinstance(artifact, dict): - raise ValueError(f"{label} must declare runtime and container references") + raise ValueError(f"{label} must declare a runtime reference and immutable artifact identity") runtime = str(artifact.get("runtime", "")) container = str(artifact.get("container", "")) - _require_digest_pin(container, f"{label} container") + artifact_id = str(artifact.get("prime_artifact_id", "")) + artifact_path = str(artifact.get("prime_artifact_path", "")) + if container: + _require_digest_pin(container, f"{label} container") + elif not artifact_id or not artifact_path.endswith("/rootfs-cas"): + raise ValueError(f"{label} must declare a Prime artifact ID and CAS path") if not runtime.startswith("prime/") or ":" not in runtime.rpartition("/")[2]: raise ValueError(f"{label} runtime must be a versioned public Prime image reference") return runtime From 41e603bc4e2c20b3aa81815bf2f9989a9dd70cf7 Mon Sep 17 00:00:00 2001 From: ydnysh Date: Fri, 30 Oct 2026 18:13:19 +0530 Subject: [PATCH 3/4] Support Verifiers 0.4 runtime grading --- .../synthetic_workspace_gym/README.md | 6 ++--- .../synthetic_workspace_gym/pyproject.toml | 4 +-- .../synthetic_workspace_gym/taskset.py | 25 +++++++++++++++---- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/environments/synthetic_workspace_gym/README.md b/environments/synthetic_workspace_gym/README.md index 6450498..58c9847 100644 --- a/environments/synthetic_workspace_gym/README.md +++ b/environments/synthetic_workspace_gym/README.md @@ -37,14 +37,14 @@ uv pip install -e environments/synthetic_workspace_gym Run the required small integration smoke with the standard harness/runtime: ```bash -uv run eval --taskset.id synthetic-workspace-gym -n 1 -r 2 --max-turns 4 +uv run vf-eval --env.taskset.id synthetic-workspace-gym -n 1 -r 2 --env.agent.max-turns 4 ``` Use the default `sft-easy-v5` mid-training manifest or select another immutable dataset manifest and filters: ```bash -uv run eval --taskset.id synthetic-workspace-gym \ - --env.taskset.manifest rl-eval-v3 \ +uv run vf-eval --env.taskset.id synthetic-workspace-gym \ + --env.taskset.manifest rl-eval-v5 \ --env.taskset.families '["script_repair","pipeline"]' \ --env.taskset.difficulties '[4,5]' \ -n 20 -r 3 --rich false --no-push diff --git a/environments/synthetic_workspace_gym/pyproject.toml b/environments/synthetic_workspace_gym/pyproject.toml index 57e84c1..88f0c08 100644 --- a/environments/synthetic_workspace_gym/pyproject.toml +++ b/environments/synthetic_workspace_gym/pyproject.toml @@ -1,13 +1,13 @@ [project] name = "synthetic-workspace-gym" -version = "0.2.2" +version = "0.2.3" description = "Native Verifiers V1 workspace tasks for training terminal agents." readme = "README.md" requires-python = ">=3.11" license = "Apache-2.0" authors = [{ name = "Yadnyesh Chakane" }] tags = ["multi-turn", "tool-use", "workspace", "code", "v1"] -dependencies = ["huggingface-hub>=0.28,<2", "verifiers>=0.3.1,<0.4"] +dependencies = ["huggingface-hub>=0.28,<2", "verifiers>=0.4.0,<0.5"] [build-system] requires = ["hatchling"] diff --git a/environments/synthetic_workspace_gym/synthetic_workspace_gym/taskset.py b/environments/synthetic_workspace_gym/synthetic_workspace_gym/taskset.py index d031d36..b297400 100644 --- a/environments/synthetic_workspace_gym/synthetic_workspace_gym/taskset.py +++ b/environments/synthetic_workspace_gym/synthetic_workspace_gym/taskset.py @@ -15,6 +15,7 @@ import verifiers.v1 as vf from huggingface_hub import hf_hub_download from pydantic import Field, field_validator +from verifiers.v1.configs.runtime import NetworkPolicyConfig from verifiers.v1.runtimes import provision_runtime Family = Literal[ @@ -163,7 +164,11 @@ class SyntheticWorkspaceTask( ): NEEDS_CONTAINER = True - async def setup(self, runtime: vf.Runtime) -> None: + async def setup( + self, + trace: vf.Trace[SyntheticWorkspaceData, SyntheticWorkspaceState], + runtime: vf.Runtime, + ) -> None: await _restore_baked_workspace(runtime, self.data.name or "swg-task") async def finalize( @@ -193,12 +198,11 @@ async def finalize( runner_config = _isolated_config(runtime, image=self.data.image or "") async with provision_runtime(runner_config, name=f"swg-runner-{trace.id}") as runner: - await runner.prepare_setup() await _restore_baked_workspace(runner, self.data.name or "swg-task") await _replace_workspace(runner, snapshot) for relative_path, content in execution_files.items(): await runner.write(_safe_path(EXECUTION_ROOT, relative_path), content) - await runner.prepare_execution([]) + await _restrict_runtime(runner) execution, execution_outputs, executed_snapshot = await _execute_plan( runner, execution_plan, @@ -213,7 +217,6 @@ async def finalize( grader_config = _isolated_config(runtime, image=self.data.grader_image) async with provision_runtime(grader_config, name=f"swg-grader-{trace.id}") as grader: - await grader.prepare_setup() mkdir = await grader.run( [ "/bin/mkdir", @@ -240,7 +243,7 @@ async def finalize( await grader.write("/opt/swg-grade/manifest.json", _json_bytes(manifest)) await grader.write("/opt/swg-grade/initial-digests.json", _json_bytes(self.data.initial_digests)) await grader.write("/opt/swg-grade/execution.json", _json_bytes(execution)) - await grader.prepare_execution([]) + await _restrict_runtime(grader) result = await grader.run( [ "/usr/local/bin/python", @@ -686,6 +689,18 @@ def _isolated_config(runtime: vf.Runtime, *, image: str) -> vf.RuntimeConfig: return type(config).model_validate(values) +async def _restrict_runtime(runtime: vf.Runtime) -> None: + policy = NetworkPolicyConfig(allow=[], block=["*"]) + set_network = getattr(runtime, "set_network", None) + if set_network is not None: + await set_network(policy) + return + prepare_execution = getattr(runtime, "prepare_execution", None) + if prepare_execution is None: + raise vf.SandboxError("SWG runner has no supported network policy API") + await prepare_execution([]) + + def _safe_path(root: str, relative_path: str) -> str: pure = PurePosixPath(relative_path.replace("\\", "/")) if pure.is_absolute() or ".." in pure.parts or not pure.parts: From f5a7987c061363d9e9048976031f5fe71e9ee4bf Mon Sep 17 00:00:00 2001 From: ydnysh Date: Sat, 31 Oct 2026 00:35:40 +0530 Subject: [PATCH 4/4] Make strict success the default SWG reward --- environments/synthetic_workspace_gym/README.md | 4 ++-- environments/synthetic_workspace_gym/pyproject.toml | 2 +- .../synthetic_workspace_gym/taskset.py | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/environments/synthetic_workspace_gym/README.md b/environments/synthetic_workspace_gym/README.md index 58c9847..48185b3 100644 --- a/environments/synthetic_workspace_gym/README.md +++ b/environments/synthetic_workspace_gym/README.md @@ -16,7 +16,7 @@ Finalization uses three trust domains: 2. A fresh runtime from the same split image executes candidate code with test inputs but no expected answers. Candidate commands run in a timeout-managed process group; timeout sends `TERM`, escalates to `KILL`, records return code `124`, skips an unavailable determinism rerun, and continues to grading. Concrete observations and profile outputs travel in a host-controlled payload outside `/workspace`; stdout never carries score-bearing data. 3. A second fresh runtime from the dedicated grader image receives those artifacts and the grader-only dataset payload. It computes reward without executing agent-controlled code. -Both temporary runtimes use `vf.provision_runtime`, are network-restricted, and are torn down automatically. The weighted `workspace_score` reward is the evaluator's normalized score in `[0, 1]`; metrics report binary success, changed/final file counts, and evaluator subscores. +Both temporary runtimes use `vf.provision_runtime`, are network-restricted, and are torn down automatically. The sole weighted `workspace_score` reward is binary: `1.0` only when the trusted evaluator reports strict success, otherwise `0.0`. Partial credit cannot affect training reward; the evaluator's normalized score remains available as the `diagnostic_score` metric, alongside success, changed/final file counts, and subscores. The task requests framework-only network access. A fresh runtime is used for every rollout, and generated paths are validated before being written or extracted. @@ -70,4 +70,4 @@ For difficulty calibration, use `eval-d1-d4-paired-panel-48-v5` and `eval-d5-fam | `task.candidate_execution_timeout_seconds` | `120` | Per-run candidate execution limit inside finalization | | `task.candidate_kill_grace_seconds` | `5` | Grace period between candidate process-group termination signals | -The package pins the dataset commit and all seven runtime images in `synthetic_workspace_gym/artifacts.json`. Changing task data, task behavior, or reward semantics requires publishing new immutable artifacts and bumping the package version. +The package pins the dataset commit and all seven runtime images in `synthetic_workspace_gym/artifacts.json`. Changes to task data or image contents require new immutable artifact pins. Reward-only changes require a package version bump; version `0.2.4` uses the same frozen V5 artifacts as `0.2.3` but switches the training reward from partial credit to strict binary success. Results produced by the two package versions are not reward-comparable. diff --git a/environments/synthetic_workspace_gym/pyproject.toml b/environments/synthetic_workspace_gym/pyproject.toml index 88f0c08..7e53221 100644 --- a/environments/synthetic_workspace_gym/pyproject.toml +++ b/environments/synthetic_workspace_gym/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "synthetic-workspace-gym" -version = "0.2.3" +version = "0.2.4" description = "Native Verifiers V1 workspace tasks for training terminal agents." readme = "README.md" requires-python = ">=3.11" diff --git a/environments/synthetic_workspace_gym/synthetic_workspace_gym/taskset.py b/environments/synthetic_workspace_gym/synthetic_workspace_gym/taskset.py index b297400..cb2fe47 100644 --- a/environments/synthetic_workspace_gym/synthetic_workspace_gym/taskset.py +++ b/environments/synthetic_workspace_gym/synthetic_workspace_gym/taskset.py @@ -282,7 +282,7 @@ async def finalize( @vf.reward(weight=1.0) async def workspace_score(self, trace: vf.Trace[SyntheticWorkspaceData, SyntheticWorkspaceState]) -> float: - return float(trace.state.evaluator_result.get("score", 0.0)) + return float(bool(trace.state.evaluator_result.get("success", False))) @vf.metric async def workspace_metrics( @@ -292,6 +292,7 @@ async def workspace_metrics( result = trace.state.evaluator_result metrics = { "success": float(bool(result.get("success", False))), + "diagnostic_score": float(result.get("score", 0.0)), "changed_file_count": float(trace.state.changed_file_count), "final_file_count": float(trace.state.final_file_count), }