| |
| """Dependency-free structural, integrity, and leakage checks for the release.""" |
|
|
| from __future__ import annotations |
|
|
| import hashlib |
| import json |
| import re |
| import sys |
| import urllib.parse |
| from collections import Counter |
| from pathlib import Path |
|
|
|
|
| ROOT = Path(__file__).resolve().parents[1] |
| EXPECTED_IDS = [ |
| "erdos_001", |
| "erdos_003", |
| "erdos_025", |
| "erdos_075", |
| "erdos_149", |
| *[f"aim_ag_{index:03d}" for index in range(1, 11)], |
| "counterexample_114", |
| "counterexample_128", |
| "counterexample_131", |
| "counterexample_134", |
| "counterexample_135", |
| ] |
| EXPECTED_STREAMS = { |
| "erdos_variant": 5, |
| "aim_ag_rl": 10, |
| "counterexample_variant": 5, |
| } |
| EXPECTED_AIM_MILESTONE_COUNTS = [5, 4, 4, 4, 5, 4, 4, 4, 4, 4] |
| EXPECTED_AIM_REFERENCE_COUNTS = [5, 7, 5, 5, 6, 6, 5, 6, 4, 5] |
| EXPECTED_JSONL_COUNTS = { |
| "data/showcase.jsonl": 20, |
| "data/erdos_variants.jsonl": 5, |
| "data/aim_ag_tasks.jsonl": 10, |
| "data/counterexample_variants.jsonl": 5, |
| "rl/data/public_tasks.jsonl": 10, |
| "rl/data/curriculum_episodes.jsonl": 53, |
| "rl/data/curriculum_episodes_hf.jsonl": 53, |
| "rl/data/curriculum_train_hf.jsonl": 18, |
| "rl/data/curriculum_validation_hf.jsonl": 14, |
| "rl/data/curriculum_test_hf.jsonl": 21, |
| "rl/data/exact_benchmark_public.jsonl": 33, |
| "rl/data/exact_benchmark_train.jsonl": 11, |
| "rl/data/exact_benchmark_validation.jsonl": 11, |
| "rl/data/exact_benchmark_test.jsonl": 11, |
| "rl/data/frontier_eval_public.jsonl": 10, |
| } |
| EXPECTED_CONFIG_PATHS = set(EXPECTED_JSONL_COUNTS) - { |
| "rl/data/curriculum_episodes.jsonl", |
| "rl/data/curriculum_episodes_hf.jsonl", |
| "rl/data/exact_benchmark_public.jsonl", |
| } |
|
|
|
|
| def fail(message: str) -> None: |
| raise AssertionError(message) |
|
|
|
|
| def sha256(path: Path) -> str: |
| digest = hashlib.sha256() |
| with path.open("rb") as handle: |
| for block in iter(lambda: handle.read(1024 * 1024), b""): |
| digest.update(block) |
| return digest.hexdigest() |
|
|
|
|
| def read_json(path: Path): |
| with path.open("r", encoding="utf-8") as handle: |
| return json.load(handle) |
|
|
|
|
| def read_jsonl(path: Path) -> list[dict]: |
| rows = [] |
| with path.open("r", encoding="utf-8") as handle: |
| for line_number, line in enumerate(handle, start=1): |
| if not line.strip(): |
| fail(f"Blank JSONL line: {path}:{line_number}") |
| try: |
| value = json.loads(line) |
| except json.JSONDecodeError as exc: |
| fail(f"Invalid JSONL: {path}:{line_number}: {exc}") |
| if not isinstance(value, dict): |
| fail(f"JSONL row is not an object: {path}:{line_number}") |
| rows.append(value) |
| return rows |
|
|
|
|
| def check_showcase(rows: list[dict]) -> None: |
| schema = read_json(ROOT / "schema/showcase.schema.json") |
| required = set(schema["required"]) |
| expected_properties = set(schema["properties"]) |
| ids = [] |
| for row_number, row in enumerate(rows, start=1): |
| missing = required - set(row) |
| extra = set(row) - expected_properties |
| if missing or extra: |
| fail(f"Showcase row {row_number}: missing={sorted(missing)}, extra={sorted(extra)}") |
| if row["schema_version"] != "1.0.0" or row["license"] != "MIT": |
| fail(f"Showcase row {row_number}: schema/license mismatch") |
| if row["stream"] not in EXPECTED_STREAMS: |
| fail(f"Showcase row {row_number}: unknown stream") |
| for field in [ |
| "problem_id", |
| "title", |
| "domain", |
| "task_type", |
| "difficulty", |
| "prompt", |
| "inspiration", |
| "rationale", |
| "expected_output", |
| "research_status", |
| "verification", |
| ]: |
| if not isinstance(row[field], str) or not row[field].strip(): |
| fail(f"Showcase row {row_number}: invalid {field}") |
| if not isinstance(row["source_record_index"], int) or row["source_record_index"] < 1: |
| fail(f"Showcase row {row_number}: invalid source_record_index") |
| if not isinstance(row["rl_ready"], bool): |
| fail(f"Showcase row {row_number}: rl_ready must be boolean") |
| if not isinstance(row["milestones"], list): |
| fail(f"Showcase row {row_number}: milestones must be a list") |
| if not isinstance(row["quality_signals"], list) or not row["quality_signals"]: |
| fail(f"Showcase row {row_number}: quality_signals must be nonempty") |
| if row["rl_ready"] != (row["stream"] == "aim_ag_rl"): |
| fail(f"Showcase row {row_number}: rl_ready conflicts with stream") |
| ids.append(row["problem_id"]) |
| if ids != EXPECTED_IDS: |
| fail(f"Unexpected showcase IDs/order: {ids}") |
| if len(ids) != len(set(ids)): |
| fail("Duplicate showcase problem IDs") |
| if Counter(row["stream"] for row in rows) != Counter(EXPECTED_STREAMS): |
| fail("Unexpected stream counts") |
|
|
|
|
| def check_aim_tasks(rows: list[dict]) -> None: |
| expected_ids = [f"aim_ag_{index:03d}" for index in range(1, 11)] |
| actual_ids = [row.get("release_problem_id") for row in rows] |
| if actual_ids != expected_ids: |
| fail(f"Unexpected AIM-AG IDs: {actual_ids}") |
| for source_index, row in enumerate(rows, start=1): |
| source_id = f"sample_{source_index}" |
| if row.get("problem_id") != source_id: |
| fail(f"Unexpected AIM-AG source ID: {row.get('problem_id')}") |
| research_status = row.get("research_status", {}) |
| if research_status.get("classification") != "candidate_open_problem": |
| fail(f"Unexpected AIM-AG research status: {source_id}") |
| if research_status.get("expert_signoff_required") is not True: |
| fail(f"AIM-AG expert-signoff flag is missing: {source_id}") |
| milestones = row.get("evaluation", {}).get("milestones", []) |
| if len(milestones) != EXPECTED_AIM_MILESTONE_COUNTS[source_index - 1]: |
| fail(f"Unexpected AIM-AG milestone count: {source_id}") |
| expected_milestones = [f"m{index}" for index in range(1, len(milestones) + 1)] |
| if [item.get("milestone_id") for item in milestones] != expected_milestones: |
| fail(f"AIM-AG milestones are not sequential: {source_id}") |
| if sum(item.get("weight_percent", 0) for item in milestones) != 100: |
| fail(f"AIM-AG milestone weights do not sum to 100: {row['release_problem_id']}") |
| references = row.get("research_context", {}).get("references", []) |
| if len(references) != EXPECTED_AIM_REFERENCE_COUNTS[source_index - 1]: |
| fail(f"Unexpected AIM-AG reference count: {source_id}") |
| for reference in references: |
| parsed = urllib.parse.urlparse(reference.get("url", "")) |
| if parsed.scheme != "https" or not parsed.netloc: |
| fail(f"Invalid AIM-AG reference URL: {source_id}") |
| if row.get("license") != "MIT": |
| fail(f"AIM-AG license missing: {row['release_problem_id']}") |
|
|
|
|
| def check_lossless_episode_view(source_rows: list[dict], normalized_rows: list[dict]) -> None: |
| if [row.get("episode_id") for row in normalized_rows] != [ |
| row.get("episode_id") for row in source_rows |
| ]: |
| fail("Type-stable curriculum IDs/order do not match production-format public rows") |
| direct_fields = [ |
| "schema_version", |
| "episode_id", |
| "episode_type", |
| "problem_id", |
| "split", |
| "reward_mode", |
| "policy_visibility", |
| "submission_schema_ref", |
| ] |
| optional_defaults = { |
| "milestone_id": "", |
| "target_milestone_ids": [], |
| "milestone_target": "", |
| "required_artifact_policy": "", |
| "verifier_id": "", |
| "candidate_required_fields": [], |
| } |
| for source, normalized in zip(source_rows, normalized_rows): |
| episode_id = source["episode_id"] |
| if normalized.get("source_fields") != sorted(source): |
| fail(f"Lossless-view source field inventory mismatch: {episode_id}") |
| prompt = source["prompt"] |
| expected_kind = "object" if isinstance(prompt, dict) else "string" |
| if normalized.get("prompt_kind") != expected_kind: |
| fail(f"Lossless-view prompt kind mismatch: {episode_id}") |
| if json.loads(normalized.get("prompt_json", "null")) != prompt: |
| fail(f"Lossless-view prompt JSON mismatch: {episode_id}") |
| expected_prompt_text = ( |
| json.dumps(prompt, ensure_ascii=False, sort_keys=True) |
| if isinstance(prompt, dict) |
| else str(prompt) |
| ) |
| if normalized.get("prompt_text") != expected_prompt_text: |
| fail(f"Lossless-view prompt text mismatch: {episode_id}") |
| expected_input = ( |
| json.dumps(source["input"], ensure_ascii=False, sort_keys=True) |
| if "input" in source |
| else "" |
| ) |
| if normalized.get("input_json") != expected_input: |
| fail(f"Lossless-view input mismatch: {episode_id}") |
| for field in direct_fields: |
| if normalized.get(field) != source.get(field): |
| fail(f"Lossless-view {field} mismatch: {episode_id}") |
| for field, default in optional_defaults.items(): |
| if normalized.get(field) != source.get(field, default): |
| fail(f"Lossless-view {field} mismatch: {episode_id}") |
|
|
|
|
| def check_rl_public(aim_rows: list[dict]) -> None: |
| public_paths = [ |
| "rl/data/public_tasks.jsonl", |
| "rl/data/curriculum_episodes.jsonl", |
| "rl/data/curriculum_episodes_hf.jsonl", |
| "rl/data/curriculum_train_hf.jsonl", |
| "rl/data/curriculum_validation_hf.jsonl", |
| "rl/data/curriculum_test_hf.jsonl", |
| "rl/data/exact_benchmark_public.jsonl", |
| "rl/data/exact_benchmark_train.jsonl", |
| "rl/data/exact_benchmark_validation.jsonl", |
| "rl/data/exact_benchmark_test.jsonl", |
| "rl/data/frontier_eval_public.jsonl", |
| ] |
| valid_problem_ids = {f"sample_{index}" for index in range(1, 11)} |
| for relative in public_paths: |
| for row in read_jsonl(ROOT / relative): |
| if row.get("policy_visibility") != "public": |
| fail(f"Non-public RL row found in {relative}") |
| if row.get("problem_id") not in valid_problem_ids: |
| fail(f"Unknown RL problem ID in {relative}: {row.get('problem_id')}") |
| schema_ref = row.get("submission_schema_ref") |
| if schema_ref and not (ROOT / "rl" / schema_ref).is_file(): |
| fail(f"Broken RL schema reference in {relative}: {schema_ref}") |
|
|
| tasks = read_jsonl(ROOT / "rl/data/public_tasks.jsonl") |
| if [row["problem_id"] for row in tasks] != [f"sample_{index}" for index in range(1, 11)]: |
| fail("Public RL tasks are not ordered sample_1 through sample_10") |
| aim_by_source_id = {row["problem_id"]: row for row in aim_rows} |
| task_by_id = {row["problem_id"]: row for row in tasks} |
| for problem_id, task in task_by_id.items(): |
| aim = aim_by_source_id[problem_id] |
| if task.get("title") != aim.get("title"): |
| fail(f"AIM-AG/RL title mismatch: {problem_id}") |
| for field in ["conjecture", "definitions"]: |
| if task.get("prompt", {}).get(field) != aim.get("prompt", {}).get(field): |
| fail(f"AIM-AG/RL core prompt mismatch ({field}): {problem_id}") |
|
|
| episodes = read_jsonl(ROOT / "rl/data/curriculum_episodes.jsonl") |
| episode_ids = [row["episode_id"] for row in episodes] |
| if len(episode_ids) != len(set(episode_ids)): |
| fail("Duplicate public curriculum episode IDs") |
| for row in episodes: |
| if isinstance(row.get("prompt"), dict): |
| task_prompt = task_by_id[row["problem_id"]]["prompt"] |
| for field in ["conjecture", "definitions"]: |
| if row["prompt"].get(field) != task_prompt.get(field): |
| fail(f"Structured episode/RL task mismatch ({field}): {row['episode_id']}") |
|
|
| exact = read_jsonl(ROOT / "rl/data/exact_benchmark_public.jsonl") |
| frontier = read_jsonl(ROOT / "rl/data/frontier_eval_public.jsonl") |
| if exact != [row for row in episodes if row["episode_type"] == "exact_benchmark"]: |
| fail("Exact benchmark file is not the exact-episode subset of the curriculum") |
| if frontier != [row for row in episodes if row["episode_type"] == "full_frontier_task"]: |
| fail("Frontier eval file is not the full-frontier subset of the curriculum") |
|
|
| normalized = read_jsonl(ROOT / "rl/data/curriculum_episodes_hf.jsonl") |
| check_lossless_episode_view(episodes, normalized) |
| split_names = {"train": "train", "dev": "validation", "eval": "test"} |
| for source_split, hf_split in split_names.items(): |
| if read_jsonl(ROOT / f"rl/data/curriculum_{hf_split}_hf.jsonl") != [ |
| row for row in normalized if row["split"] == source_split |
| ]: |
| fail(f"Curriculum {hf_split} split is inconsistent") |
| if read_jsonl(ROOT / f"rl/data/exact_benchmark_{hf_split}.jsonl") != [ |
| row for row in exact if row["split"] == source_split |
| ]: |
| fail(f"Exact benchmark {hf_split} split is inconsistent") |
|
|
| fixtures = { |
| path.stem: read_json(path) for path in (ROOT / "rl/fixtures/public").glob("*.json") |
| } |
| exact_by_id = {row["episode_id"]: row for row in exact} |
| if fixtures != exact_by_id: |
| fail("Public fixture files do not match public exact episodes one-to-one") |
|
|
| curriculum_config = read_json(ROOT / "rl/configs/training_curriculum.json") |
| public_scope = curriculum_config.get("public_release_scope", {}) |
| if public_scope.get("all_included_prompts_and_fixtures_are_public") is not True: |
| fail("Training curriculum lacks the public-release fixture override") |
| if any("held-out" in stage.get("name", "").lower() for stage in curriculum_config["stages"]): |
| fail("Training curriculum still labels a public stage as held out") |
| leaked_paths = [ |
| path.relative_to(ROOT).as_posix() |
| for path in (ROOT / "rl").rglob("*") |
| if path.is_file() and "hidden" in path.relative_to(ROOT).as_posix().lower() |
| ] |
| if leaked_paths: |
| fail(f"Hidden-path assets included: {leaked_paths}") |
|
|
|
|
| def check_version_consistency() -> None: |
| version = (ROOT / "VERSION").read_text(encoding="utf-8").strip() |
| manifest = read_json(ROOT / "MANIFEST.json") |
| citation = (ROOT / "CITATION.cff").read_text(encoding="utf-8") |
| readme = (ROOT / "README.md").read_text(encoding="utf-8") |
| if version != "0.1.0": |
| fail(f"Unexpected release version: {version}") |
| if manifest.get("release_version") != version: |
| fail("VERSION and manifest release_version differ") |
| if not re.search(rf"^version:\s*{re.escape(version)}\s*$", citation, re.MULTILINE): |
| fail("VERSION and CITATION.cff version differ") |
| if f"version = {{{version}}}" not in readme: |
| fail("VERSION and README BibTeX version differ") |
|
|
|
|
| def check_readme_configs() -> None: |
| readme = (ROOT / "README.md").read_text(encoding="utf-8") |
| if not readme.startswith("---\n") or "\nlicense: mit\n" not in readme: |
| fail("README metadata is missing YAML front matter or MIT license") |
| front_matter = readme.split("---", 2)[1] |
| config_paths = re.findall(r"^\s+path:\s+([^\s]+)\s*$", front_matter, flags=re.MULTILINE) |
| if set(config_paths) != EXPECTED_CONFIG_PATHS: |
| fail( |
| f"README config paths mismatch: missing={sorted(EXPECTED_CONFIG_PATHS-set(config_paths))}, " |
| f"extra={sorted(set(config_paths)-EXPECTED_CONFIG_PATHS)}" |
| ) |
| for relative in config_paths: |
| if not (ROOT / relative).is_file(): |
| fail(f"README config points to missing file: {relative}") |
|
|
|
|
| def check_sensitive_strings() -> None: |
| patterns = { |
| "absolute local path": re.compile("/" + "Users/" + "black" + "frog/"), |
| "Hugging Face token": re.compile(r"hf_[A-Za-z0-9]{20,}"), |
| "generic API secret": re.compile(r"sk-[A-Za-z0-9_-]{20,}"), |
| } |
| for path in ROOT.rglob("*"): |
| if not path.is_file() or path.name == ".DS_Store": |
| continue |
| try: |
| text = path.read_text(encoding="utf-8") |
| except UnicodeDecodeError: |
| continue |
| for label, pattern in patterns.items(): |
| if pattern.search(text): |
| fail(f"Possible {label} in {path.relative_to(ROOT)}") |
|
|
|
|
| def check_manifest() -> None: |
| manifest = read_json(ROOT / "MANIFEST.json") |
| if manifest.get("publication_status") != "published_public": |
| fail("Manifest publication status is not published_public") |
| listed = manifest.get("files", []) |
| for item in listed: |
| path = ROOT / item["path"] |
| if not path.is_file(): |
| fail(f"Manifest file is missing: {item['path']}") |
| if path.stat().st_size != item["bytes"] or sha256(path) != item["sha256"]: |
| fail(f"Manifest integrity mismatch: {item['path']}") |
| actual_paths = { |
| path.relative_to(ROOT).as_posix() |
| for path in ROOT.rglob("*") |
| if path.is_file() and path.name not in {".DS_Store", "MANIFEST.json"} |
| } |
| listed_paths = {item["path"] for item in listed} |
| if actual_paths != listed_paths: |
| fail( |
| f"Manifest file set mismatch: missing={sorted(actual_paths-listed_paths)}, " |
| f"stale={sorted(listed_paths-actual_paths)}" |
| ) |
| content_digest = hashlib.sha256() |
| for item in listed: |
| content_digest.update(f"{item['path']}\0{item['sha256']}\n".encode("utf-8")) |
| if content_digest.hexdigest() != manifest.get("content_set_sha256"): |
| fail("Manifest content-set hash mismatch") |
|
|
|
|
| def main() -> int: |
| for relative, expected_count in EXPECTED_JSONL_COUNTS.items(): |
| rows = read_jsonl(ROOT / relative) |
| if len(rows) != expected_count: |
| fail(f"{relative}: expected {expected_count} rows, found {len(rows)}") |
| showcase = read_jsonl(ROOT / "data/showcase.jsonl") |
| check_showcase(showcase) |
| aim_rows = read_jsonl(ROOT / "data/aim_ag_tasks.jsonl") |
| check_aim_tasks(aim_rows) |
| check_rl_public(aim_rows) |
| check_readme_configs() |
| check_sensitive_strings() |
| check_version_consistency() |
| check_manifest() |
| print("PASS: release structure, counts, IDs, public visibility, metadata, versions, and hashes") |
| print("PASS: 20 showcase problems = 5 Erdős + 10 AIM-AG + 5 counterexample variants") |
| print("PASS: AIM-AG/RL mathematical cores align; type-stable episodes round-trip losslessly") |
| print("PASS: curriculum/exact/frontier subsets, native splits, and 33 fixtures align") |
| print("PASS: no hidden-path files or obvious local paths/API tokens") |
| return 0 |
|
|
|
|
| if __name__ == "__main__": |
| try: |
| sys.exit(main()) |
| except (AssertionError, FileNotFoundError, json.JSONDecodeError) as exc: |
| print(f"FAIL: {exc}", file=sys.stderr) |
| sys.exit(1) |
|
|
