{"name":"EnvIndex artifact catalog","asOf":"2026-08-16","methodology":"https://envindex.com/methodology","semantics":{"unknown":"No reliable evidence recorded; not equivalent to No","private":"Access status only; not a verification judgment"},"artifacts":[{"slug":"surge-rl-environments-and-agents","name":"RL Environments and Agents","ownerSlug":"surge","artifactType":"RL Environment","availabilityKind":"custom-capability","access":["Custom-built","Commercial","Request demo"],"summary":"Custom RL environments and verifier design for training and evaluating agentic models.","description":"Surge describes a capability for creating complex RL environments that challenge agentic models, together with verifiers that reward their behavior. This capability record does not imply a fixed public task count or a single packaged environment.","domains":["enterprise","multi-domain","long-horizon","tool-use"],"capabilities":{"tool-use":"yes","long-horizon":"yes","expert-authored":"yes","stateful":"unknown","multi-agent":"unknown","sandboxed":"unknown"},"verifierType":"Custom verifier design","lastVerified":"2026-08-16","verificationStatus":"company-reported","confidence":"high","links":[{"label":"Product page","url":"https://surgehq.ai/products","kind":"official"}],"sourceIds":["surge-products","surge-corecraft"],"relatedArtifactSlugs":["surge-corecraft","surge-handbook-agents"]},{"slug":"surge-rubrics-and-verifiers","name":"Rubrics and Verifiers","ownerSlug":"surge","artifactType":"Verifier","availabilityKind":"custom-capability","access":["Custom-built","Commercial","Request demo"],"summary":"Custom rubric and verifier design for scoring complex model and agent behavior.","description":"Surge describes designing grading systems that capture both successful behavior and deficiencies for outputs that cannot be evaluated with simple automatic checks.","domains":["alignment","multi-domain"],"capabilities":{"expert-authored":"yes","human-in-the-loop":"yes","tool-use":"unknown"},"verifierType":"Expert-designed rubrics and verifiers","lastVerified":"2026-08-16","verificationStatus":"company-reported","confidence":"high","links":[{"label":"Product page","url":"https://surgehq.ai/products","kind":"official"}],"sourceIds":["surge-products"]},{"slug":"surge-rlhf","name":"RLHF","ownerSlug":"surge","artifactType":"Dataset","availabilityKind":"custom-capability","access":["Custom-built","Commercial","Request demo"],"summary":"Human preference and reward data for reinforcement learning from human feedback.","description":"Surge reports generating preference and reward data intended to capture nuanced judgments for model alignment and post-training.","domains":["rlhf","alignment","multi-domain"],"capabilities":{"human-in-the-loop":"yes","expert-authored":"yes","multi-turn":"unknown"},"verifierType":"Human preference and reward signals","lastVerified":"2026-08-16","verificationStatus":"company-reported","confidence":"high","links":[{"label":"Product page","url":"https://surgehq.ai/products","kind":"official"}],"sourceIds":["surge-products"]},{"slug":"surge-sft","name":"Supervised Fine-Tuning Data","ownerSlug":"surge","artifactType":"Dataset","availabilityKind":"custom-capability","access":["Custom-built","Commercial","Request demo"],"summary":"Expert demonstrations for bootstrapping model capabilities, including computer and browser use.","description":"Surge describes supervised fine-tuning demonstrations that teach models foundational skills such as computer use, web navigation, and reasoning.","domains":["computer-use","browser","multi-domain"],"capabilities":{"computer-use":"yes","browser-use":"yes","expert-authored":"yes","human-in-the-loop":"yes"},"verifierType":"Unknown","lastVerified":"2026-08-16","verificationStatus":"company-reported","confidence":"high","links":[{"label":"Product page","url":"https://surgehq.ai/products","kind":"official"}],"sourceIds":["surge-products"]},{"slug":"surge-human-evaluation","name":"Human Evaluation","ownerSlug":"surge","artifactType":"Evaluation Suite","availabilityKind":"custom-capability","access":["Custom-built","Commercial","Request demo"],"summary":"Human evaluation programs for model quality, usefulness, safety, and subjective output characteristics.","description":"Surge describes human evaluation as a managed capability for assessing qualities that automatic evaluations and academic benchmarks may not capture reliably.","domains":["alignment","multi-domain"],"capabilities":{"human-in-the-loop":"yes","expert-authored":"yes"},"verifierType":"Human evaluation","lastVerified":"2026-08-16","verificationStatus":"company-reported","confidence":"high","links":[{"label":"Product page","url":"https://surgehq.ai/products","kind":"official"}],"sourceIds":["surge-products"]},{"slug":"surge-expert-professional-domains","name":"Expert Professional Domains","ownerSlug":"surge","artifactType":"Dataset","availabilityKind":"custom-capability","access":["Custom-built","Commercial","Request demo"],"summary":"Expert-authored data and judgment across professional, STEM, and humanities domains.","description":"Surge describes recruiting professionals and academics, including doctors, lawyers, investment bankers, mathematicians, and professors, to shape training and evaluation data.","domains":["finance","legal","medical","science","math","multi-domain"],"capabilities":{"expert-authored":"yes","human-in-the-loop":"yes"},"verifierType":"Varies by engagement","lastVerified":"2026-08-16","verificationStatus":"company-reported","confidence":"high","links":[{"label":"Product page","url":"https://surgehq.ai/products","kind":"official"}],"sourceIds":["surge-products"]},{"slug":"surge-internationalization","name":"Internationalization","ownerSlug":"surge","artifactType":"Dataset","availabilityKind":"custom-capability","access":["Custom-built","Commercial","Request demo"],"summary":"Language and culturally grounded training data across more than 70 reported languages.","description":"Surge reports internationalization work across more than 70 languages, with linguists designing data for grammar, idiom, and cultural context.","domains":["multi-domain"],"capabilities":{"expert-authored":"yes","human-in-the-loop":"yes"},"verifierType":"Expert linguistic review","lastVerified":"2026-08-16","verificationStatus":"company-reported","confidence":"high","links":[{"label":"Product page","url":"https://surgehq.ai/products","kind":"official"}],"sourceIds":["surge-products"]},{"slug":"surge-multimodal-data","name":"Multimodal Data","ownerSlug":"surge","artifactType":"Dataset","availabilityKind":"custom-capability","access":["Custom-built","Commercial","Request demo"],"summary":"Training and evaluation data spanning text, images, audio, and video.","description":"Surge describes multimodal data capabilities for model understanding and generation across images, audio, and video in addition to text.","domains":["multi-domain"],"capabilities":{"multimodal":"yes","human-in-the-loop":"yes","expert-authored":"unknown"},"verifierType":"Varies by engagement","lastVerified":"2026-08-16","verificationStatus":"company-reported","confidence":"high","links":[{"label":"Product page","url":"https://surgehq.ai/products","kind":"official"}],"sourceIds":["surge-products"]},{"slug":"surge-off-the-shelf-data","name":"Off-The-Shelf Data Catalog","ownerSlug":"surge","artifactType":"Environment Bundle","availabilityKind":"private-commercial","access":["Private catalog","Commercial","Off-the-shelf"],"summary":"Pre-built commercial datasets and RL environments spanning coding, enterprise agents, STEM, tool use, and reasoning.","description":"Surge describes a ready-to-use private commercial catalog containing expert-built training datasets, evaluation sets, and RL environments. Public pages expose selected categories and reported model results but not the full underlying inventory.","domains":["coding","enterprise","science","math","tool-use","multi-domain","long-horizon"],"capabilities":{"long-horizon":"yes","tool-use":"yes","expert-authored":"yes","sandboxed":"unknown","code-execution":"yes","file-manipulation":"yes"},"verifierType":"Includes hidden verifiers and benchmark-compatible grading; varies by catalog item","lastVerified":"2026-08-16","verificationStatus":"company-reported","confidence":"high","links":[{"label":"Request full catalog","url":"https://surgehq.ai/ots","kind":"access"}],"sourceIds":["surge-products","surge-off-the-shelf"],"relatedArtifactSlugs":["surge-rl-environments-and-agents"]},{"slug":"surge-corecraft","name":"CoreCraft","ownerSlug":"surge","artifactType":"Benchmark","availabilityKind":"public-sample","access":["Public / Open"],"summary":"A simulated enterprise environment where agents complete customer-support and operational tasks inside a fictional PC retailer.","description":"CoreCraft models a company with customers, orders, support tickets, records, and tools. Surge reports evaluating nine frontier models on 150 workplace tasks ranging from lookups to multi-step operational workflows.","domains":["enterprise","tool-use","long-horizon"],"capabilities":{"tool-use":"yes","long-horizon":"yes","multi-turn":"unknown","stateful":"yes","expert-authored":"yes","real-application-replica":"no"},"taskCount":150,"verifierType":"Task-specific environment verification; full implementation details Unknown","firstReleased":"2025-11-03","lastVerified":"2026-08-16","verificationStatus":"company-reported","confidence":"high","links":[{"label":"Research article","url":"https://surgehq.ai/blog/rl-envs-real-world","kind":"official"},{"label":"Benchmark page","url":"https://surgehq.ai/benchmarks","kind":"leaderboard"}],"sourceIds":["surge-corecraft","surge-benchmarks"],"relatedArtifactSlugs":["surge-rl-environments-and-agents","surge-handbook-agents"]},{"slug":"surge-handbook-agents","name":"HANDBOOK.md Agents","ownerSlug":"surge","artifactType":"Benchmark","availabilityKind":"public-sample","access":["Public / Open"],"summary":"A long-context enterprise-agent benchmark built from unique RL environments with internal tools and external MCP servers.","description":"HANDBOOK.md evaluates whether agents can follow a long company handbook while completing day-to-day professional tasks. Surge reports that each task is a unique RL environment across five enterprise domains.","domains":["enterprise","tool-use","long-horizon"],"capabilities":{"tool-use":"yes","mcp-support":"yes","long-horizon":"yes","expert-authored":"unknown","stateful":"unknown"},"verifierType":"Unknown","lastVerified":"2026-08-16","verificationStatus":"company-reported","confidence":"high","links":[{"label":"Benchmark page","url":"https://surgehq.ai/benchmarks","kind":"leaderboard"}],"sourceIds":["surge-benchmarks"],"relatedArtifactSlugs":["surge-corecraft","surge-rl-environments-and-agents"]},{"slug":"rise-data-labs-long-horizon-knowledge-work-tasks","name":"Long-Horizon Knowledge-Work Tasks","ownerSlug":"rise-data-labs","artifactType":"Environment Bundle","availabilityKind":"private-commercial","access":["Private catalog","Commercial"],"summary":"Private commercial long-horizon tasks spanning multiple knowledge-work domains.","description":"Rise Data Labs reports private commercial inventory of long-horizon tasks across knowledge-work domains. EnvIndex has not inspected the proprietary task files, task count, tool integrations, verifier implementations, pricing, or model results.","domains":["enterprise","finance","legal","science","multi-domain","long-horizon"],"capabilities":{"long-horizon":"yes","expert-authored":"unknown","tool-use":"unknown","file-manipulation":"unknown","cross-application":"unknown","stateful":"unknown"},"verifierType":"Unknown","lastVerified":"2026-08-16","verificationStatus":"vendor-submitted","confidence":"medium","links":[{"label":"Request information","url":"https://risedatalabs.com/solutions/rl-environment","kind":"access"}],"sourceIds":["rise-data-labs-long-horizon-tasks-submission"],"relatedArtifactSlugs":["rise-data-labs-finance-tasks","rise-data-labs-custom-enterprise-environments"],"editorialNote":"Private describes access, not verification. Long-horizon support is vendor-submitted; all other technical details remain Unknown pending evidence review."},{"slug":"rise-data-labs-finance-tasks","name":"Finance Tasks","ownerSlug":"rise-data-labs","artifactType":"Environment Bundle","availabilityKind":"private-commercial","access":["Private catalog","Commercial"],"summary":"A private commercial catalog of finance tasks for RL training and evaluation.","description":"Rise Data Labs reports that finance tasks are available as private commercial inventory. EnvIndex records the catalog metadata for discovery, but has not inspected the proprietary task files, counts, verifier implementations, pricing, or model results.","domains":["finance","enterprise"],"capabilities":{"expert-authored":"unknown","long-horizon":"unknown","tool-use":"unknown","file-manipulation":"unknown","production-derived":"unknown"},"verifierType":"Unknown","lastVerified":"2026-08-16","verificationStatus":"vendor-submitted","confidence":"medium","links":[{"label":"Request information","url":"https://risedatalabs.com/solutions/rl-environment","kind":"access"}],"sourceIds":["rise-data-labs-finance-tasks-submission"],"relatedArtifactSlugs":["rise-data-labs-custom-enterprise-environments"],"editorialNote":"Private describes access, not verification. Catalog existence is vendor-submitted; technical details remain Unknown pending evidence review."},{"slug":"rise-data-labs-custom-enterprise-environments","name":"Custom Enterprise Environments","ownerSlug":"rise-data-labs","artifactType":"RL Environment","availabilityKind":"custom-capability","access":["Custom-built","Commercial","Request demo"],"summary":"A custom environment-development capability based on a buyer’s workflows, tools, data, and success criteria.","description":"Rise Data Labs describes a service for constructing custom RL environments from first-party enterprise workflows, with reward engineering, multi-agent simulations, and adversarial testing. This is a documented custom capability, not evidence of an off-the-shelf private catalog or a publicly runnable environment.","domains":["enterprise","finance","legal","cybersecurity","multi-domain"],"capabilities":{"stateful":"unknown","long-horizon":"unknown","multi-agent":"yes","adversarial":"yes","tool-use":"yes","human-in-the-loop":"yes","production-derived":"yes","cross-application":"unknown"},"verifierType":"Custom outcome-aligned reward functions; implementation details vary by engagement","lastVerified":"2026-08-16","verificationStatus":"company-reported","confidence":"medium","links":[{"label":"Capability page","url":"https://risedatalabs.com/solutions/rl-environment","kind":"official"}],"sourceIds":["rise-data-labs-rl-environments"],"editorialNote":"Custom capability does not imply existing packaged inventory. No public benchmark, SDK/API, task count, model uplift result, or off-the-shelf catalog was verified."},{"slug":"gba-eval","name":"GBA Eval","ownerSlug":"mechanize","artifactType":"Benchmark","availabilityKind":"public-sample","access":["Public / Open"],"summary":"A long-horizon coding eval in which agents build a Game Boy Advance emulator in Rust within 24 hours.","description":"GBA Eval measures whether a coding agent can implement a Game Boy Advance emulator from scratch and compile it to WebAssembly. Mechanize describes it as one carefully graded task rather than a general-purpose software-engineering benchmark.","domains":["coding","long-horizon"],"capabilities":{"long-horizon":"yes","code-execution":"yes","sandboxed":"unknown","deterministic-reset":"unknown"},"taskCount":1,"verifierType":"Programmatic tests plus deterministic gameplay replay","firstReleased":"2026-05-02","lastVerified":"2026-08-16","verificationStatus":"company-reported","confidence":"high","links":[{"label":"Technical description","url":"https://www.mechanize.work/technical-blog/introducing-gba-eval/","kind":"official"},{"label":"Benchmark and leaderboard","url":"https://gbaeval.com","kind":"leaderboard"}],"sourceIds":["mechanize-gba"],"editorialNote":"Mechanize explicitly cautions that this single task is not a complete measure of general coding capability."},{"slug":"swe-bench-pro","name":"SWE-Bench Pro","ownerSlug":"scale","artifactType":"Benchmark","availabilityKind":"public-sample","access":["Public / Open","Open source"],"summary":"A long-horizon software-engineering benchmark built from public and proprietary repositories.","description":"SWE-Bench Pro evaluates agents on realistic software-engineering problems. The published paper reports 1,865 problems from 41 actively maintained repositories; the public dataset and leaderboard are separately accessible.","domains":["coding","long-horizon"],"capabilities":{"long-horizon":"yes","code-execution":"yes","sandboxed":"yes","human-in-the-loop":"yes","expert-authored":"yes"},"taskCount":1865,"verifierType":"Repository test suites with human verification checkpoints","firstReleased":"2025-09-19","lastVerified":"2026-08-16","verificationStatus":"confirmed","confidence":"high","links":[{"label":"Research page","url":"https://labs.scale.com/papers/swe_bench_pro","kind":"paper"},{"label":"Public leaderboard","url":"https://labs.scale.com/leaderboard/swe_bench_pro_public","kind":"leaderboard"}],"sourceIds":["scale-swe-pro","scale-swe-public"],"relatedArtifactSlugs":["swe-bench-pro-private"]},{"slug":"swe-bench-pro-private","name":"SWE-Bench Pro  -  Private Dataset","ownerSlug":"scale","artifactType":"Benchmark","availabilityKind":"private-commercial","access":["Private catalog","Enterprise-only"],"summary":"The proprietary-repository subset of SWE-Bench Pro, documented publicly but not distributed as an open dataset.","description":"This private subset evaluates coding agents against commercial-grade proprietary repositories. Its existence and leaderboard are public; the underlying repository contents are not presented as publicly downloadable by EnvIndex.","domains":["coding","long-horizon","enterprise"],"capabilities":{"long-horizon":"yes","code-execution":"yes","sandboxed":"yes","human-in-the-loop":"yes","production-derived":"yes"},"verifierType":"Repository test suites","lastVerified":"2026-08-16","verificationStatus":"company-reported","confidence":"medium","links":[{"label":"Private dataset leaderboard","url":"https://labs.scale.com/leaderboard/swe_bench_pro_private","kind":"leaderboard"}],"sourceIds":["scale-swe-pro","scale-swe-private"],"relatedArtifactSlugs":["swe-bench-pro"],"editorialNote":"Private describes access, not verification. EnvIndex has not inspected the proprietary repository contents."},{"slug":"gandalf-the-grader","name":"Gandalf the Grader","ownerSlug":"handshake","artifactType":"Verifier","availabilityKind":"public-sample","access":["Public / Open","Open source"],"summary":"An open-source reactive agent-as-judge that inspects files and environment state while grading agent work.","description":"Gandalf runs inside a rollout environment and chooses what evidence to inspect using available tools. Handshake reports results on BankerVerifierBench and an internal productivity benchmark; those performance claims remain company-reported unless independently reproduced.","domains":["finance","enterprise","tool-use"],"capabilities":{"tool-use":"yes","stateful":"yes","file-manipulation":"yes","human-in-the-loop":"yes"},"verifierType":"Reactive agent-as-judge","firstReleased":"2026-05-27","lastVerified":"2026-08-16","verificationStatus":"confirmed","confidence":"high","links":[{"label":"Technical article","url":"https://joinhandshake.com/research/ai/gandalf-the-grader/","kind":"official"},{"label":"Source code","url":"https://github.com/Handshake-AI-Research/gandalf-the-grader","kind":"github"}],"sourceIds":["handshake-gandalf","handshake-gandalf-github"],"relatedArtifactSlugs":["bankertoolbench"]},{"slug":"bankertoolbench","name":"BankerToolBench","ownerSlug":"handshake","artifactType":"Benchmark","availabilityKind":"public-sample","access":["Public / Open"],"summary":"A banking-workflow benchmark used to evaluate agents and verifier behavior on artifact-heavy tasks.","description":"BankerToolBench contains banking-domain tasks whose outputs can include spreadsheets and other work products. Handshake uses a 21-task slice in its reported BankerVerifierBench meta-evaluation of grading systems.","domains":["finance","enterprise","tool-use"],"capabilities":{"tool-use":"yes","file-manipulation":"yes","expert-authored":"yes","long-horizon":"unknown"},"scenarioCount":21,"verifierType":"Rubric grading; compatible with agent-as-judge verification","lastVerified":"2026-08-16","verificationStatus":"confirmed","confidence":"medium","links":[{"label":"Dataset","url":"https://huggingface.co/datasets/handshake-ai-research/bankertoolbench","kind":"hugging-face"}],"sourceIds":["handshake-banker-dataset","handshake-gandalf"],"relatedArtifactSlugs":["gandalf-the-grader"]},{"slug":"app-bench","name":"App-Bench","ownerSlug":"afterquery","artifactType":"Benchmark","availabilityKind":"public-sample","access":["Public / Open"],"summary":"A benchmark for evaluating AI systems that generate modern web applications.","description":"App-Bench evaluates coding agents on building web applications. EnvIndex currently records only the fields supported by AfterQuery’s public benchmark page; task counts and technical access mechanisms remain unknown here.","domains":["coding","design"],"capabilities":{"code-execution":"yes","multimodal":"unknown","sandboxed":"unknown"},"verifierType":"Unknown","lastVerified":"2026-08-16","verificationStatus":"company-reported","confidence":"medium","links":[{"label":"Benchmark page","url":"https://www.afterquery.com/leaderboard/app-bench","kind":"leaderboard"}],"sourceIds":["afterquery-appbench"]},{"slug":"apex","name":"APEX","ownerSlug":"mercor","artifactType":"Benchmark","availabilityKind":"public-sample","access":["Public / Open"],"summary":"An expert-authored benchmark of economically valuable tasks in finance, consulting, law, and medicine.","description":"The AI Productivity Index assesses model performance on professional work across investment banking, management consulting, law, and primary care. Mercor provides public research, data, code, and sample-task links.","domains":["finance","legal","medical","enterprise"],"capabilities":{"expert-authored":"yes","human-in-the-loop":"yes","long-horizon":"unknown","tool-use":"unknown"},"verifierType":"Expert-authored evaluation criteria; full verifier details vary by task","lastVerified":"2026-08-16","verificationStatus":"confirmed","confidence":"high","links":[{"label":"Research overview","url":"https://www.mercor.com/research/","kind":"official"},{"label":"Dataset","url":"https://huggingface.co/datasets/Mercor/APEX","kind":"hugging-face"}],"sourceIds":["mercor-apex","mercor-apex-data"]}],"sources":[{"id":"surge-products","url":"https://surgehq.ai/products","title":"Products","publisher":"Surge AI","sourceType":"official","accessedAt":"2026-08-16","verificationStatus":"company-reported"},{"id":"surge-off-the-shelf","url":"https://surgehq.ai/ots","title":"Off-the-Shelf Training Data","publisher":"Surge AI","sourceType":"official","accessedAt":"2026-08-16","verificationStatus":"company-reported"},{"id":"surge-corecraft","url":"https://surgehq.ai/blog/rl-envs-real-world","title":"RL Environments and the Hierarchy of Agentic Capabilities","publisher":"Surge AI Research Team","sourceType":"official","publishedAt":"2025-11-03","accessedAt":"2026-08-16","verificationStatus":"company-reported"},{"id":"surge-benchmarks","url":"https://surgehq.ai/benchmarks","title":"Benchmarks","publisher":"Surge AI","sourceType":"official","accessedAt":"2026-08-16","verificationStatus":"company-reported"},{"id":"rise-data-labs-rl-environments","url":"https://risedatalabs.com/solutions/rl-environment","title":"RL Environment Data","publisher":"Rise Data Labs","sourceType":"official","accessedAt":"2026-08-16","verificationStatus":"company-reported"},{"id":"rise-data-labs-home","url":"https://risedatalabs.com/","title":"Rise Data Labs  -  High-fidelity data for the real world","publisher":"Rise Data Labs","sourceType":"official","accessedAt":"2026-08-16","verificationStatus":"company-reported"},{"id":"rise-data-labs-rlhf","url":"https://risedatalabs.com/solutions/rlhf","title":"RLHF & Preference Learning","publisher":"Rise Data Labs","sourceType":"official","accessedAt":"2026-08-16","verificationStatus":"company-reported"},{"id":"rise-data-labs-finance-tasks-submission","url":"https://risedatalabs.com/solutions/rl-environment","title":"Finance tasks catalog submission","publisher":"Rise Data Labs","sourceType":"official","accessedAt":"2026-08-16","verificationStatus":"vendor-submitted","notes":"Private commercial finance-task inventory supplied to EnvIndex; public page supports finance environment capability but does not expose the private task contents."},{"id":"rise-data-labs-long-horizon-tasks-submission","url":"https://risedatalabs.com/solutions/rl-environment","title":"Long-horizon knowledge-work tasks catalog submission","publisher":"Rise Data Labs","sourceType":"official","accessedAt":"2026-08-16","verificationStatus":"vendor-submitted","notes":"Private commercial inventory supplied to EnvIndex; underlying task contents were not provided for public inspection."},{"id":"rise-data-labs-linkedin","url":"https://www.linkedin.com/company/rise-data-labs","title":"Rise Data Labs company profile","publisher":"LinkedIn","sourceType":"database","accessedAt":"2026-08-16","verificationStatus":"third-party-reported"},{"id":"mechanize-gba","url":"https://www.mechanize.work/technical-blog/introducing-gba-eval/","title":"Introducing GBA Eval","publisher":"Mechanize","sourceType":"official","publishedAt":"2026-05-02","accessedAt":"2026-08-16","verificationStatus":"company-reported"},{"id":"scale-swe-pro","url":"https://labs.scale.com/papers/swe_bench_pro","title":"SWE-Bench Pro: Can AI Agents Solve Long-Horizon Software Engineering Tasks?","publisher":"Scale Labs","sourceType":"paper","publishedAt":"2025-09-19","accessedAt":"2026-08-16","verificationStatus":"confirmed"},{"id":"scale-swe-public","url":"https://labs.scale.com/leaderboard/swe_bench_pro_public","title":"SWE-Bench Pro  -  Public Dataset","publisher":"Scale Labs","sourceType":"official","accessedAt":"2026-08-16","verificationStatus":"confirmed"},{"id":"scale-swe-private","url":"https://labs.scale.com/leaderboard/swe_bench_pro_private","title":"SWE-Bench Pro  -  Private Dataset","publisher":"Scale Labs","sourceType":"official","accessedAt":"2026-08-16","verificationStatus":"company-reported"},{"id":"handshake-gandalf","url":"https://joinhandshake.com/research/ai/gandalf-the-grader/","title":"Your verifier is probably the bottleneck. We built one that isn’t.","publisher":"Handshake AI Research","sourceType":"official","publishedAt":"2026-05-27","accessedAt":"2026-08-16","verificationStatus":"company-reported"},{"id":"handshake-gandalf-github","url":"https://github.com/Handshake-AI-Research/gandalf-the-grader","title":"Gandalf the Grader source repository","publisher":"Handshake AI Research","sourceType":"repository","accessedAt":"2026-08-16","verificationStatus":"confirmed"},{"id":"handshake-banker-dataset","url":"https://huggingface.co/datasets/handshake-ai-research/bankertoolbench","title":"BankerToolBench dataset","publisher":"Handshake AI Research","sourceType":"dataset","accessedAt":"2026-08-16","verificationStatus":"confirmed"},{"id":"afterquery-appbench","url":"https://www.afterquery.com/leaderboard/app-bench","title":"App-Bench  -  AI Web App Generation","publisher":"AfterQuery","sourceType":"official","accessedAt":"2026-08-16","verificationStatus":"company-reported"},{"id":"mercor-apex","url":"https://www.mercor.com/research/","title":"Mercor Research  -  APEX","publisher":"Mercor","sourceType":"official","accessedAt":"2026-08-16","verificationStatus":"company-reported"},{"id":"mercor-apex-data","url":"https://huggingface.co/datasets/Mercor/APEX","title":"APEX dataset","publisher":"Mercor","sourceType":"dataset","accessedAt":"2026-08-16","verificationStatus":"confirmed"}]}