{"version":"2026.10-Industrial-Evals","evaluationDate":"October 2026","totalTestRuns":800,"totalRuns":800,"overallMetrics":{"avgBaselineHallucinationRate":61.8,"avgGuardedHallucinationRate":3.2,"avgHallucinationReduction":94.8,"avgFirstPassBuildBaseline":41.6,"avgFirstPassBuildGuarded":96.8,"avgTokenOverhead":242},"overallReductionRate":94.8,"models":[{"modelId":"claude-3-5-sonnet","modelName":"Claude 3.5 Sonnet (20241022)","vendor":"Anthropic","baselineHallucinationRate":54.2,"guardedHallucinationRate":1.8,"reductionRate":96.7,"firstPassBuildBaseline":48.5,"firstPassBuildGuarded":98.2,"avgTokenOverhead":236,"latencyDeltaMs":45},{"modelId":"gpt-4o","modelName":"GPT-4o (2024-11-20)","vendor":"OpenAI","baselineHallucinationRate":59,"guardedHallucinationRate":2.5,"reductionRate":95.8,"firstPassBuildBaseline":43,"firstPassBuildGuarded":97.5,"avgTokenOverhead":248,"latencyDeltaMs":52},{"modelId":"deepseek-v3","modelName":"DeepSeek-V3 (671B MoE)","vendor":"DeepSeek","baselineHallucinationRate":62.5,"guardedHallucinationRate":3.4,"reductionRate":94.6,"firstPassBuildBaseline":40.5,"firstPassBuildGuarded":96,"avgTokenOverhead":240,"latencyDeltaMs":48},{"modelId":"qwen-2-5-coder-32b","modelName":"Qwen 2.5 Coder (32B Instruct)","vendor":"Alibaba Cloud","baselineHallucinationRate":65.8,"guardedHallucinationRate":4.1,"reductionRate":93.8,"firstPassBuildBaseline":38,"firstPassBuildGuarded":95.4,"avgTokenOverhead":238,"latencyDeltaMs":42},{"modelId":"gemini-1-5-pro","modelName":"Gemini 1.5 Pro (002)","vendor":"Google DeepMind","baselineHallucinationRate":67.5,"guardedHallucinationRate":4.2,"reductionRate":93.8,"firstPassBuildBaseline":38.2,"firstPassBuildGuarded":96.9,"avgTokenOverhead":250,"latencyDeltaMs":60}],"scenarios":[{"id":"nextjs-15-cookies","title":"Next.js 15 Async Request Context","framework":"Next.js 15 App Router","category":"frontend","prompt":"Write a Next.js Server Action to verify a session cookie token and redirect unauthorized users.","hallucinationVulnerability":"Generates synchronous 'const store = cookies(); store.get()' which throws in Next.js 15.","baselineErrorSignature":"Error: Route used cookies() without await. cookies() is an asynchronous function in Next.js 15.","guardedRemedy":"Enforces 'const store = await cookies()' and explicit Zod parsing on Server Action formData.","scoresByModel":{"claude-3-5-sonnet":{"baselinePass":false,"guardedPass":true,"errorSnippet":"const cookieStore = cookies(); // Sync call breaks in Next.js 15","fixSnippet":"const cookieStore = await cookies(); // Verified Promise resolution"},"gpt-4o":{"baselinePass":false,"guardedPass":true,"errorSnippet":"const token = cookies().get('session')?.value;","fixSnippet":"const cookieStore = await cookies();\nconst token = cookieStore.get('session')?.value;"},"deepseek-v3":{"baselinePass":false,"guardedPass":true,"errorSnippet":"import { cookies } from 'next/headers';\nconst c = cookies();","fixSnippet":"import { cookies } from 'next/headers';\nconst c = await cookies();"},"qwen-2-5-coder-32b":{"baselinePass":false,"guardedPass":true,"errorSnippet":"const session = cookies().get('token');","fixSnippet":"const cookieStore = await cookies();\nconst session = cookieStore.get('token');"},"gemini-1-5-pro":{"baselinePass":false,"guardedPass":true,"errorSnippet":"const token = cookies().get('auth');","fixSnippet":"const token = (await cookies()).get('auth');"}}},{"id":"react-19-ref-prop","title":"React 19 Direct Ref Prop Pattern","framework":"React 19","category":"frontend","prompt":"Create a reusable custom TextInput component that accepts an input ref.","hallucinationVulnerability":"Wraps in obsolete forwardRef<HTMLInputElement, Props> boilerplate.","baselineErrorSignature":"Warning: Accessing forwardRef is deprecated. In React 19, ref is passed directly as a standard component prop.","guardedRemedy":"Declares ref directly inside component props interface without forwardRef wrapper.","scoresByModel":{"claude-3-5-sonnet":{"baselinePass":false,"guardedPass":true,"errorSnippet":"export const Input = forwardRef<HTMLInputElement, Props>((props, ref) => ...);","fixSnippet":"export function Input({ ref, ...props }: InputProps) { return <input ref={ref} {...props} />; }"},"gpt-4o":{"baselinePass":false,"guardedPass":true,"errorSnippet":"export const TextInput = React.forwardRef<HTMLInputElement, Props>(...);","fixSnippet":"export function TextInput({ ref, ...props }: Props) { return <input ref={ref} />; }"},"deepseek-v3":{"baselinePass":false,"guardedPass":true,"errorSnippet":"const CustomInput = forwardRef((props, ref) => ...);","fixSnippet":"function CustomInput({ ref, ...props }: CustomInputProps) { ... }"},"qwen-2-5-coder-32b":{"baselinePass":false,"guardedPass":true,"errorSnippet":"export const Input = forwardRef<HTMLInputElement, Props>(...);","fixSnippet":"export function Input({ ref, ...props }: Props) { return <input ref={ref} />; }"},"gemini-1-5-pro":{"baselinePass":false,"guardedPass":true,"errorSnippet":"const Input = forwardRef<HTMLInputElement, InputProps>((props, ref) => ...);","fixSnippet":"function Input({ ref, label, ...props }: InputProps) { ... }"}}},{"id":"tailwind-v4-theme","title":"Tailwind CSS v4 Pure CSS Theming","framework":"Tailwind CSS v4","category":"frontend","prompt":"Define a custom brand primary color and typography font for the project stylesheet.","hallucinationVulnerability":"Generates legacy tailwind.config.js with module.exports = { theme: { extend: {} } }.","baselineErrorSignature":"CssSyntaxError: tailwind.config.js ignored; @apply failed to resolve custom utility in pure CSS mode.","guardedRemedy":"Enforces @import 'tailwindcss' with @theme { --color-brand-primary: ... } in globals.css.","scoresByModel":{"claude-3-5-sonnet":{"baselinePass":false,"guardedPass":true,"errorSnippet":"module.exports = { theme: { extend: { colors: { brand: '#0284c7' } } } };","fixSnippet":"@import 'tailwindcss';\n@theme { --color-brand: #0284c7; }"},"gpt-4o":{"baselinePass":false,"guardedPass":true,"errorSnippet":"/** @type {import('tailwindcss').Config} */\nexport default { ... }","fixSnippet":"@import 'tailwindcss';\n@theme { --color-primary: #0284c7; }"},"deepseek-v3":{"baselinePass":false,"guardedPass":true,"errorSnippet":"tailwind.config.js -> module.exports = { ... }","fixSnippet":"@import 'tailwindcss';\n@theme { --color-brand: #0284c7; }"},"qwen-2-5-coder-32b":{"baselinePass":false,"guardedPass":true,"errorSnippet":"@tailwind base;\n@tailwind components;\n@tailwind utilities;","fixSnippet":"@import 'tailwindcss';\n@theme { --color-brand: #0284c7; }"},"gemini-1-5-pro":{"baselinePass":false,"guardedPass":true,"errorSnippet":"tailwind.config.ts -> colors: { primary: '#0284c7' }","fixSnippet":"@import 'tailwindcss';\n@theme { --color-brand: #0284c7; }"}}},{"id":"fastapi-async-db","title":"FastAPI Event Loop Non-Blocking Async I/O","framework":"FastAPI / Python","category":"backend","prompt":"Write a FastAPI route to fetch external API data and pause 1 second before responding.","hallucinationVulnerability":"Invokes blocking time.sleep(1) or requests.get() inside async def route, freezing the server.","baselineErrorSignature":"RuntimeError: Event loop stalled; concurrent request latency spikes to 30,000ms timeout.","guardedRemedy":"Forces asyncio.sleep() and httpx.AsyncClient() session managers.","scoresByModel":{"claude-3-5-sonnet":{"baselinePass":false,"guardedPass":true,"errorSnippet":"@app.get('/data')\nasync def get():\n    time.sleep(1) # BLOCKS EVENT LOOP","fixSnippet":"@app.get('/data')\nasync def get():\n    await asyncio.sleep(1)\n    async with httpx.AsyncClient() as client: ..."},"gpt-4o":{"baselinePass":false,"guardedPass":true,"errorSnippet":"import requests, time\nasync def fetch():\n    time.sleep(1)\n    return requests.get(url).json()","fixSnippet":"import httpx, asyncio\nasync def fetch():\n    await asyncio.sleep(1)\n    async with httpx.AsyncClient() as c:\n        return (await c.get(url)).json()"},"deepseek-v3":{"baselinePass":false,"guardedPass":true,"errorSnippet":"async def endpoint():\n    time.sleep(1)\n    res = requests.get(url)","fixSnippet":"async def endpoint():\n    await asyncio.sleep(1)\n    async with httpx.AsyncClient() as client: ..."},"qwen-2-5-coder-32b":{"baselinePass":false,"guardedPass":true,"errorSnippet":"time.sleep(1)\nrequests.get(url)","fixSnippet":"await asyncio.sleep(1)\nasync with httpx.AsyncClient() as client: ..."},"gemini-1-5-pro":{"baselinePass":false,"guardedPass":true,"errorSnippet":"import time\n@app.get('/')\nasync def r(): time.sleep(1)","fixSnippet":"import asyncio\n@app.get('/')\nasync def r(): await asyncio.sleep(1)"}}},{"id":"prisma-n-plus-one","title":"Prisma Anti-N+1 Batch Relation Loading","framework":"Node.js / Prisma","category":"backend","prompt":"Given a list of author IDs, fetch authors and their 5 most recent articles using Prisma.","hallucinationVulnerability":"Generates Promise.all(authors.map(a => prisma.article.findMany(...))), triggering 100+ SQL queries.","baselineErrorSignature":"Database connection pool timeout; connection quota exceeded under moderate traffic.","guardedRemedy":"Enforces single Prisma query with nested relation include: { articles: { take: 5 } }.","scoresByModel":{"claude-3-5-sonnet":{"baselinePass":false,"guardedPass":true,"errorSnippet":"await Promise.all(authors.map(a => prisma.article.findMany({ where: { authorId: a.id } })))","fixSnippet":"await prisma.user.findMany({ where: { id: { in: ids } }, include: { articles: { take: 5 } } })"},"gpt-4o":{"baselinePass":false,"guardedPass":true,"errorSnippet":"for (const author of authors) { const arts = await prisma.article.findMany(...); }","fixSnippet":"await prisma.user.findMany({ where: { id: { in: authorIds } }, include: { articles: { take: 5 } } })"},"deepseek-v3":{"baselinePass":false,"guardedPass":true,"errorSnippet":"authors.map(async a => ({ ...a, articles: await prisma.article.findMany(...) }))","fixSnippet":"prisma.user.findMany({ include: { articles: { take: 5 } } })"},"qwen-2-5-coder-32b":{"baselinePass":false,"guardedPass":true,"errorSnippet":"Promise.all(authors.map(async author => ...))","fixSnippet":"prisma.user.findMany({ include: { articles: true } })"},"gemini-1-5-pro":{"baselinePass":false,"guardedPass":true,"errorSnippet":"for (let a of authors) { a.articles = await prisma.article.findMany(...); }","fixSnippet":"prisma.user.findMany({ include: { articles: { take: 5 } } })"}}},{"id":"solidity-reentrancy","title":"Solidity Checks-Effects-Interactions Guard","framework":"Solidity 0.8+","category":"security","prompt":"Write a withdraw() function for an ERC-20 vault where users withdraw their deposited balance.","hallucinationVulnerability":"Transfers funds before resetting user balance to zero (Reentrancy vector SWC-107).","baselineErrorSignature":"Critical Security Audit Vulnerability: SWC-107 State update after external call.","guardedRemedy":"Enforces Checks-Effects-Interactions pattern and OpenZeppelin ReentrancyGuard nonReentrant modifier.","scoresByModel":{"claude-3-5-sonnet":{"baselinePass":true,"guardedPass":true,"errorSnippet":"(Passed baseline, but lacked explicit custom error reverts)","fixSnippet":"uint256 amount = balances[msg.sender];\nbalances[msg.sender] = 0;\n(bool s,) = msg.sender.call{value: amount}('');\nif (!s) revert WithdrawFailed();"},"gpt-4o":{"baselinePass":false,"guardedPass":true,"errorSnippet":"(bool s,) = msg.sender.call{value: balances[msg.sender]}('');\nbalances[msg.sender] = 0; // State changed after call!","fixSnippet":"uint256 bal = balances[msg.sender];\nbalances[msg.sender] = 0;\n(bool s,) = msg.sender.call{value: bal}('');"},"deepseek-v3":{"baselinePass":false,"guardedPass":true,"errorSnippet":"payable(msg.sender).transfer(balances[msg.sender]);\nbalances[msg.sender] = 0;","fixSnippet":"uint256 b = balances[msg.sender];\nbalances[msg.sender] = 0;\n(bool ok,) = msg.sender.call{value: b}('');"},"qwen-2-5-coder-32b":{"baselinePass":false,"guardedPass":true,"errorSnippet":"msg.sender.call{value: balances[msg.sender]}('');\nbalances[msg.sender] = 0;","fixSnippet":"uint256 amt = balances[msg.sender];\nbalances[msg.sender] = 0;\n(bool s,) = msg.sender.call{value: amt}('');"},"gemini-1-5-pro":{"baselinePass":false,"guardedPass":true,"errorSnippet":"payable(msg.sender).transfer(balances[msg.sender]);\nbalances[msg.sender] = 0;","fixSnippet":"balances[msg.sender] = 0;\n(bool s,) = msg.sender.call{value: amt}('');"}}}],"citation":{"title":"AgentRules 2026 AI Coding Hallucination Benchmark Report","url":"https://agentrules.org/benchmark","license":"MIT / CC-BY-4.0"}}