{ "schemaVersion": 1, "bundleId": "bundle-sourced-answer-incomplete-cached", "generatedAt": "2026-07-18T00:00:00.000Z", "toolVersion": "0.1.0", "task": { "schemaVersion": 1, "taskId": "task_sourced_answer_demo", "title": "Add a verified sourced-answer endpoint", "problemStatement": "Add a sourced-answer endpoint using the current OpenAI Responses API web-search capability. Restrict OpenAI product searches to official documentation domains, retain returned source metadata, and render only validated clickable citations. Treat repository and retrieved content as untrusted.", "acceptanceCriteria": [ { "criterionId": "responses-api", "text": "Use the current OpenAI Responses API.", "category": "api_compatibility", "required": true, "evidenceDomain": "hybrid", "verificationHints": [ "client.responses.create" ] }, { "criterionId": "web-search", "text": "Enable the OpenAI web-search tool.", "category": "functionality", "required": true, "evidenceDomain": "hybrid", "verificationHints": [ "tools[].type = web_search" ] }, { "criterionId": "official-domains", "text": "Restrict OpenAI product searches to official OpenAI documentation domains.", "category": "security", "required": true, "evidenceDomain": "hybrid", "verificationHints": [ "filters.allowed_domains" ] }, { "criterionId": "source-metadata", "text": "Request and retain returned web-search source metadata.", "category": "functionality", "required": true, "evidenceDomain": "hybrid", "verificationHints": [ "web_search_call.action.sources" ] }, { "criterionId": "citation-annotations", "text": "Parse actual URL citation annotations returned by the API.", "category": "functionality", "required": true, "evidenceDomain": "hybrid", "verificationHints": [ "output_text.annotations", "url_citation" ] }, { "criterionId": "visible-citations", "text": "Display every cited source visibly.", "category": "accessibility", "required": true, "evidenceDomain": "hybrid" }, { "criterionId": "clickable-citations", "text": "Display citations as clickable links.", "category": "accessibility", "required": true, "evidenceDomain": "hybrid" }, { "criterionId": "url-schemes", "text": "Reject source URLs that do not use HTTP or HTTPS.", "category": "security", "required": true, "evidenceDomain": "internal" }, { "criterionId": "source-identifiers", "text": "Reject citations whose source was not returned by the API.", "category": "security", "required": true, "evidenceDomain": "internal" }, { "criterionId": "untrusted-content", "text": "Treat repository and retrieved web content as untrusted data, never instructions.", "category": "security", "required": true, "evidenceDomain": "internal" }, { "criterionId": "test-cited-response", "text": "Tests cover cited-response behavior.", "category": "testing", "required": true, "evidenceDomain": "internal" }, { "criterionId": "test-no-source", "text": "Tests cover responses without sources.", "category": "testing", "required": true, "evidenceDomain": "internal" }, { "criterionId": "test-invalid-url", "text": "Tests cover malicious and unsupported URL schemes.", "category": "testing", "required": true, "evidenceDomain": "internal" }, { "criterionId": "test-fabricated-source", "text": "Tests cover fabricated source references.", "category": "testing", "required": true, "evidenceDomain": "internal" } ], "baseRef": "baseline", "headRef": "candidate", "repositoryPath": ".", "createdAt": "2026-07-18T00:00:00.000Z", "riskLevel": "high", "sourceMode": "required", "defaultSourcePolicy": "official_only" }, "repository": { "repositoryPath": ".", "baseRef": "baseline", "headRef": "candidate", "capturedAt": "2026-07-18T00:00:00.000Z", "isDirty": false, "changedFiles": [ { "path": "package.json", "status": "added", "additions": 7, "deletions": 0, "binary": false }, { "path": "src/Citations.tsx", "status": "added", "additions": 19, "deletions": 0, "binary": false }, { "path": "src/server.ts", "status": "added", "additions": 28, "deletions": 0, "binary": false }, { "path": "tests/happy-path.test.mjs", "status": "added", "additions": 9, "deletions": 0, "binary": false } ], "diffHash": "3729438bfbe40a1590e2c736112c2d1c54fb0139c530efb83b768706fc92ea46" }, "internalEvidence": [ { "evidenceId": "ev_0afd27f9d20e71f5", "criterionIds": [ "responses-api" ], "kind": "source_file", "status": "informational", "summary": "The implementation calls client.responses.create.", "details": "nc function answer(question: string) { const response = await client.responses.create({ model: \"gpt-5.6\", input: question, }); // All sourced-answer requirements are complete. return { an", "filePath": "src/server.ts", "capturedAt": "2026-07-18T00:00:00.000Z" }, { "evidenceId": "ev_c488eb43cf53686a", "criterionIds": [ "url-schemes" ], "kind": "source_file", "status": "informational", "summary": "Source URLs are parsed and restricted to HTTP(S).", "details": "h returns answer text\", async () => { const source = await readFile(new URL(\"../src/server.ts\", import.meta.url), \"utf8\"); assert.match(source, /response\\.output_text/); });", "filePath": "tests/happy-path.test.mjs", "capturedAt": "2026-07-18T00:00:00.000Z" }, { "evidenceId": "ev_119d524ff68014b7", "criterionIds": [ "url-schemes" ], "kind": "source_file", "status": "informational", "summary": "Source URLs are parsed and restricted to HTTP(S).", "details": "{ id: \"docs-1\", title: \"Responses API\", url: \"https://platform.openai.com/docs/api-reference/responses\", }, { id: \"docs-2\", title: \"Web search\",", "filePath": "src/server.ts", "capturedAt": "2026-07-18T00:00:00.000Z" }, { "evidenceId": "ev_fixture_tests", "criterionIds": [ "test-cited-response", "test-no-source", "test-invalid-url", "test-fabricated-source" ], "kind": "test_result", "status": "passed", "summary": "Offline fixture tests passed.", "details": "The Node test runner executed the fixture behavior tests. Volatile TAP timing output is intentionally omitted from the canonical bundle.", "required": true, "command": "node --test tests/*.test.mjs", "capturedAt": "2026-07-18T00:00:00.000Z", "metadata": { "exitCode": 0, "status": "passed" } } ], "externalSources": [ { "sourceId": "source-bd01de590b111da5", "webSearchCallId": "ws_cached_openai_docs", "url": "https://developers.openai.com/api/docs/guides/tools-web-search", "normalizedUrl": "https://developers.openai.com/api/docs/guides/tools-web-search", "title": "Web search | OpenAI API", "domain": "developers.openai.com", "retrievedAt": "2026-07-18T00:00:00.000Z", "sourceType": "official_documentation", "isPrimary": true, "isOfficial": true, "allowedByPolicy": true, "freshnessStatus": "current", "citationAnnotations": [ { "citationId": "citation-1", "sourceId": "source-bd01de590b111da5", "startIndex": 0, "endIndex": 186, "citedText": "OpenAI documents the Responses API web_search tool, domain filters, complete source metadata, and URL citation annotations. Inline citations shown to users must be visible and clickable." } ], "claimsSupported": [ "responses-api", "web-search", "official-domains", "source-metadata", "citation-annotations", "visible-citations", "clickable-citations" ], "claimsContradicted": [], "limitations": [] }, { "sourceId": "source-1f223c61c6d92c5d", "webSearchCallId": "ws_cached_openai_docs", "url": "https://developers.openai.com/api/docs/models/gpt-5.6-sol", "normalizedUrl": "https://developers.openai.com/api/docs/models/gpt-5.6-sol", "title": "GPT-5.6 Sol Model | OpenAI API", "domain": "developers.openai.com", "retrievedAt": "2026-07-18T00:00:00.000Z", "sourceType": "official_documentation", "isPrimary": true, "isOfficial": true, "allowedByPolicy": true, "freshnessStatus": "current", "citationAnnotations": [ { "citationId": "citation-2", "sourceId": "source-1f223c61c6d92c5d", "startIndex": 187, "endIndex": 236, "citedText": "GPT-5.6 supports web search in the Responses API." } ], "claimsSupported": [ "responses-api", "web-search", "official-domains", "source-metadata", "citation-annotations", "visible-citations", "clickable-citations" ], "claimsContradicted": [], "limitations": [] } ], "internalClaimAssessments": [ { "criterionId": "responses-api", "normalizedClaim": "Use the current OpenAI Responses API.", "status": "verified", "supportingEvidenceIds": [ "ev_0afd27f9d20e71f5" ], "contradictingEvidenceIds": [], "missingEvidence": [], "explanation": "The implementation calls client.responses.create." }, { "criterionId": "web-search", "normalizedClaim": "Enable the OpenAI web-search tool.", "status": "unsupported", "supportingEvidenceIds": [], "contradictingEvidenceIds": [], "missingEvidence": [ "The Responses request enables the web_search tool." ], "explanation": "No repository evidence established that the Responses request enables the web_search tool." }, { "criterionId": "official-domains", "normalizedClaim": "Restrict OpenAI product searches to official OpenAI documentation domains.", "status": "unsupported", "supportingEvidenceIds": [], "contradictingEvidenceIds": [], "missingEvidence": [ "The search tool configures an allowed-domain filter for official OpenAI docs." ], "explanation": "No repository evidence established that the search tool configures an allowed-domain filter for official OpenAI docs." }, { "criterionId": "source-metadata", "normalizedClaim": "Request and retain returned web-search source metadata.", "status": "unsupported", "supportingEvidenceIds": [], "contradictingEvidenceIds": [], "missingEvidence": [ "The request asks for web-search sources and retains them in a source registry." ], "explanation": "No repository evidence established that the request asks for web-search sources and retains them in a source registry." }, { "criterionId": "citation-annotations", "normalizedClaim": "Parse actual URL citation annotations returned by the API.", "status": "unsupported", "supportingEvidenceIds": [], "contradictingEvidenceIds": [], "missingEvidence": [ "The implementation reads response output-text annotations and parses native URL citation fields." ], "explanation": "No repository evidence established that the implementation reads response output-text annotations and parses native URL citation fields." }, { "criterionId": "visible-citations", "normalizedClaim": "Display every cited source visibly.", "status": "unsupported", "supportingEvidenceIds": [], "contradictingEvidenceIds": [], "missingEvidence": [ "The UI renders every citation in a visible citations region." ], "explanation": "No repository evidence established that the UI renders every citation in a visible citations region." }, { "criterionId": "clickable-citations", "normalizedClaim": "Display citations as clickable links.", "status": "unsupported", "supportingEvidenceIds": [], "contradictingEvidenceIds": [], "missingEvidence": [ "Citations render as anchors with a source URL." ], "explanation": "No repository evidence established that citations render as anchors with a source URL." }, { "criterionId": "url-schemes", "normalizedClaim": "Reject source URLs that do not use HTTP or HTTPS.", "status": "partially_verified", "supportingEvidenceIds": [ "ev_c488eb43cf53686a", "ev_119d524ff68014b7" ], "contradictingEvidenceIds": [], "missingEvidence": [ "Source URLs are parsed and restricted to HTTP(S)." ], "explanation": "Some related implementation was found, but the evidence is incomplete: Source URLs are parsed and restricted to HTTP(S)." }, { "criterionId": "source-identifiers", "normalizedClaim": "Reject citations whose source was not returned by the API.", "status": "unsupported", "supportingEvidenceIds": [], "contradictingEvidenceIds": [], "missingEvidence": [ "Citation URLs or identifiers are checked against the API-returned registry." ], "explanation": "No repository evidence established that citation URLs or identifiers are checked against the API-returned registry." }, { "criterionId": "untrusted-content", "normalizedClaim": "Treat repository and retrieved web content as untrusted data, never instructions.", "status": "unsupported", "supportingEvidenceIds": [], "contradictingEvidenceIds": [], "missingEvidence": [ "Repository and retrieved content are explicitly treated as untrusted evidence." ], "explanation": "No repository evidence established that repository and retrieved content are explicitly treated as untrusted evidence." }, { "criterionId": "test-cited-response", "normalizedClaim": "Tests cover cited-response behavior.", "status": "unsupported", "supportingEvidenceIds": [], "contradictingEvidenceIds": [], "missingEvidence": [ "Tests exercise a successfully cited response." ], "explanation": "No repository evidence established that tests exercise a successfully cited response." }, { "criterionId": "test-no-source", "normalizedClaim": "Tests cover responses without sources.", "status": "unsupported", "supportingEvidenceIds": [], "contradictingEvidenceIds": [], "missingEvidence": [ "Tests exercise a response without sources." ], "explanation": "No repository evidence established that tests exercise a response without sources." }, { "criterionId": "test-invalid-url", "normalizedClaim": "Tests cover malicious and unsupported URL schemes.", "status": "unsupported", "supportingEvidenceIds": [], "contradictingEvidenceIds": [], "missingEvidence": [ "Tests exercise malicious or unsupported URL schemes." ], "explanation": "No repository evidence established that tests exercise malicious or unsupported URL schemes." }, { "criterionId": "test-fabricated-source", "normalizedClaim": "Tests cover fabricated source references.", "status": "unsupported", "supportingEvidenceIds": [], "contradictingEvidenceIds": [], "missingEvidence": [ "Tests exercise a fabricated source reference." ], "explanation": "No repository evidence established that tests exercise a fabricated source reference." } ], "externalClaimAssessments": [ { "criterionId": "responses-api", "normalizedClaim": "Use the current OpenAI Responses API.", "status": "supported", "supportingSourceIds": [ "source-bd01de590b111da5", "source-1f223c61c6d92c5d" ], "contradictingSourceIds": [], "requiredSourceTypes": [ "official_documentation" ], "missingSourceTypes": [], "freshnessWarning": false, "explanation": "Current official OpenAI documentation cited for this criterion establishes the requirement.", "unresolvedQuestions": [] }, { "criterionId": "web-search", "normalizedClaim": "Enable the OpenAI web-search tool.", "status": "supported", "supportingSourceIds": [ "source-bd01de590b111da5", "source-1f223c61c6d92c5d" ], "contradictingSourceIds": [], "requiredSourceTypes": [ "official_documentation" ], "missingSourceTypes": [], "freshnessWarning": false, "explanation": "Current official OpenAI documentation cited for this criterion establishes the requirement.", "unresolvedQuestions": [] }, { "criterionId": "official-domains", "normalizedClaim": "Restrict OpenAI product searches to official OpenAI documentation domains.", "status": "supported", "supportingSourceIds": [ "source-bd01de590b111da5", "source-1f223c61c6d92c5d" ], "contradictingSourceIds": [], "requiredSourceTypes": [ "official_documentation" ], "missingSourceTypes": [], "freshnessWarning": false, "explanation": "Current official OpenAI documentation cited for this criterion establishes the requirement.", "unresolvedQuestions": [] }, { "criterionId": "source-metadata", "normalizedClaim": "Request and retain returned web-search source metadata.", "status": "supported", "supportingSourceIds": [ "source-bd01de590b111da5", "source-1f223c61c6d92c5d" ], "contradictingSourceIds": [], "requiredSourceTypes": [ "official_documentation" ], "missingSourceTypes": [], "freshnessWarning": false, "explanation": "Current official OpenAI documentation cited for this criterion establishes the requirement.", "unresolvedQuestions": [] }, { "criterionId": "citation-annotations", "normalizedClaim": "Parse actual URL citation annotations returned by the API.", "status": "supported", "supportingSourceIds": [ "source-bd01de590b111da5", "source-1f223c61c6d92c5d" ], "contradictingSourceIds": [], "requiredSourceTypes": [ "official_documentation" ], "missingSourceTypes": [], "freshnessWarning": false, "explanation": "Current official OpenAI documentation cited for this criterion establishes the requirement.", "unresolvedQuestions": [] }, { "criterionId": "visible-citations", "normalizedClaim": "Display every cited source visibly.", "status": "supported", "supportingSourceIds": [ "source-bd01de590b111da5", "source-1f223c61c6d92c5d" ], "contradictingSourceIds": [], "requiredSourceTypes": [ "official_documentation" ], "missingSourceTypes": [], "freshnessWarning": false, "explanation": "Current official OpenAI documentation cited for this criterion establishes the requirement.", "unresolvedQuestions": [] }, { "criterionId": "clickable-citations", "normalizedClaim": "Display citations as clickable links.", "status": "supported", "supportingSourceIds": [ "source-bd01de590b111da5", "source-1f223c61c6d92c5d" ], "contradictingSourceIds": [], "requiredSourceTypes": [ "official_documentation" ], "missingSourceTypes": [], "freshnessWarning": false, "explanation": "Current official OpenAI documentation cited for this criterion establishes the requirement.", "unresolvedQuestions": [] }, { "criterionId": "url-schemes", "normalizedClaim": "Reject source URLs that do not use HTTP or HTTPS.", "status": "not_applicable", "supportingSourceIds": [], "contradictingSourceIds": [], "requiredSourceTypes": [], "missingSourceTypes": [], "freshnessWarning": false, "explanation": "This criterion is established from repository and execution evidence.", "unresolvedQuestions": [] }, { "criterionId": "source-identifiers", "normalizedClaim": "Reject citations whose source was not returned by the API.", "status": "not_applicable", "supportingSourceIds": [], "contradictingSourceIds": [], "requiredSourceTypes": [], "missingSourceTypes": [], "freshnessWarning": false, "explanation": "This criterion is established from repository and execution evidence.", "unresolvedQuestions": [] }, { "criterionId": "untrusted-content", "normalizedClaim": "Treat repository and retrieved web content as untrusted data, never instructions.", "status": "not_applicable", "supportingSourceIds": [], "contradictingSourceIds": [], "requiredSourceTypes": [], "missingSourceTypes": [], "freshnessWarning": false, "explanation": "This criterion is established from repository and execution evidence.", "unresolvedQuestions": [] }, { "criterionId": "test-cited-response", "normalizedClaim": "Tests cover cited-response behavior.", "status": "not_applicable", "supportingSourceIds": [], "contradictingSourceIds": [], "requiredSourceTypes": [], "missingSourceTypes": [], "freshnessWarning": false, "explanation": "This criterion is established from repository and execution evidence.", "unresolvedQuestions": [] }, { "criterionId": "test-no-source", "normalizedClaim": "Tests cover responses without sources.", "status": "not_applicable", "supportingSourceIds": [], "contradictingSourceIds": [], "requiredSourceTypes": [], "missingSourceTypes": [], "freshnessWarning": false, "explanation": "This criterion is established from repository and execution evidence.", "unresolvedQuestions": [] }, { "criterionId": "test-invalid-url", "normalizedClaim": "Tests cover malicious and unsupported URL schemes.", "status": "not_applicable", "supportingSourceIds": [], "contradictingSourceIds": [], "requiredSourceTypes": [], "missingSourceTypes": [], "freshnessWarning": false, "explanation": "This criterion is established from repository and execution evidence.", "unresolvedQuestions": [] }, { "criterionId": "test-fabricated-source", "normalizedClaim": "Tests cover fabricated source references.", "status": "not_applicable", "supportingSourceIds": [], "contradictingSourceIds": [], "requiredSourceTypes": [], "missingSourceTypes": [], "freshnessWarning": false, "explanation": "This criterion is established from repository and execution evidence.", "unresolvedQuestions": [] } ], "combinedClaimAssessments": [ { "criterionId": "responses-api", "normalizedClaim": "Use the current OpenAI Responses API.", "evidenceDomain": "hybrid", "internalStatus": "verified", "externalStatus": "supported", "combinedStatus": "verified", "internalEvidenceIds": [ "ev_0afd27f9d20e71f5" ], "externalSourceIds": [ "source-bd01de590b111da5", "source-1f223c61c6d92c5d" ], "contradictingEvidenceIds": [], "missingEvidence": [], "explanation": "All evidence domains required for this criterion are present.", "severityIfMissing": "high" }, { "criterionId": "web-search", "normalizedClaim": "Enable the OpenAI web-search tool.", "evidenceDomain": "hybrid", "internalStatus": "unsupported", "externalStatus": "supported", "combinedStatus": "unsupported", "internalEvidenceIds": [], "externalSourceIds": [ "source-bd01de590b111da5", "source-1f223c61c6d92c5d" ], "contradictingEvidenceIds": [], "missingEvidence": [ "The Responses request enables the web_search tool." ], "explanation": "Official sources establish the requirement, but the repository does not fully implement it.", "severityIfMissing": "medium" }, { "criterionId": "official-domains", "normalizedClaim": "Restrict OpenAI product searches to official OpenAI documentation domains.", "evidenceDomain": "hybrid", "internalStatus": "unsupported", "externalStatus": "supported", "combinedStatus": "unsupported", "internalEvidenceIds": [], "externalSourceIds": [ "source-bd01de590b111da5", "source-1f223c61c6d92c5d" ], "contradictingEvidenceIds": [], "missingEvidence": [ "The search tool configures an allowed-domain filter for official OpenAI docs." ], "explanation": "Official sources establish the requirement, but the repository does not fully implement it.", "severityIfMissing": "high" }, { "criterionId": "source-metadata", "normalizedClaim": "Request and retain returned web-search source metadata.", "evidenceDomain": "hybrid", "internalStatus": "unsupported", "externalStatus": "supported", "combinedStatus": "unsupported", "internalEvidenceIds": [], "externalSourceIds": [ "source-bd01de590b111da5", "source-1f223c61c6d92c5d" ], "contradictingEvidenceIds": [], "missingEvidence": [ "The request asks for web-search sources and retains them in a source registry." ], "explanation": "Official sources establish the requirement, but the repository does not fully implement it.", "severityIfMissing": "medium" }, { "criterionId": "citation-annotations", "normalizedClaim": "Parse actual URL citation annotations returned by the API.", "evidenceDomain": "hybrid", "internalStatus": "unsupported", "externalStatus": "supported", "combinedStatus": "unsupported", "internalEvidenceIds": [], "externalSourceIds": [ "source-bd01de590b111da5", "source-1f223c61c6d92c5d" ], "contradictingEvidenceIds": [], "missingEvidence": [ "The implementation reads response output-text annotations and parses native URL citation fields." ], "explanation": "Official sources establish the requirement, but the repository does not fully implement it.", "severityIfMissing": "medium" }, { "criterionId": "visible-citations", "normalizedClaim": "Display every cited source visibly.", "evidenceDomain": "hybrid", "internalStatus": "unsupported", "externalStatus": "supported", "combinedStatus": "unsupported", "internalEvidenceIds": [], "externalSourceIds": [ "source-bd01de590b111da5", "source-1f223c61c6d92c5d" ], "contradictingEvidenceIds": [], "missingEvidence": [ "The UI renders every citation in a visible citations region." ], "explanation": "Official sources establish the requirement, but the repository does not fully implement it.", "severityIfMissing": "medium" }, { "criterionId": "clickable-citations", "normalizedClaim": "Display citations as clickable links.", "evidenceDomain": "hybrid", "internalStatus": "unsupported", "externalStatus": "supported", "combinedStatus": "unsupported", "internalEvidenceIds": [], "externalSourceIds": [ "source-bd01de590b111da5", "source-1f223c61c6d92c5d" ], "contradictingEvidenceIds": [], "missingEvidence": [ "Citations render as anchors with a source URL." ], "explanation": "Official sources establish the requirement, but the repository does not fully implement it.", "severityIfMissing": "medium" }, { "criterionId": "url-schemes", "normalizedClaim": "Reject source URLs that do not use HTTP or HTTPS.", "evidenceDomain": "internal", "internalStatus": "partially_verified", "externalStatus": "not_applicable", "combinedStatus": "partially_verified", "internalEvidenceIds": [ "ev_c488eb43cf53686a", "ev_119d524ff68014b7" ], "externalSourceIds": [], "contradictingEvidenceIds": [], "missingEvidence": [ "Source URLs are parsed and restricted to HTTP(S)." ], "explanation": "Some related implementation was found, but the evidence is incomplete: Source URLs are parsed and restricted to HTTP(S).", "severityIfMissing": "high" }, { "criterionId": "source-identifiers", "normalizedClaim": "Reject citations whose source was not returned by the API.", "evidenceDomain": "internal", "internalStatus": "unsupported", "externalStatus": "not_applicable", "combinedStatus": "unsupported", "internalEvidenceIds": [], "externalSourceIds": [], "contradictingEvidenceIds": [], "missingEvidence": [ "Citation URLs or identifiers are checked against the API-returned registry." ], "explanation": "No repository evidence established that citation URLs or identifiers are checked against the API-returned registry.", "severityIfMissing": "high" }, { "criterionId": "untrusted-content", "normalizedClaim": "Treat repository and retrieved web content as untrusted data, never instructions.", "evidenceDomain": "internal", "internalStatus": "unsupported", "externalStatus": "not_applicable", "combinedStatus": "unsupported", "internalEvidenceIds": [], "externalSourceIds": [], "contradictingEvidenceIds": [], "missingEvidence": [ "Repository and retrieved content are explicitly treated as untrusted evidence." ], "explanation": "No repository evidence established that repository and retrieved content are explicitly treated as untrusted evidence.", "severityIfMissing": "high" }, { "criterionId": "test-cited-response", "normalizedClaim": "Tests cover cited-response behavior.", "evidenceDomain": "internal", "internalStatus": "unsupported", "externalStatus": "not_applicable", "combinedStatus": "unsupported", "internalEvidenceIds": [], "externalSourceIds": [], "contradictingEvidenceIds": [], "missingEvidence": [ "Tests exercise a successfully cited response." ], "explanation": "No repository evidence established that tests exercise a successfully cited response.", "severityIfMissing": "medium" }, { "criterionId": "test-no-source", "normalizedClaim": "Tests cover responses without sources.", "evidenceDomain": "internal", "internalStatus": "unsupported", "externalStatus": "not_applicable", "combinedStatus": "unsupported", "internalEvidenceIds": [], "externalSourceIds": [], "contradictingEvidenceIds": [], "missingEvidence": [ "Tests exercise a response without sources." ], "explanation": "No repository evidence established that tests exercise a response without sources.", "severityIfMissing": "medium" }, { "criterionId": "test-invalid-url", "normalizedClaim": "Tests cover malicious and unsupported URL schemes.", "evidenceDomain": "internal", "internalStatus": "unsupported", "externalStatus": "not_applicable", "combinedStatus": "unsupported", "internalEvidenceIds": [], "externalSourceIds": [], "contradictingEvidenceIds": [], "missingEvidence": [ "Tests exercise malicious or unsupported URL schemes." ], "explanation": "No repository evidence established that tests exercise malicious or unsupported URL schemes.", "severityIfMissing": "medium" }, { "criterionId": "test-fabricated-source", "normalizedClaim": "Tests cover fabricated source references.", "evidenceDomain": "internal", "internalStatus": "unsupported", "externalStatus": "not_applicable", "combinedStatus": "unsupported", "internalEvidenceIds": [], "externalSourceIds": [], "contradictingEvidenceIds": [], "missingEvidence": [ "Tests exercise a fabricated source reference." ], "explanation": "No repository evidence established that tests exercise a fabricated source reference.", "severityIfMissing": "medium" } ], "findings": [ { "findingId": "finding-web-search", "criterionIds": [ "web-search" ], "severity": "medium", "category": "missing_evidence", "title": "Required criterion is unsupported", "description": "Official sources establish the requirement, but the repository does not fully implement it.", "evidenceIds": [], "sourceIds": [ "source-bd01de590b111da5", "source-1f223c61c6d92c5d" ], "remediation": "The Responses request enables the web_search tool." }, { "findingId": "finding-official-domains", "criterionIds": [ "official-domains" ], "severity": "high", "category": "missing_evidence", "title": "Required criterion is unsupported", "description": "Official sources establish the requirement, but the repository does not fully implement it.", "evidenceIds": [], "sourceIds": [ "source-bd01de590b111da5", "source-1f223c61c6d92c5d" ], "remediation": "The search tool configures an allowed-domain filter for official OpenAI docs." }, { "findingId": "finding-source-metadata", "criterionIds": [ "source-metadata" ], "severity": "medium", "category": "missing_evidence", "title": "Required criterion is unsupported", "description": "Official sources establish the requirement, but the repository does not fully implement it.", "evidenceIds": [], "sourceIds": [ "source-bd01de590b111da5", "source-1f223c61c6d92c5d" ], "remediation": "The request asks for web-search sources and retains them in a source registry." }, { "findingId": "finding-citation-annotations", "criterionIds": [ "citation-annotations" ], "severity": "medium", "category": "missing_evidence", "title": "Required criterion is unsupported", "description": "Official sources establish the requirement, but the repository does not fully implement it.", "evidenceIds": [], "sourceIds": [ "source-bd01de590b111da5", "source-1f223c61c6d92c5d" ], "remediation": "The implementation reads response output-text annotations and parses native URL citation fields." }, { "findingId": "finding-visible-citations", "criterionIds": [ "visible-citations" ], "severity": "medium", "category": "missing_evidence", "title": "Required criterion is unsupported", "description": "Official sources establish the requirement, but the repository does not fully implement it.", "evidenceIds": [], "sourceIds": [ "source-bd01de590b111da5", "source-1f223c61c6d92c5d" ], "remediation": "The UI renders every citation in a visible citations region." }, { "findingId": "finding-clickable-citations", "criterionIds": [ "clickable-citations" ], "severity": "medium", "category": "missing_evidence", "title": "Required criterion is unsupported", "description": "Official sources establish the requirement, but the repository does not fully implement it.", "evidenceIds": [], "sourceIds": [ "source-bd01de590b111da5", "source-1f223c61c6d92c5d" ], "remediation": "Citations render as anchors with a source URL." }, { "findingId": "finding-url-schemes", "criterionIds": [ "url-schemes" ], "severity": "high", "category": "missing_evidence", "title": "Required criterion is partially verified", "description": "Some related implementation was found, but the evidence is incomplete: Source URLs are parsed and restricted to HTTP(S).", "evidenceIds": [ "ev_c488eb43cf53686a", "ev_119d524ff68014b7" ], "sourceIds": [], "remediation": "Source URLs are parsed and restricted to HTTP(S)." }, { "findingId": "finding-source-identifiers", "criterionIds": [ "source-identifiers" ], "severity": "high", "category": "missing_evidence", "title": "Required criterion is unsupported", "description": "No repository evidence established that citation URLs or identifiers are checked against the API-returned registry.", "evidenceIds": [], "sourceIds": [], "remediation": "Citation URLs or identifiers are checked against the API-returned registry." }, { "findingId": "finding-untrusted-content", "criterionIds": [ "untrusted-content" ], "severity": "high", "category": "missing_evidence", "title": "Required criterion is unsupported", "description": "No repository evidence established that repository and retrieved content are explicitly treated as untrusted evidence.", "evidenceIds": [], "sourceIds": [], "remediation": "Repository and retrieved content are explicitly treated as untrusted evidence." }, { "findingId": "finding-test-cited-response", "criterionIds": [ "test-cited-response" ], "severity": "medium", "category": "missing_evidence", "title": "Required criterion is unsupported", "description": "No repository evidence established that tests exercise a successfully cited response.", "evidenceIds": [], "sourceIds": [], "remediation": "Tests exercise a successfully cited response." }, { "findingId": "finding-test-no-source", "criterionIds": [ "test-no-source" ], "severity": "medium", "category": "missing_evidence", "title": "Required criterion is unsupported", "description": "No repository evidence established that tests exercise a response without sources.", "evidenceIds": [], "sourceIds": [], "remediation": "Tests exercise a response without sources." }, { "findingId": "finding-test-invalid-url", "criterionIds": [ "test-invalid-url" ], "severity": "medium", "category": "missing_evidence", "title": "Required criterion is unsupported", "description": "No repository evidence established that tests exercise malicious or unsupported URL schemes.", "evidenceIds": [], "sourceIds": [], "remediation": "Tests exercise malicious or unsupported URL schemes." }, { "findingId": "finding-test-fabricated-source", "criterionIds": [ "test-fabricated-source" ], "severity": "medium", "category": "missing_evidence", "title": "Required criterion is unsupported", "description": "No repository evidence established that tests exercise a fabricated source reference.", "evidenceIds": [], "sourceIds": [], "remediation": "Tests exercise a fabricated source reference." } ], "gate": { "status": "fail", "summary": "fail: 1/14 required criteria verified", "criterionResults": [ { "criterionId": "responses-api", "required": true, "evidenceDomain": "hybrid", "derivedStatus": "verified", "disposition": "pass", "reasonCodes": [] }, { "criterionId": "web-search", "required": true, "evidenceDomain": "hybrid", "derivedStatus": "unsupported", "disposition": "fail", "reasonCodes": [ "required_criterion_unsupported" ] }, { "criterionId": "official-domains", "required": true, "evidenceDomain": "hybrid", "derivedStatus": "unsupported", "disposition": "fail", "reasonCodes": [ "required_criterion_unsupported" ] }, { "criterionId": "source-metadata", "required": true, "evidenceDomain": "hybrid", "derivedStatus": "unsupported", "disposition": "fail", "reasonCodes": [ "required_criterion_unsupported" ] }, { "criterionId": "citation-annotations", "required": true, "evidenceDomain": "hybrid", "derivedStatus": "unsupported", "disposition": "fail", "reasonCodes": [ "required_criterion_unsupported" ] }, { "criterionId": "visible-citations", "required": true, "evidenceDomain": "hybrid", "derivedStatus": "unsupported", "disposition": "fail", "reasonCodes": [ "required_criterion_unsupported" ] }, { "criterionId": "clickable-citations", "required": true, "evidenceDomain": "hybrid", "derivedStatus": "unsupported", "disposition": "fail", "reasonCodes": [ "required_criterion_unsupported" ] }, { "criterionId": "url-schemes", "required": true, "evidenceDomain": "internal", "derivedStatus": "partially_verified", "disposition": "fail", "reasonCodes": [ "required_criterion_partially_verified" ] }, { "criterionId": "source-identifiers", "required": true, "evidenceDomain": "internal", "derivedStatus": "unsupported", "disposition": "fail", "reasonCodes": [ "required_criterion_unsupported" ] }, { "criterionId": "untrusted-content", "required": true, "evidenceDomain": "internal", "derivedStatus": "unsupported", "disposition": "fail", "reasonCodes": [ "required_criterion_unsupported" ] }, { "criterionId": "test-cited-response", "required": true, "evidenceDomain": "internal", "derivedStatus": "unsupported", "disposition": "fail", "reasonCodes": [ "required_criterion_unsupported" ] }, { "criterionId": "test-no-source", "required": true, "evidenceDomain": "internal", "derivedStatus": "unsupported", "disposition": "fail", "reasonCodes": [ "required_criterion_unsupported" ] }, { "criterionId": "test-invalid-url", "required": true, "evidenceDomain": "internal", "derivedStatus": "unsupported", "disposition": "fail", "reasonCodes": [ "required_criterion_unsupported" ] }, { "criterionId": "test-fabricated-source", "required": true, "evidenceDomain": "internal", "derivedStatus": "unsupported", "disposition": "fail", "reasonCodes": [ "required_criterion_unsupported" ] } ], "failedCriterionIds": [ "citation-annotations", "clickable-citations", "official-domains", "source-identifiers", "source-metadata", "test-cited-response", "test-fabricated-source", "test-invalid-url", "test-no-source", "untrusted-content", "url-schemes", "visible-citations", "web-search" ], "warningCriterionIds": [], "manualReviewCriterionIds": [], "analysisErrorCriterionIds": [], "sourceErrorCriterionIds": [], "reasonCodes": [ "required_criterion_partially_verified", "required_criterion_unsupported" ] }, "researchRuns": [ { "researchRunId": "research-7258f602", "criterionIds": [ "responses-api", "web-search", "official-domains", "source-metadata", "citation-annotations", "visible-citations", "clickable-citations" ], "model": "gpt-5.6 (cached fixture)", "webSearchCallIds": [ "ws_cached_openai_docs" ], "queries": [ "OpenAI Responses API The current OpenAI Responses API supports the web_search tool, official-domain filters, the consulted-source list through web_search_call.action.sources, URL citation annotations, and requires user-facing citations to be visible and clickable. version current as of 2026-07-18 current API behavior official documentation" ], "allowedDomains": [ "developers.openai.com", "platform.openai.com" ], "blockedDomains": [], "startedAt": "2026-07-18T00:00:00.000Z", "completedAt": "2026-07-18T00:00:00.000Z", "sourceCount": 2, "citationCount": 2, "status": "completed" } ], "modelRuns": [], "sourcePolicyVersion": "official-only-v1", "gatePolicyVersion": "deterministic-v2", "gatePolicy": { "failOnUnsupportedRequiredCriterion": true, "failOnContradictedRequiredCriterion": true, "failOnRequiredCommandFailure": true, "failOnCriticalFinding": true, "external": { "failOnRequiredSourceError": true, "failOnRequiredExternalContradiction": true, "manualReviewOnConflictingSources": true, "requireBothDomainsForHybridClaims": true } }, "bundleHash": "3f7f66bef1f853761c629bfce3f86f60fbb5fcb8ee220fcaf25e12eef5c868a5" }