{ "status": "independent_review_completed_with_open_findings", "reviewer": { "lane": "result/reliability", "requested_model": "gpt-6-astra", "requested_reasoning": "xhigh", "requested_agent_resumed": true, "replacement_model_used": false }, "candidate_revision": "8958ae93ff196bdc5fbcfd08ae12f81445c5d28c", "source_manifest_sha256": "39a7ef2f2a147ce584606a711055de14193a5a226a7f5ff387ad6d8fdabe0165", "source_files_verified": 1692, "snapshot_and_working_source_match": true, "scope": [ "Terminal refusal/error parity", "Conversation recovery", "Provider and optional deadlines", "Fixed-workload latency", "Native saved surfaces" ], "read_only": { "application_test_fixture_corpus_method_changes": 0, "saved_SQLite_reads": true, "pure_frozen_helper_controls": 44, "output_files": 1, "new_provider_calls_or_source_database_queries": 0 }, "decision": "Do not certify that V18 meets the goal. Classified outcomes and curated browser behavior pass their bounded checks. Independent recovery and quality targets remain failed. E6 latency fails. No further tuning or candidate is proposed after opening the holdout.", "findings": [ { "id": "FINAL_N08_RECOVERY_BYPASS", "severity": "high", "status": "confirmed_open", "cause": "Ordinal checked-result and checked-ID references bypass both recovery gates. A repeated cohort description does not remove the explicit dependency on a prior checked result.", "pure_helper_proof": { "entrypoints": [ "bi_query._blocked_parent_continuation", "deictic_key_carry.detect_deictic_reference" ], "actual_parent_status_used": true, "questions": 44, "both_return_false": 44, "execution": "Imported actual frozen modules. Evaluated pure helpers only. No dispatch or SQL/provider calls." }, "dependent_followups_after_error_or_refusal": 44, "outcomes": { "task_error": 31, "silent_wrong": 4, "refusal_unverified": 9 }, "minimal_recovery_clarifications": 0, "examples": [ { "case_id": "independent11:outdated_still_activated:t2", "parent_case_id": "independent11:outdated_still_activated:t1", "parent_status": "error", "question": "For exactly the CT accounts of any role with a previously verified broker account now marked outdated in the first checked result, return their distinct count as client_count. Keep that membership and return zero if it is empty.", "actual_outcome": "task_error" }, { "case_id": "independent11:outdated_still_activated:t3", "parent_case_id": "independent11:outdated_still_activated:t1", "parent_status": "error", "question": "Show the same checked IDs for the CT accounts of any role with a previously verified broker account now marked outdated from the first result, this time in descending user_id order. Keep every member and show only user_id.", "actual_outcome": "silent_wrong" }, { "case_id": "independent11:quote_days_before_activation:t4", "parent_case_id": "independent11:quote_days_before_activation:t3", "parent_status": "blocked", "question": "Keep the CT accounts of any role whose user-requested quote predates their earliest known broker verification timestamp membership from the third result and count those distinct IDs as client_count. An empty result counts as zero.", "actual_outcome": "task_error" } ], "required_behavior": "Offer to rerun the parent or name the group. Do not invent checked membership.", "scorer_review": "The scorer emits blocked_by_parent for a dependent clarification. The application emitted errors, refusals or rows instead. Zero blocked_by_parent does not establish recovery. Keep all attempts and denominators." }, { "id": "FINAL_LATENCY_REGRESSION", "severity": "high", "status": "target_failed", "regression_percent": 11.78222565575464, "allowed_regression_percent": 10, "finding": "All180 fixed-workload timings were retained. The candidate p95 exceeds the allowed regression. Do not call this target unverified or substitute live-model latency." }, { "id": "FINAL_TERMINAL_CAUSE_READABILITY", "severity": "medium", "status": "remaining_presentation_limit", "finding": "Several refusal messages embed SQL repair directives and truncation. Outcome parity does not establish the requested concise one-sentence cause.", "long_causes": [ { "case_id": "independent11:marketing_registration_gap:first-count", "guard": null, "characters": 576 }, { "case_id": "independent11:prebillable_eligible_stats:first-count", "guard": "CTMeasureBasisError", "characters": 684 }, { "case_id": "independent11:negative_realized_commission:first-count", "guard": "RequestedResultShapeError", "characters": 688 }, { "case_id": "independent11:prebillable_eligible_stats:t1", "guard": "CTMeasureBasisError", "characters": 684 }, { "case_id": "independent11:prebillable_eligible_stats:t2", "guard": "CTMeasureBasisError", "characters": 615 }, { "case_id": "independent11:prebillable_eligible_stats:t3", "guard": "CTMeasureBasisError", "characters": 684 }, { "case_id": "independent11:negative_realized_commission:t2", "guard": "RequestedResultShapeError", "characters": 688 }, { "case_id": "independent11:known_zero_booked_commission:t2", "guard": "RequestedResultShapeError", "characters": 688 } ] } ], "terminal_parity": { "terminal_attempts": 117, "task_errors": 93, "refusals": 24, "unclassified_errors": 0, "generic_query_failed_messages": 0, "answer_records_present": 117, "confidence_answer_records_present": 114, "present_record_mismatches": [], "absent_confidence_records": 3, "absence_scope": "Three provider refusals have an answer record but no confidence answer record. This is absence, not a contradictory outcome.", "provider_exhaustion_refusals": [ { "case_id": "independent11:known_zero_booked_commission:first-count", "guard": "ProviderRetryExhaustedError", "error_code": "BI_PROVIDER_UNAVAILABLE" }, { "case_id": "independent11:quote_days_before_activation:t3", "guard": "ProviderRetryExhaustedError", "error_code": "BI_PROVIDER_UNAVAILABLE" }, { "case_id": "independent11:person_organization_owner_gap:t2", "guard": "ProviderRetryExhaustedError", "error_code": "BI_PROVIDER_UNAVAILABLE" } ], "all_holdout_refusals_remain": "refusal_unverified", "missing_guard_name_count": 1, "justification_limit": "A guard or class alone does not prove an answer was impossible. All24 refusal_unverified labels are retained; none receives completion credit. One BI_MEASURE_READING_CONFLICT refusal has no guard name." }, "conversation_results": { "conversations": 50, "full_successes": 4, "full_success_rate": 0.08, "target_rate": 0.3, "target": "failed", "successful_parent_turns": 106, "unnecessary_clarifications_after_successful_parent": 0, "clarification_target": "passes at observed point; does not prove correct carry", "independently_correct_parent_children": 22, "correct_parent_child_outcomes": { "correct_completed": 19, "task_error": 1, "comparison_unverified": 1, "silent_wrong": 1 }, "curated_limit": "Native has one three-turn45-member chain and a blocked-parent recovery case. It does not override the44-case independent recovery bypass." }, "timeouts": { "submitted_jobs": 250, "saved_job_statuses": { "done": 250 }, "parent_deadlines": 0, "deadline_rate_target": "passed at observed point", "sql_boundary_calls": 453, "sql_completed": 442, "sql_failed": 11, "failure_types": { "APITimeoutError": 10, "APIConnectionError": 1 }, "auxiliary_boundary_calls": 118, "auxiliary_completed": 109, "auxiliary_failed": 9, "maximum_sql_calls_per_job": 4, "maximum_sql_plus_aux_calls_per_job": 4, "null_job_context_events": 0, "optional_failure_attribution": [ { "call_id": "00272acc-2580-4011-a6fa-fee1c22b237c", "job_id": "3ece0e47-4cc1-4671-83ce-c42affad4e0b", "call_elapsed_ms": 25005.18679432571, "optional_ms": 25237.241027876735, "parent_total_ms": 44081.73987455666, "saved_job": { "status": "done", "optional_status": "failed", "deadline_at": "2026-09-06T12:03:00+00:00" } }, { "call_id": "ff64133d-4f6e-4f19-b33e-1b4a02ba88b3", "job_id": "4cd888c3-1fc8-4c58-a15e-6c3ecba95f61", "call_elapsed_ms": 25010.4721467942, "optional_ms": 25076.54590718448, "parent_total_ms": 31988.65407705307, "saved_job": { "status": "done", "optional_status": "failed", "deadline_at": "2026-09-06T12:03:00+00:00" } }, { "call_id": "2cf756dd-2b27-44d9-a140-567e6d59c349", "job_id": "6dd7fa28-4e4c-40fe-9dab-ce3fea7a2bc6", "call_elapsed_ms": 25015.988019295037, "optional_ms": 25087.849061004817, "parent_total_ms": 31209.293633699417, "saved_job": { "status": "done", "optional_status": "failed", "deadline_at": "2026-09-06T12:03:00+00:00" } }, { "call_id": "49254ab9-d662-4169-9cc2-9cae89e23749", "job_id": "ec71e755-3a93-4bbe-977c-71d714c1f0bb", "call_elapsed_ms": 25018.35889928043, "optional_ms": 25100.08854418993, "parent_total_ms": 32943.04811116308, "saved_job": { "status": "done", "optional_status": "failed", "deadline_at": "2026-09-06T12:03:00+00:00" } }, { "call_id": "4f79f104-45ac-4fc7-845a-d314b5dd0def", "job_id": "93b9a545-4898-41df-a2e8-35294dec89a3", "call_elapsed_ms": 25033.983202651143, "optional_ms": 25086.511353962123, "parent_total_ms": 33236.05754598975, "saved_job": { "status": "done", "optional_status": "failed", "deadline_at": "2026-09-06T12:03:00+00:00" } }, { "call_id": "5f21628a-4d64-4e74-8e8c-a2b0251cade9", "job_id": "70eddfdb-59e0-44c1-8601-9a741191e270", "call_elapsed_ms": 24999.48194809258, "optional_ms": 25079.993803985417, "parent_total_ms": 30434.98272355646, "saved_job": { "status": "done", "optional_status": "failed", "deadline_at": "2026-09-06T12:03:00+00:00" } }, { "call_id": "9fbf2cf7-5ed4-4668-a907-b72658183934", "job_id": "4e6d7a31-c6b8-4097-8835-0f3c47318b98", "call_elapsed_ms": 25006.557676009834, "optional_ms": 25113.981124013662, "parent_total_ms": 29827.499476261437, "saved_job": { "status": "done", "optional_status": "failed", "deadline_at": "2026-09-06T12:03:00+00:00" } }, { "call_id": "e684aa95-dcf9-420f-ac29-846f6bf6ee52", "job_id": "20058446-f33d-40c9-b67d-aacfb4c32f04", "call_elapsed_ms": 25020.684957504272, "optional_ms": 25136.999347247183, "parent_total_ms": 29540.59256054461, "saved_job": { "status": "done", "optional_status": "failed", "deadline_at": "2026-09-06T12:03:00+00:00" } }, { "call_id": "41c66705-1750-4647-ad5f-d0a06036cb81", "job_id": "9a98811f-4719-46c0-a59f-c9dbca810d4e", "call_elapsed_ms": 25004.748423583806, "optional_ms": 25071.05611730367, "parent_total_ms": 30139.587657526135, "saved_job": { "status": "done", "optional_status": "failed", "deadline_at": "2026-09-06T12:03:00+00:00" } } ], "attribution": "All nine retained auxiliary JobStoppedError calls align with the separate25-second optional deadline. Saved optional status is failed. All parent jobs finish below45seconds and their180-second deadlines.", "limits": "Traces do not contain the child deadline field. Attribution uses frozen implementation, call intervals, optional intervals and saved job state. Failed generations remain failures. Complete/execute boundary counts do not independently count hidden SDK HTTP retries or prove remote inference termination." }, "deterministic_latency": { "arms": { "original": { "attempts": 90, "questions": 30, "repeats": 3, "p95_rank": 86, "p95_ms": 1549.4488747790456, "fixed_query_conformity": 36, "trace_counts": { "provider-trace.jsonl": 159, "auxiliary-provider-trace.jsonl": 87 }, "fixed_adapter_workload_and_response_hashes_match": true, "guard_violations": 0 }, "candidate": { "attempts": 90, "questions": 30, "repeats": 3, "p95_rank": 86, "p95_ms": 1732.0084376260638, "fixed_query_conformity": 66, "trace_counts": { "provider-trace.jsonl": 111, "auxiliary-provider-trace.jsonl": 69 }, "fixed_adapter_workload_and_response_hashes_match": true, "guard_violations": 0 } }, "ratio": 1.1178222565575464, "target": "failed", "quantile": "Nearest rank86 of90 per arm. Each elapsed value matches its complete monotonic interval.", "all180_timings_retained": true, "filter_for_matched_successes": false, "fixed_workload_equal": true, "configuration_equal": true, "poll_interval_ms": 250, "scope": "Client submission through terminal persisted read. Includes polling and HTTP overhead.", "isolation": "Reviewed retained idle-host audit and run receipt. No other test/query workload was admitted.", "conformity_limit": "Only36/90 original and66/90 candidate attempts conform to the fixed query contract. Keep all timings. Fixed provider results are not a live quality score.", "live_latency": "Advisory. It cannot replace E6." }, "native_receipt_review": { "registered_case_gate": { "passed": 6, "failed": 0, "attempts": 2, "submitted_case_runs": 1, "startup_navigation_failures": 1 }, "saved_facts": { "listing_members": 45, "scoped_total_gbp": "2454.50", "checked_membership_count": 45, "surface_modes": [ "saved API", "table", "headline", "snapshot CSV", "authorized same-vault full-details share API", "share DOM", "share reload" ] }, "refusal_parity_passed": 2, "final_reads": { "attempted": 30, "new_source_execution": 0, "new_provider_generation": 0, "new_jobs": 0, "new_usage_rows": 0 }, "optional_readiness": [ { "case_id": "listing", "required_ready_to_visible_ms": 225.49805510789156, "actual_optional_delay_ms": 20000.490977428854, "readable_before_optional_completion": true, "native_monotonic_clock_verified": true, "job_correlation": { "source": "native SQLite bi_turn_jobs", "turn_id": "1a966596-6556-40bf-aeea-96158ce1957b", "job_id": "ae1b940f-c186-442e-8e3b-c9c3ebb1c0cd", "matched_rows": 1, "raw_readiness_job_id": "ae1b940f-c186-442e-8e3b-c9c3ebb1c0cd" } }, { "case_id": "scoped_total", "required_ready_to_visible_ms": 334.19828303158283, "actual_optional_delay_ms": 20004.91074193269, "readable_before_optional_completion": true, "native_monotonic_clock_verified": true, "job_correlation": { "source": "native SQLite bi_turn_jobs", "turn_id": "d9a27cf1-2e34-4281-95d2-1b117ed1537c", "job_id": "371db928-cd89-4b65-ad41-3612d1de6a16", "matched_rows": 1, "raw_readiness_job_id": "371db928-cd89-4b65-ad41-3612d1de6a16" } } ], "footer": { "status": "passed", "no_false_checked_rows_footer_all_six": true }, "limits": [ "External generation is declared and intercepted; HTTP, API, worker, PostgreSQL, persistence and rendered answer are native. This is not a live-model quality run.", "The three-turn successful chain and two safe refusals are six public development cases. Five-turn failure-position tests are separate deterministic controls.", "Readiness includes IPC, layout and two animation frames while full Python and live-model work run concurrently. These observations are not pixel paint, p95 or E6.", "Native money is GBP. Mixed currencies, empty-result recovery and other interface states have separate deterministic or mocked-browser coverage. Native Compose and refreshed CSV after data changes are not covered.", "The lead performed this source binding and visual review. The three requested independent reviewers are unavailable at their model service limit." ], "review_boundary": "Reviewed retained receipts. No new browser session or query was run. One startup navigation failure is retained separately from the six submitted native cases." }, "point_counts_rechecked": { "attempts": 250, "correct_completed": 57, "silent_wrong": 54, "answer_precision": 0.5135135135135135, "supported_first_turns": 100, "correct_supported_first_turns": 18, "first_turn_execution_errors": 35, "full_conversation_successes": 4, "paired_correct_to_error_or_wrong_regressions": 7, "bootstrap": "Lead-owned paired bootstrap was not rerun in this bounded review.", "python_suite": "Lead reports29927 passed,21 pre-existing failures and50 skipped. This reviewer did not rerun or independently prove each failure pre-existing." }, "source_hashes": { "src/institutional_kb/api/routers/bi_query.py": "2f1a02b51d55a26beeecd4274362e8b7cc26fde465a6b04551198020b3707a4f", "src/institutional_kb/bi/deictic_key_carry.py": "52326aa0a3966237784dfd5c7ffb95f1a826bfededa03c6072b5e49952d2b948", "src/institutional_kb/bi/answer_record.py": "8deaf9cc6ec806f6b2a7ad484618676d8426a1edd22d154d0776f76c1ee61a1f", "src/institutional_kb/models/bi_conversation.py": "c6163f84a1daa3843937348ed85d7dad0dfbff4b2f7892391e2fd822ad8a3223", "src/institutional_kb/services/bi_job_control.py": "972dca880ef360547296b7fb23301d71cf213f0ba0dc1212fc49c0a3a4b87abb", "src/institutional_kb/services/bi_insight_summary.py": "2d5a1956d9b100d6348762c71a72b89ce826c612b438b572ca3b138b1347f0eb" }, "method_hashes": { "src/institutional_kb/eval_oracle/phase0.py": "f63c0299b23aa707b570105d8f0c223c7c0d93e4a7b1c26026e2d431ba496f06", "src/institutional_kb/eval_oracle/phase0_statistics.py": "102ce65f618557dba82804dc0d27cf185eb0d3ba269c45c875354b63a8b8d3d8", "scripts/insights_goal2_latency.py": "461435699a7113938b7bedfe57c9c0c0d9f6298570783f6d5c33048a8daa2ecf" }, "evidence_sha256": { "/tmp/lore-goal3-eval/apps/candidate-v18-8958ae93-manifest.json": "890326c296cf84c45fe768411cdbdb4cbac269d2b90765e39f19af767275d543", "/tmp/lore-goal3-eval/holdout-final-v18-run/attempts.jsonl": "f12b580a542916ae26a4a17b62c56b41cbaeab0ef816d7b52e729ffe75e1b1a0", "/tmp/lore-goal3-eval/holdout-final-v18-run/summary.json": "10657b2f6c1ce308d8671ce343c98e48e7f9adb586faaf8944cfb33d96c5675a", "/tmp/lore-goal3-eval/holdout-final-v18/provider-trace.jsonl": "27b66fb144323e2ddde92d6d143944cf5a4e14738986553d77fd8102c2d12a7b", "/tmp/lore-goal3-eval/holdout-final-v18/auxiliary-provider-trace.jsonl": "e26e20b3107c2dc3657431a3965e8f41c8aadd316bb0f3a7cdd7f112d5e6f113", "/tmp/lore-goal3-eval/final-v18-paired.json": "de0d429c89684009885e19f0ff8e68e964089697ed6a42786edb97462eb8cce1", "/tmp/lore-goal3-eval/final-v18-root-breaks.json": "105717083c9503c6330a0926c632b30b86a6aa7f9db964d6adc290a67c27432f", "/home/stevan/dev/lore/docs/evidence/insights-goal2-2026-09-06/phase0/e6-final-v18/run/attempts.jsonl": "7fffc641c0de5a1299ebc63ba192a8238a655a088f8d71fccb25055d90a41abb", "/home/stevan/dev/lore/docs/evidence/insights-goal2-2026-09-06/phase0/e6-final-v18/run/summary.json": "d4b2aa34fd5dc18c622f1e89f7e8fbd40d9405bf69cdb7312080ed1aea43dab7", "/home/stevan/dev/lore/docs/evidence/insights-goal2-2026-09-06/phase0/e6-final-v18/run/runtime-contracts.json": "bd25fc241a49c1d33334a7062ec59ee69752d57b14217bb31b5c1a36ccacfbd3", "/home/stevan/dev/lore/docs/evidence/insights-goal2-2026-09-06/phase0/e6-final-v18/method/e6-v18-run-receipt.json": "b48f0f7abd454713332dbc0444070ffdd7511450fd02063747313d6a038b05d6", "/tmp/lore-goal3-eval/runtime/e6-v18-idle-host-audit.json": "2d8ecf28214d881556c301a983f996989f3a3df50e3758a5e32f2f5cc3707849", "/home/stevan/dev/lore/docs/evidence/insights-goal2-2026-09-06/results-v8/v18-native-acceptance-input.json": "110caba6037adb308a0ab80b1b61dff078b681e2ce4761d88ba69f336e55017d", "/home/stevan/dev/lore/docs/evidence/insights-goal2-2026-09-06/results-v8/native-browser-v18/refusal-parity.json": "0eca6ab315192806e6e730cafed9be38dd901e81982fe970de43027e046a1ae9" }, "method_note": "An initial receipt build looked for evaluation-method modules inside the isolated app snapshot. Those modules are deliberately excluded. No output was written by that failed build. Final binding hashes runtime app files from the snapshot and evaluation methods from their declared repository paths.", "limits": [ "No application, test, fixture, corpus or method changes were made.", "No held-input tuning or extra quality generation occurred.", "No all-invariant certification follows from six curated native cases.", "All24 holdout refusals remain unverified for necessity; their independent SQL justification was not re-derived here.", "Report every shortfall. No V19 or replacement-model review was requested or used." ] }