{ "reviewer": "query_meaning; separate from lead and evaluation implementation lane", "requested_configuration": { "model": "gpt-6-astra", "reasoning": "xhigh", "service_available_this_turn": true, "replacement_model_used": false }, "review_status": "completed_bounded_independent_review_with_material_findings", "candidate_revision": "8958ae93ff196bdc5fbcfd08ae12f81445c5d28c", "source_manifest_sha256": "39a7ef2f2a147ce584606a711055de14193a5a226a7f5ff387ad6d8fdabe0165", "source_bindings": { "src/institutional_kb/services/bi_query_execution_service.py": { "frozen_sha256": "2c0af9aa99337483f40c0606d5a987aeceb0712f70bf39776ef33b4031f84061", "manifest_matches": true, "working_source_matches_frozen": true }, "src/institutional_kb/services/bi_federated_execution_service.py": { "frozen_sha256": "8578f188ee904f3885ac18d5385024e7fca8e8d2bbc1887a353eac2b8b9ea366", "manifest_matches": true, "working_source_matches_frozen": true }, "src/institutional_kb/services/bi_federated_planner.py": { "frozen_sha256": "40f17dafec2843858b56ca1bad49918605b4aeed1579ef51fcc4f69d3d097ad3", "manifest_matches": true, "working_source_matches_frozen": true }, "src/institutional_kb/bi/membership_source_obligations.py": { "frozen_sha256": "c48795d342c27d629c33a2a9fdeaa40245409269947491eca4bae39194457827", "manifest_matches": true, "working_source_matches_frozen": true }, "src/institutional_kb/bi/requested_result_shape.py": { "frozen_sha256": "8ab092aa487dae391d4823f948b965902d94dedc2b844400cec261c5f0fd68e7", "manifest_matches": true, "working_source_matches_frozen": true }, "src/institutional_kb/bi/semantic_obligations.py": { "frozen_sha256": "e79607be23b5c4e1762dfad780c88c4239f012b3fc2d309ca31b6c2f682bff50", "manifest_matches": true, "working_source_matches_frozen": true } }, "observed_at_unix": 1788777232.7439878, "review_timing": "After the sole final holdout completed. No tuning follows.", "input_bindings": { "raw_attempts": { "path": "/tmp/lore-goal3-eval/holdout-final-v18-run/attempts.jsonl", "sha256": "f12b580a542916ae26a4a17b62c56b41cbaeab0ef816d7b52e729ffe75e1b1a0", "attempts": 250 }, "manifest": { "path": "/tmp/lore-goal3-eval/apps/candidate-v18-8958ae93-manifest.json", "sha256": "890326c296cf84c45fe768411cdbdb4cbac269d2b90765e39f19af767275d543" }, "lead_counterexamples": { "path": "docs/evidence/insights-goal2-2026-09-06/phase0/held-v18-lead-semantic-counterexamples.json", "sha256": "49cde4ac846d7ff59a08f61aedcbaa4d3d43c6857920006bf13c4b8cb4afd9db" }, "late_fixture_audit": { "path": "docs/evidence/insights-goal2-2026-09-06/phase0/final-v18-late-predicate-fidelity-audit.json", "sha256": "8fbf14053ec7d3c2011fe1a752a67bb5e2afe3307e6c3033f953a655e9a81c13" } }, "observed_outcomes_unchanged": { "refusal_unverified": 24, "task_error": 93, "correct_completed": 57, "unnecessary_clarification": 1, "comparison_unverified": 21, "silent_wrong": 54 }, "scope": "Independent read-only challenge of the two supplied scored-correct cases, selected new first-turn wrong successes, and ratified employee-predicate fixture sensitivity. This is not an exhaustive proof of all 250 question interpretations.", "independent_native_checks": { "values_counterexamples": 3, "values_sql_statements": 6, "fixture_coverage_selects": 1, "all_transactions_read_only": true, "new_model_calls": 0, "new_quality_attempts": 0 }, "findings": [ { "id": "V18-QF01", "severity": "material", "status": "independently_reproduced_semantic_counterexample", "case": { "case_id": "independent11:funnel_incomplete_contact:first-count", "registered_outcome_unchanged": "correct_completed", "is_first_turn": true, "question": "How many distinct ordinary CT user accounts with complete registration but at least one missing or blank first-name, last-name, or telephone field are there? Return client_count, including zero if none qualify.", "generated_sql": "SELECT COUNT(DISTINCT a.id) AS client_count, NULL AS phone FROM public.accounts AS a JOIN public.clients AS c ON c.user_id = a.id WHERE NOT c.completed_registration_at IS NULL AND (a.role = 'User' OR a.role IS NULL) AND (NULLIF(TRIM(a.first_name), '') IS NULL OR NULLIF(TRIM(a.last_name), '') IS NULL OR NULLIF(TRIM(a.telephone), '') IS NULL)", "source_legs": [], "actual_row_count": 1, "registered_reference_row_count": 1, "actual_sample": [ { "client_count": 16 } ], "registered_reference_sample": [ { "client_count": 16 } ] }, "cause": "The candidate treats a NULL role as an ordinary User. The question and reference require role User. This input changes the count from zero to one.", "native_independent_reproduction": { "connection": "Existing disposable PostgreSQL reader; transaction read-only; explicit VALUES relations only.", "queries": [ "WITH accounts(id,role,first_name,last_name,telephone) AS (VALUES (900001, NULL::text, NULL::text, ''::text, ''::text)), clients(user_id,completed_registration_at) AS (VALUES (900001, TIMESTAMP '2026-01-01')) SELECT COUNT(*) AS client_count FROM (SELECT DISTINCT a.id AS user_id FROM accounts a JOIN clients c ON c.user_id=a.id WHERE a.role='User' AND c.completed_registration_at IS NOT NULL AND (a.first_name IS NULL OR a.first_name='' OR a.last_name IS NULL OR a.last_name='' OR a.telephone IS NULL OR a.telephone='')) cohort", "WITH accounts(id,role,first_name,last_name,telephone) AS (VALUES (900001, NULL::text, NULL::text, ''::text, ''::text)), clients(user_id,completed_registration_at) AS (VALUES (900001, TIMESTAMP '2026-01-01')) SELECT COUNT(DISTINCT a.id) AS client_count, NULL AS phone FROM accounts AS a JOIN clients AS c ON c.user_id = a.id WHERE NOT c.completed_registration_at IS NULL AND (a.role = 'User' OR a.role IS NULL) AND (NULLIF(TRIM(a.first_name), '') IS NULL OR NULLIF(TRIM(a.last_name), '') IS NULL OR NULLIF(TRIM(a.telephone), '') IS NULL)" ], "expected_first_value": 0, "candidate_first_value": 1, "observed_native_rows": [ [ [ 0 ] ], [ [ 1, null ] ] ] }, "implication": "The registered fixture match cannot establish the same meaning on another valid input. Keep the numeric score unchanged.", "ratified_basis": "docs/bi/knowledge/user-base-funnel.md:15 explicitly requires accounts.role=User; NULL is not proof of that role." }, { "id": "V18-QF02", "severity": "material", "status": "independently_reproduced_semantic_counterexample", "case": { "case_id": "independent11:note_author_deal_owner_difference:first-count", "registered_outcome_unchanged": "correct_completed", "is_first_turn": true, "question": "How many distinct Pipedrive people with an active deal note authored by someone other than the deal owner are there? Return client_count, including zero if none qualify.", "generated_sql": "SELECT COUNT(DISTINCT n.person_id) AS client_count FROM pipedrive.notes n JOIN pipedrive.deals d ON d.id = n.deal_id WHERE n.active_flag IS TRUE AND n.person_id IS NOT NULL AND n.author_user_id IS NOT NULL AND n.author_user_id <> d.owner_id", "source_legs": [], "actual_row_count": 1, "registered_reference_row_count": 1, "actual_sample": [ { "client_count": 43 } ], "registered_reference_sample": [ { "client_count": 43 } ] }, "cause": "The candidate counts the person attached to the note. The reference counts the person attached to the deal. Independent foreign keys do not make those identities equal. Two deal people can share one note person.", "native_independent_reproduction": { "connection": "Existing disposable PostgreSQL reader; transaction read-only; explicit VALUES relations only.", "queries": [ "WITH deals(id,person_id,owner_id) AS (VALUES (1,101,1),(2,102,1)), notes(person_id,deal_id,active_flag,author_user_id) AS (VALUES (201,1,true,2),(201,2,true,2)) SELECT COUNT(*) AS client_count FROM (SELECT DISTINCT d.person_id AS user_id FROM notes n JOIN deals d ON d.id=n.deal_id WHERE n.active_flag IS TRUE AND n.author_user_id IS NOT NULL AND d.owner_id IS NOT NULL AND n.author_user_id<>d.owner_id) cohort", "WITH deals(id,person_id,owner_id) AS (VALUES (1,101,1),(2,102,1)), notes(person_id,deal_id,active_flag,author_user_id) AS (VALUES (201,1,true,2),(201,2,true,2)) SELECT COUNT(DISTINCT n.person_id) AS client_count FROM notes n JOIN deals d ON d.id = n.deal_id WHERE n.active_flag IS TRUE AND n.person_id IS NOT NULL AND n.author_user_id IS NOT NULL AND n.author_user_id <> d.owner_id" ], "expected_first_value": 2, "candidate_first_value": 1, "observed_native_rows": [ [ [ 2 ] ], [ [ 1 ] ] ] }, "implication": "The registered fixture match cannot establish the same meaning on another valid input. Keep the numeric score unchanged.", "interpretation_limit": "The mismatch is proved against the registered deal-person interpretation. Independent person foreign keys on a note and a deal are not interchangeable. More explicit wording would make that identity choice easier for the end user to assess." }, { "id": "V18-QF03", "severity": "material", "status": "confirmed_from_retained_wrong_successes", "cases": [ { "case_id": "independent11:marketing_registration_gap:t1", "registered_outcome_unchanged": "silent_wrong", "is_first_turn": true, "question": "List every distinct ordinary CT user accounts opted in to marketing emails whose registration is still incomplete. Return only user_id, in ascending order; use the person ID for a Pipedrive-only person.", "generated_sql": "SELECT DISTINCT user_id, NULL AS email FROM ct.client_360 WHERE NOT user_id IS NULL LIMIT 10000", "source_legs": [], "actual_row_count": 45, "registered_reference_row_count": 8, "actual_sample": [ { "user_id": 11040, "client": "Fictional Company 40", "client_type": "cfx (corporate)", "email": "synthetic-40@example.invalid" }, { "user_id": 11053, "client": "Fictional Company 53", "client_type": "unknown", "email": "synthetic-53@example.invalid" } ], "registered_reference_sample": [ { "user_id": 11005 }, { "user_id": 11010 } ] }, { "case_id": "independent11:known_external_person_qualification:t1", "registered_outcome_unchanged": "silent_wrong", "is_first_turn": true, "question": "List every distinct Pipedrive people explicitly marked as non-employees whose lead qualification date follows their signup date. Return only user_id, in ascending order; use the person ID for a Pipedrive-only person.", "generated_sql": "SELECT a.id AS user_id FROM public.accounts AS a ORDER BY a.id ASC LIMIT 10000", "source_legs": [], "actual_row_count": 70, "registered_reference_row_count": 18, "actual_sample": [ { "user_id": 1090, "client": "Fictional Company 2", "client_type": "cfx (corporate)" }, { "user_id": 9871, "client": "Fictional Company 1", "client_type": "cfx (corporate)" } ], "registered_reference_sample": [ { "user_id": 31001 }, { "user_id": 31003 } ] } ], "cause": "Successful source SQL omits the defining user predicates. The marketing query checks only that a Gold user_id exists; it drops ordinary role, marketing opt-in and incomplete registration. The explicitly non-employee qualification query returns every CT account and drops the Pipedrive entity, employee flag and qualification/signup comparison.", "observed_effect": "45 rows instead of 8; 70 rows instead of 18. These are declared frozen-fixture outcomes, not new measurements.", "implication": "First-turn meaning and source obligations are not enforced for these new families. Curated source repairs did not generalize to these combinations." }, { "id": "V18-QF04", "severity": "material", "status": "independently_reproduced_semantic_counterexample", "case": { "case_id": "independent11:verified_after_registration:t1", "registered_outcome_unchanged": "silent_wrong", "is_first_turn": true, "question": "List every distinct CT accounts of any role whose earliest verified broker account date is after their completed registration date. Return only user_id, in ascending order; use the person ID for a Pipedrive-only person.", "generated_sql": "SELECT DISTINCT a.id AS user_id FROM public.accounts AS a JOIN public.clients AS c ON c.user_id = a.id WHERE NOT c.completed_registration_at IS NULL AND EXISTS(SELECT 1 FROM public.broker_accounts AS ba WHERE ba.user_id = a.id AND NOT ba.verified_at IS NULL AND ba.verified_at > c.completed_registration_at) ORDER BY 1 LIMIT 10000", "source_legs": [], "actual_row_count": 26, "registered_reference_row_count": 0, "actual_sample": [ { "user_id": 9871, "client": "Fictional Company 1", "client_type": "cfx (corporate)" }, { "user_id": 11003, "client": "Fictional Company 3", "client_type": "cfx (corporate)" } ], "registered_reference_sample": [] }, "cause": "EXISTS a verification after registration does not prove that the earliest verification was after registration.", "native_independent_reproduction": { "expected_query": "WITH accounts(id) AS (VALUES(900001)), clients(user_id,completed_registration_at) AS (VALUES(900001,TIMESTAMP '2026-01-01')), broker_accounts(user_id,verified_at) AS (VALUES(900001,TIMESTAMP '2025-01-01'),(900001,TIMESTAMP '2027-01-01')) SELECT COUNT(*) FROM accounts a JOIN clients c ON c.user_id=a.id WHERE (SELECT MIN(verified_at) FROM broker_accounts b WHERE b.user_id=a.id)>c.completed_registration_at", "candidate_query": "WITH accounts(id) AS (VALUES(900001)), clients(user_id,completed_registration_at) AS (VALUES(900001,TIMESTAMP '2026-01-01')), broker_accounts(user_id,verified_at) AS (VALUES(900001,TIMESTAMP '2025-01-01'),(900001,TIMESTAMP '2027-01-01')) SELECT COUNT(*) FROM accounts a JOIN clients c ON c.user_id=a.id WHERE EXISTS(SELECT 1 FROM broker_accounts b WHERE b.user_id=a.id AND b.verified_at>c.completed_registration_at)", "expected_rows": [ [ 0 ] ], "candidate_rows": [ [ 1 ] ], "database_writes": 0 }, "observed_effect": "The frozen run returned 26 clients; the reference returned none. The separate two-verification VALUES example confirms the lost earliest-date meaning." }, { "id": "V18-QF05", "severity": "material", "status": "confirmed_from_retained_wrong_successes", "cases": [ { "case_id": "independent11:organization_wallet_share_unknown:first-count", "registered_outcome_unchanged": "silent_wrong", "is_first_turn": true, "question": "How many distinct Pipedrive people at an organization with a known service level but an unknown wallet-share value are there? Return client_count, including zero if none qualify.", "generated_sql": null, "source_legs": [ { "label": "gold", "sql": "SELECT c360.user_id, o.name AS org_name, o.cf_level_of_service, o.cf_wallet_share FROM ct.client_360 AS c360 JOIN pipedrive.persons_enriched AS pe ON pe.ct_id = c360.user_id JOIN pipedrive.organizations AS o ON o.id = pe.org_id WHERE NOT o.cf_level_of_service IS NULL AND o.cf_wallet_share IS NULL GROUP BY c360.user_id, o.name, o.cf_level_of_service, o.cf_wallet_share LIMIT 10001" }, { "label": "ct", "sql": "SELECT a.id AS user_id FROM public.accounts AS a WHERE a.deleted_at IS NULL GROUP BY 1 LIMIT 10001" } ], "actual_row_count": 13, "registered_reference_row_count": 1, "actual_sample": [ { "user_id": 11005, "org_name": "Fictional organization 5", "cf_level_of_service": "managed", "cf_wallet_share": null }, { "user_id": 11013, "org_name": "Fictional organization 5", "cf_level_of_service": "managed", "cf_wallet_share": null } ], "registered_reference_sample": [ { "client_count": 24 } ] }, { "case_id": "independent11:organization_client_type_multiple:first-count", "registered_outcome_unchanged": "silent_wrong", "is_first_turn": true, "question": "How many distinct Pipedrive people at an organization with more than one client-type tag are there? Return client_count, including zero if none qualify.", "generated_sql": null, "source_legs": [ { "label": "gold", "sql": "SELECT c.user_id, COUNT(DISTINCT pe.id) FILTER (WHERE array_length(o.cf_client_type_tags, 1) > 1)::int AS client_count FROM ct.client_360 c LEFT JOIN pipedrive.persons_enriched pe ON pe.ct_id = c.user_id LEFT JOIN pipedrive.organizations o ON o.id = pe.org_id GROUP BY 1 LIMIT 10001" }, { "label": "ct", "sql": "SELECT a.id AS user_id FROM public.accounts AS a WHERE a.deleted_at IS NULL LIMIT 10001" } ], "actual_row_count": 45, "registered_reference_row_count": 1, "actual_sample": [ { "user_id": 11003, "client_count": 1 }, { "user_id": 11004, "client_count": 0 } ], "registered_reference_sample": [ { "client_count": 32 } ] } ], "cause": "A scalar count of Pipedrive people becomes a CT-keyed per-client merge. The first result returns 13 attribute rows instead of client_count=24. The second returns 45 rows with individual counts instead of client_count=32.", "implication": "The final requested grain and the Pipedrive population both change. A successful keyed merge does not prove the user asked for that key space or output shape." }, { "id": "V18-QF06", "severity": "evaluation_validity", "status": "independently_confirmed_fixture_predicate_sensitivity_gap", "ratified_source": { "path": "docs/bi/knowledge/pipedrive-activities.md", "line": 151, "sha256": "c10a801d370f2629465e7b0696d8a506a8a58e2578558820e794ef45d6483016", "predicate": "pd_is_ct_employee IS NOT TRUE" }, "independent_read_only_query": "SELECT count(*), count(*) FILTER(WHERE pd_is_ct_employee IS TRUE), count(*) FILTER(WHERE pd_is_ct_employee IS FALSE), count(*) FILTER(WHERE pd_is_ct_employee IS NULL), count(*) FILTER(WHERE pd_is_ct_employee IS NOT TRUE) FROM pipedrive.persons_enriched", "independent_observed_values": { "total": 64, "true": 0, "false": 52, "null": 12, "after_predicate": 64, "removed": 0 }, "cause": "The fixture has the required column and type, but no positive employee row. It cannot expose omission of this ratified staff-exclusion predicate.", "implication": "E1 requires each ratified predicate to change at least one answer. Full fixture fidelity is not established. Keep paired outcomes as observed data, without an unqualified independent quality certification." } ], "prohibited_mutations_performed": { "application": 0, "tests": 0, "fixture": 0, "corpus": 0, "method": 0, "numeric_scores": 0 }, "only_authored_artifact": "docs/evidence/insights-goal2-2026-09-06/query-v8/final-v18-query-review.json", "verdict": { "unqualified_independent_quality_certification": false, "full_E1_predicate_sensitivity_established": false, "all_first_turn_meaning_failures_fixed": false, "registered_numeric_results_must_remain_unchanged": true, "conclusion": "The frozen run is reproducible observational evidence. It does not establish complete fixture fidelity or general query-meaning correctness. At least two scored-correct first turns fail separate semantic counterexamples. Additional scored-wrong first turns omit explicit predicates, use the wrong temporal quantifier, or return the wrong population and output grain. The final stop boundary remains in force; no V19 or replacement holdout is proposed." } }