{ "before": { "attempts": 250, "outcomes": { "refusal_unverified": 4, "task_error": 161, "correct_completed": 36, "silent_wrong": 31, "comparison_unverified": 18 }, "silent_wrong": 31, "silent_wrong_rate": 0.124, "correct_completed": 36, "answer_precision": 0.5373134328358209, "answer_precision_denominator": 67, "supported_first_turns": 100, "first_turn_execution_errors": 60, "first_turn_error_rate": 0.6, "correct_completed_supported_first_turns": 14, "correct_completed_first_turn_rate": 0.14, "turns_with_successful_parent": 62, "unnecessary_clarifications_with_successful_parent": 0, "unnecessary_clarification_rate": 0.0, "first_turn_unnecessary_clarifications": 0, "conversations": 50, "full_conversation_successes": 1, "full_conversation_success_rate": 0.02, "submitted_jobs": 250, "jobs_reaching_deadline": 0, "unclassified_errors": 21 }, "after": { "attempts": 250, "outcomes": { "refusal_unverified": 24, "task_error": 93, "correct_completed": 57, "unnecessary_clarification": 1, "comparison_unverified": 21, "silent_wrong": 54 }, "silent_wrong": 54, "silent_wrong_rate": 0.216, "correct_completed": 57, "answer_precision": 0.5135135135135135, "answer_precision_denominator": 111, "supported_first_turns": 100, "first_turn_execution_errors": 35, "first_turn_error_rate": 0.35, "correct_completed_supported_first_turns": 18, "correct_completed_first_turn_rate": 0.18, "turns_with_successful_parent": 106, "unnecessary_clarifications_with_successful_parent": 0, "unnecessary_clarification_rate": 0.0, "first_turn_unnecessary_clarifications": 1, "conversations": 50, "full_conversation_successes": 4, "full_conversation_success_rate": 0.08, "submitted_jobs": 250, "jobs_reaching_deadline": 0, "unclassified_errors": 0 }, "bootstrap": { "unit": "paired intent-family cluster", "repeats": 10000, "seed": 2026090611, "interval": "percentile, nearest-rank, two-sided95%", "families": 50, "ci95": { "relative_silent_wrong_reduction": [ -2.3529411764705883, 0.022727272727272728 ], "precision_difference": [ -0.23182397959183676, 0.18543292456335936 ], "candidate_first_turn_error_rate": [ 0.24, 0.46 ], "candidate_first_turn_completion_rate": [ 0.1, 0.27 ], "candidate_full_conversation_success_rate": [ 0.02, 0.16 ] }, "undefined_zero_baseline_wrong_draws": 0, "undefined_precision_draws": 0 }, "relative_silent_wrong_reduction": -0.7419354838709677, "paired_regressions": [ "independent11:quote_only_system_activity:first-count:base:repeat1", "independent11:quote_days_before_activation:first-count:base:repeat1", "independent11:customer_referrals_with_reward_event:t4:base:repeat1", "independent11:organization_wallet_share_unknown:t1:base:repeat1", "independent11:organization_wallet_share_unknown:t3:base:repeat1", "independent11:activity_duration_missing_due_time:t3:base:repeat1", "independent11:deal_stage_timestamp_order:t1:base:repeat1" ], "transitions": [ { "original": "comparison_unverified", "candidate": "comparison_unverified", "count": 12 }, { "original": "comparison_unverified", "candidate": "correct_completed", "count": 4 }, { "original": "comparison_unverified", "candidate": "silent_wrong", "count": 2 }, { "original": "correct_completed", "candidate": "correct_completed", "count": 24 }, { "original": "correct_completed", "candidate": "refusal_unverified", "count": 4 }, { "original": "correct_completed", "candidate": "silent_wrong", "count": 4 }, { "original": "correct_completed", "candidate": "task_error", "count": 3 }, { "original": "correct_completed", "candidate": "unnecessary_clarification", "count": 1 }, { "original": "refusal_unverified", "candidate": "correct_completed", "count": 2 }, { "original": "refusal_unverified", "candidate": "refusal_unverified", "count": 1 }, { "original": "refusal_unverified", "candidate": "silent_wrong", "count": 1 }, { "original": "silent_wrong", "candidate": "comparison_unverified", "count": 6 }, { "original": "silent_wrong", "candidate": "correct_completed", "count": 8 }, { "original": "silent_wrong", "candidate": "refusal_unverified", "count": 4 }, { "original": "silent_wrong", "candidate": "silent_wrong", "count": 9 }, { "original": "silent_wrong", "candidate": "task_error", "count": 4 }, { "original": "task_error", "candidate": "comparison_unverified", "count": 3 }, { "original": "task_error", "candidate": "correct_completed", "count": 19 }, { "original": "task_error", "candidate": "refusal_unverified", "count": 15 }, { "original": "task_error", "candidate": "silent_wrong", "count": 38 }, { "original": "task_error", "candidate": "task_error", "count": 86 } ], "transition_per_case": [ { "attempt_id": "independent11:activity_deadline_before_creation:first-count:base:repeat1", "case_id": "independent11:activity_deadline_before_creation:first-count", "original": "comparison_unverified", "candidate": "comparison_unverified" }, { "attempt_id": "independent11:activity_deadline_before_creation:t1:base:repeat1", "case_id": "independent11:activity_deadline_before_creation:t1", "original": "correct_completed", "candidate": "correct_completed" }, { "attempt_id": "independent11:activity_deadline_before_creation:t2:base:repeat1", "case_id": "independent11:activity_deadline_before_creation:t2", "original": "comparison_unverified", "candidate": "correct_completed" }, { "attempt_id": "independent11:activity_deadline_before_creation:t3:base:repeat1", "case_id": "independent11:activity_deadline_before_creation:t3", "original": "correct_completed", "candidate": "correct_completed" }, { "attempt_id": "independent11:activity_deadline_before_creation:t4:base:repeat1", "case_id": "independent11:activity_deadline_before_creation:t4", "original": "comparison_unverified", "candidate": "correct_completed" }, { "attempt_id": "independent11:activity_duration_missing_due_time:first-count:base:repeat1", "case_id": "independent11:activity_duration_missing_due_time:first-count", "original": "task_error", "candidate": "comparison_unverified" }, { "attempt_id": "independent11:activity_duration_missing_due_time:t1:base:repeat1", "case_id": "independent11:activity_duration_missing_due_time:t1", "original": "correct_completed", "candidate": "correct_completed" }, { "attempt_id": "independent11:activity_duration_missing_due_time:t2:base:repeat1", "case_id": "independent11:activity_duration_missing_due_time:t2", "original": "comparison_unverified", "candidate": "correct_completed" }, { "attempt_id": "independent11:activity_duration_missing_due_time:t3:base:repeat1", "case_id": "independent11:activity_duration_missing_due_time:t3", "original": "correct_completed", "candidate": "silent_wrong" }, { "attempt_id": "independent11:activity_duration_missing_due_time:t4:base:repeat1", "case_id": "independent11:activity_duration_missing_due_time:t4", "original": "comparison_unverified", "candidate": "silent_wrong" }, { "attempt_id": "independent11:all_commission_unknown:first-count:base:repeat1", "case_id": "independent11:all_commission_unknown:first-count", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:all_commission_unknown:t1:base:repeat1", "case_id": "independent11:all_commission_unknown:t1", "original": "task_error", "candidate": "silent_wrong" }, { "attempt_id": "independent11:all_commission_unknown:t2:base:repeat1", "case_id": "independent11:all_commission_unknown:t2", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:all_commission_unknown:t3:base:repeat1", "case_id": "independent11:all_commission_unknown:t3", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:all_commission_unknown:t4:base:repeat1", "case_id": "independent11:all_commission_unknown:t4", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:booked_without_settled_history:first-count:base:repeat1", "case_id": "independent11:booked_without_settled_history:first-count", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:booked_without_settled_history:t1:base:repeat1", "case_id": "independent11:booked_without_settled_history:t1", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:booked_without_settled_history:t2:base:repeat1", "case_id": "independent11:booked_without_settled_history:t2", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:booked_without_settled_history:t3:base:repeat1", "case_id": "independent11:booked_without_settled_history:t3", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:booked_without_settled_history:t4:base:repeat1", "case_id": "independent11:booked_without_settled_history:t4", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:campaign_and_web_referral:first-count:base:repeat1", "case_id": "independent11:campaign_and_web_referral:first-count", "original": "correct_completed", "candidate": "correct_completed" }, { "attempt_id": "independent11:campaign_and_web_referral:t1:base:repeat1", "case_id": "independent11:campaign_and_web_referral:t1", "original": "comparison_unverified", "candidate": "comparison_unverified" }, { "attempt_id": "independent11:campaign_and_web_referral:t2:base:repeat1", "case_id": "independent11:campaign_and_web_referral:t2", "original": "correct_completed", "candidate": "correct_completed" }, { "attempt_id": "independent11:campaign_and_web_referral:t3:base:repeat1", "case_id": "independent11:campaign_and_web_referral:t3", "original": "silent_wrong", "candidate": "comparison_unverified" }, { "attempt_id": "independent11:campaign_and_web_referral:t4:base:repeat1", "case_id": "independent11:campaign_and_web_referral:t4", "original": "correct_completed", "candidate": "correct_completed" }, { "attempt_id": "independent11:corporate_registration_without_trade:first-count:base:repeat1", "case_id": "independent11:corporate_registration_without_trade:first-count", "original": "correct_completed", "candidate": "correct_completed" }, { "attempt_id": "independent11:corporate_registration_without_trade:t1:base:repeat1", "case_id": "independent11:corporate_registration_without_trade:t1", "original": "comparison_unverified", "candidate": "comparison_unverified" }, { "attempt_id": "independent11:corporate_registration_without_trade:t2:base:repeat1", "case_id": "independent11:corporate_registration_without_trade:t2", "original": "correct_completed", "candidate": "correct_completed" }, { "attempt_id": "independent11:corporate_registration_without_trade:t3:base:repeat1", "case_id": "independent11:corporate_registration_without_trade:t3", "original": "silent_wrong", "candidate": "comparison_unverified" }, { "attempt_id": "independent11:corporate_registration_without_trade:t4:base:repeat1", "case_id": "independent11:corporate_registration_without_trade:t4", "original": "correct_completed", "candidate": "correct_completed" }, { "attempt_id": "independent11:customer_referrals_with_reward_event:first-count:base:repeat1", "case_id": "independent11:customer_referrals_with_reward_event:first-count", "original": "correct_completed", "candidate": "unnecessary_clarification" }, { "attempt_id": "independent11:customer_referrals_with_reward_event:t1:base:repeat1", "case_id": "independent11:customer_referrals_with_reward_event:t1", "original": "correct_completed", "candidate": "correct_completed" }, { "attempt_id": "independent11:customer_referrals_with_reward_event:t2:base:repeat1", "case_id": "independent11:customer_referrals_with_reward_event:t2", "original": "correct_completed", "candidate": "correct_completed" }, { "attempt_id": "independent11:customer_referrals_with_reward_event:t3:base:repeat1", "case_id": "independent11:customer_referrals_with_reward_event:t3", "original": "correct_completed", "candidate": "correct_completed" }, { "attempt_id": "independent11:customer_referrals_with_reward_event:t4:base:repeat1", "case_id": "independent11:customer_referrals_with_reward_event:t4", "original": "correct_completed", "candidate": "task_error" }, { "attempt_id": "independent11:deadline_actual_date_disagreement:first-count:base:repeat1", "case_id": "independent11:deadline_actual_date_disagreement:first-count", "original": "task_error", "candidate": "refusal_unverified" }, { "attempt_id": "independent11:deadline_actual_date_disagreement:t1:base:repeat1", "case_id": "independent11:deadline_actual_date_disagreement:t1", "original": "silent_wrong", "candidate": "refusal_unverified" }, { "attempt_id": "independent11:deadline_actual_date_disagreement:t2:base:repeat1", "case_id": "independent11:deadline_actual_date_disagreement:t2", "original": "task_error", "candidate": "refusal_unverified" }, { "attempt_id": "independent11:deadline_actual_date_disagreement:t3:base:repeat1", "case_id": "independent11:deadline_actual_date_disagreement:t3", "original": "task_error", "candidate": "refusal_unverified" }, { "attempt_id": "independent11:deadline_actual_date_disagreement:t4:base:repeat1", "case_id": "independent11:deadline_actual_date_disagreement:t4", "original": "task_error", "candidate": "refusal_unverified" }, { "attempt_id": "independent11:deal_assignment_override:first-count:base:repeat1", "case_id": "independent11:deal_assignment_override:first-count", "original": "comparison_unverified", "candidate": "comparison_unverified" }, { "attempt_id": "independent11:deal_assignment_override:t1:base:repeat1", "case_id": "independent11:deal_assignment_override:t1", "original": "task_error", "candidate": "silent_wrong" }, { "attempt_id": "independent11:deal_assignment_override:t2:base:repeat1", "case_id": "independent11:deal_assignment_override:t2", "original": "task_error", "candidate": "correct_completed" }, { "attempt_id": "independent11:deal_assignment_override:t3:base:repeat1", "case_id": "independent11:deal_assignment_override:t3", "original": "task_error", "candidate": "correct_completed" }, { "attempt_id": "independent11:deal_assignment_override:t4:base:repeat1", "case_id": "independent11:deal_assignment_override:t4", "original": "task_error", "candidate": "correct_completed" }, { "attempt_id": "independent11:deal_person_organization_conflict:first-count:base:repeat1", "case_id": "independent11:deal_person_organization_conflict:first-count", "original": "silent_wrong", "candidate": "correct_completed" }, { "attempt_id": "independent11:deal_person_organization_conflict:t1:base:repeat1", "case_id": "independent11:deal_person_organization_conflict:t1", "original": "task_error", "candidate": "correct_completed" }, { "attempt_id": "independent11:deal_person_organization_conflict:t2:base:repeat1", "case_id": "independent11:deal_person_organization_conflict:t2", "original": "silent_wrong", "candidate": "correct_completed" }, { "attempt_id": "independent11:deal_person_organization_conflict:t3:base:repeat1", "case_id": "independent11:deal_person_organization_conflict:t3", "original": "task_error", "candidate": "correct_completed" }, { "attempt_id": "independent11:deal_person_organization_conflict:t4:base:repeat1", "case_id": "independent11:deal_person_organization_conflict:t4", "original": "task_error", "candidate": "correct_completed" }, { "attempt_id": "independent11:deal_stage_timestamp_order:first-count:base:repeat1", "case_id": "independent11:deal_stage_timestamp_order:first-count", "original": "task_error", "candidate": "correct_completed" }, { "attempt_id": "independent11:deal_stage_timestamp_order:t1:base:repeat1", "case_id": "independent11:deal_stage_timestamp_order:t1", "original": "correct_completed", "candidate": "silent_wrong" }, { "attempt_id": "independent11:deal_stage_timestamp_order:t2:base:repeat1", "case_id": "independent11:deal_stage_timestamp_order:t2", "original": "task_error", "candidate": "correct_completed" }, { "attempt_id": "independent11:deal_stage_timestamp_order:t3:base:repeat1", "case_id": "independent11:deal_stage_timestamp_order:t3", "original": "task_error", "candidate": "silent_wrong" }, { "attempt_id": "independent11:deal_stage_timestamp_order:t4:base:repeat1", "case_id": "independent11:deal_stage_timestamp_order:t4", "original": "task_error", "candidate": "silent_wrong" }, { "attempt_id": "independent11:deleted_booking_live_account:first-count:base:repeat1", "case_id": "independent11:deleted_booking_live_account:first-count", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:deleted_booking_live_account:t1:base:repeat1", "case_id": "independent11:deleted_booking_live_account:t1", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:deleted_booking_live_account:t2:base:repeat1", "case_id": "independent11:deleted_booking_live_account:t2", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:deleted_booking_live_account:t3:base:repeat1", "case_id": "independent11:deleted_booking_live_account:t3", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:deleted_booking_live_account:t4:base:repeat1", "case_id": "independent11:deleted_booking_live_account:t4", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:different_sales_ownership:first-count:base:repeat1", "case_id": "independent11:different_sales_ownership:first-count", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:different_sales_ownership:t1:base:repeat1", "case_id": "independent11:different_sales_ownership:t1", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:different_sales_ownership:t2:base:repeat1", "case_id": "independent11:different_sales_ownership:t2", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:different_sales_ownership:t3:base:repeat1", "case_id": "independent11:different_sales_ownership:t3", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:different_sales_ownership:t4:base:repeat1", "case_id": "independent11:different_sales_ownership:t4", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:first_booked_before_first_broker:first-count:base:repeat1", "case_id": "independent11:first_booked_before_first_broker:first-count", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:first_booked_before_first_broker:t1:base:repeat1", "case_id": "independent11:first_booked_before_first_broker:t1", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:first_booked_before_first_broker:t2:base:repeat1", "case_id": "independent11:first_booked_before_first_broker:t2", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:first_booked_before_first_broker:t3:base:repeat1", "case_id": "independent11:first_booked_before_first_broker:t3", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:first_booked_before_first_broker:t4:base:repeat1", "case_id": "independent11:first_booked_before_first_broker:t4", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:funnel_incomplete_contact:first-count:base:repeat1", "case_id": "independent11:funnel_incomplete_contact:first-count", "original": "silent_wrong", "candidate": "correct_completed" }, { "attempt_id": "independent11:funnel_incomplete_contact:t1:base:repeat1", "case_id": "independent11:funnel_incomplete_contact:t1", "original": "silent_wrong", "candidate": "comparison_unverified" }, { "attempt_id": "independent11:funnel_incomplete_contact:t2:base:repeat1", "case_id": "independent11:funnel_incomplete_contact:t2", "original": "silent_wrong", "candidate": "silent_wrong" }, { "attempt_id": "independent11:funnel_incomplete_contact:t3:base:repeat1", "case_id": "independent11:funnel_incomplete_contact:t3", "original": "comparison_unverified", "candidate": "comparison_unverified" }, { "attempt_id": "independent11:funnel_incomplete_contact:t4:base:repeat1", "case_id": "independent11:funnel_incomplete_contact:t4", "original": "silent_wrong", "candidate": "correct_completed" }, { "attempt_id": "independent11:fx_missing_on_settled:first-count:base:repeat1", "case_id": "independent11:fx_missing_on_settled:first-count", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:fx_missing_on_settled:t1:base:repeat1", "case_id": "independent11:fx_missing_on_settled:t1", "original": "task_error", "candidate": "silent_wrong" }, { "attempt_id": "independent11:fx_missing_on_settled:t2:base:repeat1", "case_id": "independent11:fx_missing_on_settled:t2", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:fx_missing_on_settled:t3:base:repeat1", "case_id": "independent11:fx_missing_on_settled:t3", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:fx_missing_on_settled:t4:base:repeat1", "case_id": "independent11:fx_missing_on_settled:t4", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:invoice_before_settlement:first-count:base:repeat1", "case_id": "independent11:invoice_before_settlement:first-count", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:invoice_before_settlement:t1:base:repeat1", "case_id": "independent11:invoice_before_settlement:t1", "original": "task_error", "candidate": "silent_wrong" }, { "attempt_id": "independent11:invoice_before_settlement:t2:base:repeat1", "case_id": "independent11:invoice_before_settlement:t2", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:invoice_before_settlement:t3:base:repeat1", "case_id": "independent11:invoice_before_settlement:t3", "original": "task_error", "candidate": "silent_wrong" }, { "attempt_id": "independent11:invoice_before_settlement:t4:base:repeat1", "case_id": "independent11:invoice_before_settlement:t4", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:known_external_person_qualification:first-count:base:repeat1", "case_id": "independent11:known_external_person_qualification:first-count", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:known_external_person_qualification:t1:base:repeat1", "case_id": "independent11:known_external_person_qualification:t1", "original": "task_error", "candidate": "silent_wrong" }, { "attempt_id": "independent11:known_external_person_qualification:t2:base:repeat1", "case_id": "independent11:known_external_person_qualification:t2", "original": "task_error", "candidate": "refusal_unverified" }, { "attempt_id": "independent11:known_external_person_qualification:t3:base:repeat1", "case_id": "independent11:known_external_person_qualification:t3", "original": "silent_wrong", "candidate": "silent_wrong" }, { "attempt_id": "independent11:known_external_person_qualification:t4:base:repeat1", "case_id": "independent11:known_external_person_qualification:t4", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:known_zero_booked_commission:first-count:base:repeat1", "case_id": "independent11:known_zero_booked_commission:first-count", "original": "silent_wrong", "candidate": "refusal_unverified" }, { "attempt_id": "independent11:known_zero_booked_commission:t1:base:repeat1", "case_id": "independent11:known_zero_booked_commission:t1", "original": "comparison_unverified", "candidate": "comparison_unverified" }, { "attempt_id": "independent11:known_zero_booked_commission:t2:base:repeat1", "case_id": "independent11:known_zero_booked_commission:t2", "original": "correct_completed", "candidate": "refusal_unverified" }, { "attempt_id": "independent11:known_zero_booked_commission:t3:base:repeat1", "case_id": "independent11:known_zero_booked_commission:t3", "original": "comparison_unverified", "candidate": "comparison_unverified" }, { "attempt_id": "independent11:known_zero_booked_commission:t4:base:repeat1", "case_id": "independent11:known_zero_booked_commission:t4", "original": "correct_completed", "candidate": "correct_completed" }, { "attempt_id": "independent11:london_signup_month_edge:first-count:base:repeat1", "case_id": "independent11:london_signup_month_edge:first-count", "original": "task_error", "candidate": "refusal_unverified" }, { "attempt_id": "independent11:london_signup_month_edge:t1:base:repeat1", "case_id": "independent11:london_signup_month_edge:t1", "original": "task_error", "candidate": "refusal_unverified" }, { "attempt_id": "independent11:london_signup_month_edge:t2:base:repeat1", "case_id": "independent11:london_signup_month_edge:t2", "original": "task_error", "candidate": "refusal_unverified" }, { "attempt_id": "independent11:london_signup_month_edge:t3:base:repeat1", "case_id": "independent11:london_signup_month_edge:t3", "original": "silent_wrong", "candidate": "refusal_unverified" }, { "attempt_id": "independent11:london_signup_month_edge:t4:base:repeat1", "case_id": "independent11:london_signup_month_edge:t4", "original": "task_error", "candidate": "refusal_unverified" }, { "attempt_id": "independent11:marketing_registration_gap:first-count:base:repeat1", "case_id": "independent11:marketing_registration_gap:first-count", "original": "refusal_unverified", "candidate": "refusal_unverified" }, { "attempt_id": "independent11:marketing_registration_gap:t1:base:repeat1", "case_id": "independent11:marketing_registration_gap:t1", "original": "task_error", "candidate": "silent_wrong" }, { "attempt_id": "independent11:marketing_registration_gap:t2:base:repeat1", "case_id": "independent11:marketing_registration_gap:t2", "original": "task_error", "candidate": "silent_wrong" }, { "attempt_id": "independent11:marketing_registration_gap:t3:base:repeat1", "case_id": "independent11:marketing_registration_gap:t3", "original": "task_error", "candidate": "silent_wrong" }, { "attempt_id": "independent11:marketing_registration_gap:t4:base:repeat1", "case_id": "independent11:marketing_registration_gap:t4", "original": "task_error", "candidate": "silent_wrong" }, { "attempt_id": "independent11:multi_broker_activation:first-count:base:repeat1", "case_id": "independent11:multi_broker_activation:first-count", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:multi_broker_activation:t1:base:repeat1", "case_id": "independent11:multi_broker_activation:t1", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:multi_broker_activation:t2:base:repeat1", "case_id": "independent11:multi_broker_activation:t2", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:multi_broker_activation:t3:base:repeat1", "case_id": "independent11:multi_broker_activation:t3", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:multi_broker_activation:t4:base:repeat1", "case_id": "independent11:multi_broker_activation:t4", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:negative_realized_commission:first-count:base:repeat1", "case_id": "independent11:negative_realized_commission:first-count", "original": "correct_completed", "candidate": "refusal_unverified" }, { "attempt_id": "independent11:negative_realized_commission:t1:base:repeat1", "case_id": "independent11:negative_realized_commission:t1", "original": "comparison_unverified", "candidate": "comparison_unverified" }, { "attempt_id": "independent11:negative_realized_commission:t2:base:repeat1", "case_id": "independent11:negative_realized_commission:t2", "original": "correct_completed", "candidate": "refusal_unverified" }, { "attempt_id": "independent11:negative_realized_commission:t3:base:repeat1", "case_id": "independent11:negative_realized_commission:t3", "original": "comparison_unverified", "candidate": "comparison_unverified" }, { "attempt_id": "independent11:negative_realized_commission:t4:base:repeat1", "case_id": "independent11:negative_realized_commission:t4", "original": "correct_completed", "candidate": "correct_completed" }, { "attempt_id": "independent11:note_author_deal_owner_difference:first-count:base:repeat1", "case_id": "independent11:note_author_deal_owner_difference:first-count", "original": "task_error", "candidate": "correct_completed" }, { "attempt_id": "independent11:note_author_deal_owner_difference:t1:base:repeat1", "case_id": "independent11:note_author_deal_owner_difference:t1", "original": "task_error", "candidate": "silent_wrong" }, { "attempt_id": "independent11:note_author_deal_owner_difference:t2:base:repeat1", "case_id": "independent11:note_author_deal_owner_difference:t2", "original": "task_error", "candidate": "silent_wrong" }, { "attempt_id": "independent11:note_author_deal_owner_difference:t3:base:repeat1", "case_id": "independent11:note_author_deal_owner_difference:t3", "original": "task_error", "candidate": "silent_wrong" }, { "attempt_id": "independent11:note_author_deal_owner_difference:t4:base:repeat1", "case_id": "independent11:note_author_deal_owner_difference:t4", "original": "task_error", "candidate": "silent_wrong" }, { "attempt_id": "independent11:notified_without_verification:first-count:base:repeat1", "case_id": "independent11:notified_without_verification:first-count", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:notified_without_verification:t1:base:repeat1", "case_id": "independent11:notified_without_verification:t1", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:notified_without_verification:t2:base:repeat1", "case_id": "independent11:notified_without_verification:t2", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:notified_without_verification:t3:base:repeat1", "case_id": "independent11:notified_without_verification:t3", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:notified_without_verification:t4:base:repeat1", "case_id": "independent11:notified_without_verification:t4", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:org_contact_without_website:first-count:base:repeat1", "case_id": "independent11:org_contact_without_website:first-count", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:org_contact_without_website:t1:base:repeat1", "case_id": "independent11:org_contact_without_website:t1", "original": "silent_wrong", "candidate": "silent_wrong" }, { "attempt_id": "independent11:org_contact_without_website:t2:base:repeat1", "case_id": "independent11:org_contact_without_website:t2", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:org_contact_without_website:t3:base:repeat1", "case_id": "independent11:org_contact_without_website:t3", "original": "silent_wrong", "candidate": "silent_wrong" }, { "attempt_id": "independent11:org_contact_without_website:t4:base:repeat1", "case_id": "independent11:org_contact_without_website:t4", "original": "silent_wrong", "candidate": "task_error" }, { "attempt_id": "independent11:organization_client_type_multiple:first-count:base:repeat1", "case_id": "independent11:organization_client_type_multiple:first-count", "original": "task_error", "candidate": "silent_wrong" }, { "attempt_id": "independent11:organization_client_type_multiple:t1:base:repeat1", "case_id": "independent11:organization_client_type_multiple:t1", "original": "task_error", "candidate": "silent_wrong" }, { "attempt_id": "independent11:organization_client_type_multiple:t2:base:repeat1", "case_id": "independent11:organization_client_type_multiple:t2", "original": "task_error", "candidate": "silent_wrong" }, { "attempt_id": "independent11:organization_client_type_multiple:t3:base:repeat1", "case_id": "independent11:organization_client_type_multiple:t3", "original": "task_error", "candidate": "comparison_unverified" }, { "attempt_id": "independent11:organization_client_type_multiple:t4:base:repeat1", "case_id": "independent11:organization_client_type_multiple:t4", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:organization_employee_data_gap:first-count:base:repeat1", "case_id": "independent11:organization_employee_data_gap:first-count", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:organization_employee_data_gap:t1:base:repeat1", "case_id": "independent11:organization_employee_data_gap:t1", "original": "task_error", "candidate": "silent_wrong" }, { "attempt_id": "independent11:organization_employee_data_gap:t2:base:repeat1", "case_id": "independent11:organization_employee_data_gap:t2", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:organization_employee_data_gap:t3:base:repeat1", "case_id": "independent11:organization_employee_data_gap:t3", "original": "silent_wrong", "candidate": "silent_wrong" }, { "attempt_id": "independent11:organization_employee_data_gap:t4:base:repeat1", "case_id": "independent11:organization_employee_data_gap:t4", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:organization_erp_multiple:first-count:base:repeat1", "case_id": "independent11:organization_erp_multiple:first-count", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:organization_erp_multiple:t1:base:repeat1", "case_id": "independent11:organization_erp_multiple:t1", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:organization_erp_multiple:t2:base:repeat1", "case_id": "independent11:organization_erp_multiple:t2", "original": "silent_wrong", "candidate": "task_error" }, { "attempt_id": "independent11:organization_erp_multiple:t3:base:repeat1", "case_id": "independent11:organization_erp_multiple:t3", "original": "task_error", "candidate": "silent_wrong" }, { "attempt_id": "independent11:organization_erp_multiple:t4:base:repeat1", "case_id": "independent11:organization_erp_multiple:t4", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:organization_hedging_multiple:first-count:base:repeat1", "case_id": "independent11:organization_hedging_multiple:first-count", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:organization_hedging_multiple:t1:base:repeat1", "case_id": "independent11:organization_hedging_multiple:t1", "original": "task_error", "candidate": "silent_wrong" }, { "attempt_id": "independent11:organization_hedging_multiple:t2:base:repeat1", "case_id": "independent11:organization_hedging_multiple:t2", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:organization_hedging_multiple:t3:base:repeat1", "case_id": "independent11:organization_hedging_multiple:t3", "original": "task_error", "candidate": "silent_wrong" }, { "attempt_id": "independent11:organization_hedging_multiple:t4:base:repeat1", "case_id": "independent11:organization_hedging_multiple:t4", "original": "task_error", "candidate": "silent_wrong" }, { "attempt_id": "independent11:organization_introducer_link:first-count:base:repeat1", "case_id": "independent11:organization_introducer_link:first-count", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:organization_introducer_link:t1:base:repeat1", "case_id": "independent11:organization_introducer_link:t1", "original": "task_error", "candidate": "silent_wrong" }, { "attempt_id": "independent11:organization_introducer_link:t2:base:repeat1", "case_id": "independent11:organization_introducer_link:t2", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:organization_introducer_link:t3:base:repeat1", "case_id": "independent11:organization_introducer_link:t3", "original": "task_error", "candidate": "silent_wrong" }, { "attempt_id": "independent11:organization_introducer_link:t4:base:repeat1", "case_id": "independent11:organization_introducer_link:t4", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:organization_wallet_share_unknown:first-count:base:repeat1", "case_id": "independent11:organization_wallet_share_unknown:first-count", "original": "task_error", "candidate": "silent_wrong" }, { "attempt_id": "independent11:organization_wallet_share_unknown:t1:base:repeat1", "case_id": "independent11:organization_wallet_share_unknown:t1", "original": "correct_completed", "candidate": "silent_wrong" }, { "attempt_id": "independent11:organization_wallet_share_unknown:t2:base:repeat1", "case_id": "independent11:organization_wallet_share_unknown:t2", "original": "task_error", "candidate": "silent_wrong" }, { "attempt_id": "independent11:organization_wallet_share_unknown:t3:base:repeat1", "case_id": "independent11:organization_wallet_share_unknown:t3", "original": "correct_completed", "candidate": "silent_wrong" }, { "attempt_id": "independent11:organization_wallet_share_unknown:t4:base:repeat1", "case_id": "independent11:organization_wallet_share_unknown:t4", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:outdated_still_activated:first-count:base:repeat1", "case_id": "independent11:outdated_still_activated:first-count", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:outdated_still_activated:t1:base:repeat1", "case_id": "independent11:outdated_still_activated:t1", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:outdated_still_activated:t2:base:repeat1", "case_id": "independent11:outdated_still_activated:t2", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:outdated_still_activated:t3:base:repeat1", "case_id": "independent11:outdated_still_activated:t3", "original": "task_error", "candidate": "silent_wrong" }, { "attempt_id": "independent11:outdated_still_activated:t4:base:repeat1", "case_id": "independent11:outdated_still_activated:t4", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:overdue_expected_close:first-count:base:repeat1", "case_id": "independent11:overdue_expected_close:first-count", "original": "task_error", "candidate": "correct_completed" }, { "attempt_id": "independent11:overdue_expected_close:t1:base:repeat1", "case_id": "independent11:overdue_expected_close:t1", "original": "correct_completed", "candidate": "correct_completed" }, { "attempt_id": "independent11:overdue_expected_close:t2:base:repeat1", "case_id": "independent11:overdue_expected_close:t2", "original": "task_error", "candidate": "correct_completed" }, { "attempt_id": "independent11:overdue_expected_close:t3:base:repeat1", "case_id": "independent11:overdue_expected_close:t3", "original": "correct_completed", "candidate": "correct_completed" }, { "attempt_id": "independent11:overdue_expected_close:t4:base:repeat1", "case_id": "independent11:overdue_expected_close:t4", "original": "task_error", "candidate": "correct_completed" }, { "attempt_id": "independent11:partner_paid_and_unpaid_statements:first-count:base:repeat1", "case_id": "independent11:partner_paid_and_unpaid_statements:first-count", "original": "silent_wrong", "candidate": "correct_completed" }, { "attempt_id": "independent11:partner_paid_and_unpaid_statements:t1:base:repeat1", "case_id": "independent11:partner_paid_and_unpaid_statements:t1", "original": "comparison_unverified", "candidate": "correct_completed" }, { "attempt_id": "independent11:partner_paid_and_unpaid_statements:t2:base:repeat1", "case_id": "independent11:partner_paid_and_unpaid_statements:t2", "original": "correct_completed", "candidate": "correct_completed" }, { "attempt_id": "independent11:partner_paid_and_unpaid_statements:t3:base:repeat1", "case_id": "independent11:partner_paid_and_unpaid_statements:t3", "original": "comparison_unverified", "candidate": "comparison_unverified" }, { "attempt_id": "independent11:partner_paid_and_unpaid_statements:t4:base:repeat1", "case_id": "independent11:partner_paid_and_unpaid_statements:t4", "original": "correct_completed", "candidate": "correct_completed" }, { "attempt_id": "independent11:person_communication_preferences:first-count:base:repeat1", "case_id": "independent11:person_communication_preferences:first-count", "original": "silent_wrong", "candidate": "comparison_unverified" }, { "attempt_id": "independent11:person_communication_preferences:t1:base:repeat1", "case_id": "independent11:person_communication_preferences:t1", "original": "refusal_unverified", "candidate": "silent_wrong" }, { "attempt_id": "independent11:person_communication_preferences:t2:base:repeat1", "case_id": "independent11:person_communication_preferences:t2", "original": "task_error", "candidate": "correct_completed" }, { "attempt_id": "independent11:person_communication_preferences:t3:base:repeat1", "case_id": "independent11:person_communication_preferences:t3", "original": "refusal_unverified", "candidate": "correct_completed" }, { "attempt_id": "independent11:person_communication_preferences:t4:base:repeat1", "case_id": "independent11:person_communication_preferences:t4", "original": "refusal_unverified", "candidate": "correct_completed" }, { "attempt_id": "independent11:person_multi_frequency_declarations:first-count:base:repeat1", "case_id": "independent11:person_multi_frequency_declarations:first-count", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:person_multi_frequency_declarations:t1:base:repeat1", "case_id": "independent11:person_multi_frequency_declarations:t1", "original": "task_error", "candidate": "refusal_unverified" }, { "attempt_id": "independent11:person_multi_frequency_declarations:t2:base:repeat1", "case_id": "independent11:person_multi_frequency_declarations:t2", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:person_multi_frequency_declarations:t3:base:repeat1", "case_id": "independent11:person_multi_frequency_declarations:t3", "original": "task_error", "candidate": "silent_wrong" }, { "attempt_id": "independent11:person_multi_frequency_declarations:t4:base:repeat1", "case_id": "independent11:person_multi_frequency_declarations:t4", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:person_note_pinned_exclusively:first-count:base:repeat1", "case_id": "independent11:person_note_pinned_exclusively:first-count", "original": "task_error", "candidate": "comparison_unverified" }, { "attempt_id": "independent11:person_note_pinned_exclusively:t1:base:repeat1", "case_id": "independent11:person_note_pinned_exclusively:t1", "original": "task_error", "candidate": "silent_wrong" }, { "attempt_id": "independent11:person_note_pinned_exclusively:t2:base:repeat1", "case_id": "independent11:person_note_pinned_exclusively:t2", "original": "task_error", "candidate": "correct_completed" }, { "attempt_id": "independent11:person_note_pinned_exclusively:t3:base:repeat1", "case_id": "independent11:person_note_pinned_exclusively:t3", "original": "task_error", "candidate": "silent_wrong" }, { "attempt_id": "independent11:person_note_pinned_exclusively:t4:base:repeat1", "case_id": "independent11:person_note_pinned_exclusively:t4", "original": "task_error", "candidate": "silent_wrong" }, { "attempt_id": "independent11:person_organization_owner_gap:first-count:base:repeat1", "case_id": "independent11:person_organization_owner_gap:first-count", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:person_organization_owner_gap:t1:base:repeat1", "case_id": "independent11:person_organization_owner_gap:t1", "original": "silent_wrong", "candidate": "silent_wrong" }, { "attempt_id": "independent11:person_organization_owner_gap:t2:base:repeat1", "case_id": "independent11:person_organization_owner_gap:t2", "original": "task_error", "candidate": "refusal_unverified" }, { "attempt_id": "independent11:person_organization_owner_gap:t3:base:repeat1", "case_id": "independent11:person_organization_owner_gap:t3", "original": "task_error", "candidate": "silent_wrong" }, { "attempt_id": "independent11:person_organization_owner_gap:t4:base:repeat1", "case_id": "independent11:person_organization_owner_gap:t4", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:prebillable_eligible_stats:first-count:base:repeat1", "case_id": "independent11:prebillable_eligible_stats:first-count", "original": "task_error", "candidate": "refusal_unverified" }, { "attempt_id": "independent11:prebillable_eligible_stats:t1:base:repeat1", "case_id": "independent11:prebillable_eligible_stats:t1", "original": "silent_wrong", "candidate": "refusal_unverified" }, { "attempt_id": "independent11:prebillable_eligible_stats:t2:base:repeat1", "case_id": "independent11:prebillable_eligible_stats:t2", "original": "task_error", "candidate": "refusal_unverified" }, { "attempt_id": "independent11:prebillable_eligible_stats:t3:base:repeat1", "case_id": "independent11:prebillable_eligible_stats:t3", "original": "task_error", "candidate": "refusal_unverified" }, { "attempt_id": "independent11:prebillable_eligible_stats:t4:base:repeat1", "case_id": "independent11:prebillable_eligible_stats:t4", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:private_declared_currency_overlap:first-count:base:repeat1", "case_id": "independent11:private_declared_currency_overlap:first-count", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:private_declared_currency_overlap:t1:base:repeat1", "case_id": "independent11:private_declared_currency_overlap:t1", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:private_declared_currency_overlap:t2:base:repeat1", "case_id": "independent11:private_declared_currency_overlap:t2", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:private_declared_currency_overlap:t3:base:repeat1", "case_id": "independent11:private_declared_currency_overlap:t3", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:private_declared_currency_overlap:t4:base:repeat1", "case_id": "independent11:private_declared_currency_overlap:t4", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:professional_zero_rate_referrals:first-count:base:repeat1", "case_id": "independent11:professional_zero_rate_referrals:first-count", "original": "silent_wrong", "candidate": "correct_completed" }, { "attempt_id": "independent11:professional_zero_rate_referrals:t1:base:repeat1", "case_id": "independent11:professional_zero_rate_referrals:t1", "original": "silent_wrong", "candidate": "comparison_unverified" }, { "attempt_id": "independent11:professional_zero_rate_referrals:t2:base:repeat1", "case_id": "independent11:professional_zero_rate_referrals:t2", "original": "silent_wrong", "candidate": "correct_completed" }, { "attempt_id": "independent11:professional_zero_rate_referrals:t3:base:repeat1", "case_id": "independent11:professional_zero_rate_referrals:t3", "original": "silent_wrong", "candidate": "comparison_unverified" }, { "attempt_id": "independent11:professional_zero_rate_referrals:t4:base:repeat1", "case_id": "independent11:professional_zero_rate_referrals:t4", "original": "silent_wrong", "candidate": "correct_completed" }, { "attempt_id": "independent11:qualified_without_deal_stage:first-count:base:repeat1", "case_id": "independent11:qualified_without_deal_stage:first-count", "original": "task_error", "candidate": "correct_completed" }, { "attempt_id": "independent11:qualified_without_deal_stage:t1:base:repeat1", "case_id": "independent11:qualified_without_deal_stage:t1", "original": "task_error", "candidate": "correct_completed" }, { "attempt_id": "independent11:qualified_without_deal_stage:t2:base:repeat1", "case_id": "independent11:qualified_without_deal_stage:t2", "original": "task_error", "candidate": "correct_completed" }, { "attempt_id": "independent11:qualified_without_deal_stage:t3:base:repeat1", "case_id": "independent11:qualified_without_deal_stage:t3", "original": "task_error", "candidate": "correct_completed" }, { "attempt_id": "independent11:qualified_without_deal_stage:t4:base:repeat1", "case_id": "independent11:qualified_without_deal_stage:t4", "original": "task_error", "candidate": "correct_completed" }, { "attempt_id": "independent11:quote_conversion_valid_link:first-count:base:repeat1", "case_id": "independent11:quote_conversion_valid_link:first-count", "original": "silent_wrong", "candidate": "task_error" }, { "attempt_id": "independent11:quote_conversion_valid_link:t1:base:repeat1", "case_id": "independent11:quote_conversion_valid_link:t1", "original": "silent_wrong", "candidate": "silent_wrong" }, { "attempt_id": "independent11:quote_conversion_valid_link:t2:base:repeat1", "case_id": "independent11:quote_conversion_valid_link:t2", "original": "task_error", "candidate": "silent_wrong" }, { "attempt_id": "independent11:quote_conversion_valid_link:t3:base:repeat1", "case_id": "independent11:quote_conversion_valid_link:t3", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:quote_conversion_valid_link:t4:base:repeat1", "case_id": "independent11:quote_conversion_valid_link:t4", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:quote_days_before_activation:first-count:base:repeat1", "case_id": "independent11:quote_days_before_activation:first-count", "original": "correct_completed", "candidate": "task_error" }, { "attempt_id": "independent11:quote_days_before_activation:t1:base:repeat1", "case_id": "independent11:quote_days_before_activation:t1", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:quote_days_before_activation:t2:base:repeat1", "case_id": "independent11:quote_days_before_activation:t2", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:quote_days_before_activation:t3:base:repeat1", "case_id": "independent11:quote_days_before_activation:t3", "original": "task_error", "candidate": "refusal_unverified" }, { "attempt_id": "independent11:quote_days_before_activation:t4:base:repeat1", "case_id": "independent11:quote_days_before_activation:t4", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:quote_only_system_activity:first-count:base:repeat1", "case_id": "independent11:quote_only_system_activity:first-count", "original": "correct_completed", "candidate": "task_error" }, { "attempt_id": "independent11:quote_only_system_activity:t1:base:repeat1", "case_id": "independent11:quote_only_system_activity:t1", "original": "task_error", "candidate": "silent_wrong" }, { "attempt_id": "independent11:quote_only_system_activity:t2:base:repeat1", "case_id": "independent11:quote_only_system_activity:t2", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:quote_only_system_activity:t3:base:repeat1", "case_id": "independent11:quote_only_system_activity:t3", "original": "silent_wrong", "candidate": "silent_wrong" }, { "attempt_id": "independent11:quote_only_system_activity:t4:base:repeat1", "case_id": "independent11:quote_only_system_activity:t4", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:trade_sell_not_declared:first-count:base:repeat1", "case_id": "independent11:trade_sell_not_declared:first-count", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:trade_sell_not_declared:t1:base:repeat1", "case_id": "independent11:trade_sell_not_declared:t1", "original": "task_error", "candidate": "silent_wrong" }, { "attempt_id": "independent11:trade_sell_not_declared:t2:base:repeat1", "case_id": "independent11:trade_sell_not_declared:t2", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:trade_sell_not_declared:t3:base:repeat1", "case_id": "independent11:trade_sell_not_declared:t3", "original": "task_error", "candidate": "silent_wrong" }, { "attempt_id": "independent11:trade_sell_not_declared:t4:base:repeat1", "case_id": "independent11:trade_sell_not_declared:t4", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:unknown_sti_with_broker:first-count:base:repeat1", "case_id": "independent11:unknown_sti_with_broker:first-count", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:unknown_sti_with_broker:t1:base:repeat1", "case_id": "independent11:unknown_sti_with_broker:t1", "original": "silent_wrong", "candidate": "task_error" }, { "attempt_id": "independent11:unknown_sti_with_broker:t2:base:repeat1", "case_id": "independent11:unknown_sti_with_broker:t2", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:unknown_sti_with_broker:t3:base:repeat1", "case_id": "independent11:unknown_sti_with_broker:t3", "original": "silent_wrong", "candidate": "silent_wrong" }, { "attempt_id": "independent11:unknown_sti_with_broker:t4:base:repeat1", "case_id": "independent11:unknown_sti_with_broker:t4", "original": "task_error", "candidate": "task_error" }, { "attempt_id": "independent11:verified_after_registration:first-count:base:repeat1", "case_id": "independent11:verified_after_registration:first-count", "original": "correct_completed", "candidate": "refusal_unverified" }, { "attempt_id": "independent11:verified_after_registration:t1:base:repeat1", "case_id": "independent11:verified_after_registration:t1", "original": "comparison_unverified", "candidate": "silent_wrong" }, { "attempt_id": "independent11:verified_after_registration:t2:base:repeat1", "case_id": "independent11:verified_after_registration:t2", "original": "correct_completed", "candidate": "correct_completed" }, { "attempt_id": "independent11:verified_after_registration:t3:base:repeat1", "case_id": "independent11:verified_after_registration:t3", "original": "correct_completed", "candidate": "correct_completed" }, { "attempt_id": "independent11:verified_after_registration:t4:base:repeat1", "case_id": "independent11:verified_after_registration:t4", "original": "correct_completed", "candidate": "correct_completed" }, { "attempt_id": "independent11:verified_private_rejected_registration:first-count:base:repeat1", "case_id": "independent11:verified_private_rejected_registration:first-count", "original": "correct_completed", "candidate": "correct_completed" }, { "attempt_id": "independent11:verified_private_rejected_registration:t1:base:repeat1", "case_id": "independent11:verified_private_rejected_registration:t1", "original": "comparison_unverified", "candidate": "comparison_unverified" }, { "attempt_id": "independent11:verified_private_rejected_registration:t2:base:repeat1", "case_id": "independent11:verified_private_rejected_registration:t2", "original": "correct_completed", "candidate": "correct_completed" }, { "attempt_id": "independent11:verified_private_rejected_registration:t3:base:repeat1", "case_id": "independent11:verified_private_rejected_registration:t3", "original": "comparison_unverified", "candidate": "comparison_unverified" }, { "attempt_id": "independent11:verified_private_rejected_registration:t4:base:repeat1", "case_id": "independent11:verified_private_rejected_registration:t4", "original": "correct_completed", "candidate": "correct_completed" } ], "registered_denominator": { "distinct_holdout_question_texts": 250, "holdout_intent_families": 50, "holdout_conversations": 50, "attempts_by_split": { "locked_holdout": 250 }, "cross_family_duplicate_texts": 0 } }