{ "schema": "axonllm.routing-evidence/v2", "evaluation_status": "model-reviewed synthetic benchmark", "generated_at": "2026-10-02T19:24:57.019066+00:00", "corpus_manifest_sha256": "ec68993165e9cbf24e63a3e09d561e6162edc784885acd9ccb9f22479044f53b", "artifact_sha256": { "primary_report": "4c1158c8e5d435f8bdb7b23a09e3f4c39c08924c36749661428d1f4e978e9d73", "analysis_code": "02b7cb4d5b460aac99eb63ab104a70a13fa0f4365e0d8fc13f59ae92b2c50df4", "protocol": "852753684199ad5fb8a5b713845079d08cc554601d7764eac1b1bf31ca8fe4b4", "repeatability_report": "d4b017ebfd964279013d1547e39616511b161e0db0354094adb1e6d9e46b85fe" }, "methodology": { "unique_cases": 960, "development_cases": 240, "families": 240, "domains": 10, "variants_per_family": 4, "language": "English", "repetitions": 1, "concurrency": 1, "generator": "gpt-4.1-2025-04-14", "reviewer": "claude-sonnet-4-6", "bootstrap": { "method": "class-stratified paired scenario-family percentile bootstrap", "resamples": 10000, "seed": 775, "families": 240, "cases": 960, "familywise_method": "Bonferroni across all pairwise accuracy comparisons" }, "scope": "short-prompt task routing; downstream answer quality excluded", "hardware": "Apple M4 Pro, 48 GiB RAM; MPS for local models", "hardware_description_source": "operator-supplied description; not inferred from model scores", "limitations": [ "Synthetic, model-reviewed labels; no human annotation claim.", "Variants are correlated; intervals resample whole scenario families.", "Consensus filtering and generator/reviewer model-family bias remain.", "No production traffic, long-context or multilingual generalization claim.", "Laya float32 and Strands bfloat16; Strands reference causal-convolution kernel." ] }, "strategies": [ { "strategy": "axon-heuristic", "requests": 960, "correct": 478, "accuracy": 0.4979166666666667, "macro_f1": 0.507053226202057, "errors": 0, "cache_hits": 0, "latency_ms": { "mean": 0.47354699375000003, "p50": 0.338208, "p95": 1.3954583499999997, "p99": 3.1101100600000002 }, "latency_scope": "successful requests only", "all_attempts_latency_ms": { "p50": 0.338208, "p95": 1.3954583499999997 }, "llm_calls": 0, "llm_call_rate": 0.0, "input_tokens": 0, "output_tokens": 0, "router_cost_usd": 0.0, "known_router_cost_usd": 0.0, "cost_coverage": 1.0, "cost_scope": "API charges only; local hardware and electricity excluded", "hardware_electricity_cost_usd": null, "hardware_electricity_cost_status": "unmeasured", "router_cost_per_1k_requests_usd": 0.0, "confusion_matrix": { "coding": { "coding": 102, "general": 48, "reasoning": 6, "summarization": 4 }, "reasoning": { "coding": 2, "general": 122, "reasoning": 35, "summarization": 1 }, "creative_writing": { "coding": 6, "creative_writing": 58, "general": 93, "reasoning": 1, "summarization": 2 }, "summarization": { "coding": 1, "creative_writing": 5, "general": 53, "summarization": 101 }, "math": { "coding": 2, "general": 123, "math": 28, "reasoning": 1, "summarization": 6 }, "general": { "coding": 3, "general": 154, "math": 1, "reasoning": 1, "summarization": 1 } }, "by_task": { "coding": { "requests": 160, "correct": 102, "accuracy": 0.6375 }, "reasoning": { "requests": 160, "correct": 35, "accuracy": 0.21875 }, "creative_writing": { "requests": 160, "correct": 58, "accuracy": 0.3625 }, "summarization": { "requests": 160, "correct": 101, "accuracy": 0.63125 }, "math": { "requests": 160, "correct": 28, "accuracy": 0.175 }, "general": { "requests": 160, "correct": 154, "accuracy": 0.9625 } }, "by_tag": { "arts_and_media": { "requests": 96, "accuracy": 0.59375 }, "education_and_history": { "requests": 96, "accuracy": 0.5104166666666666 }, "english": { "requests": 960, "accuracy": 0.4979166666666667 }, "explicit": { "requests": 240, "accuracy": 0.6083333333333333 }, "health_and_wellbeing": { "requests": 96, "accuracy": 0.5416666666666666 }, "home_and_travel": { "requests": 96, "accuracy": 0.4583333333333333 }, "implicit": { "requests": 240, "accuracy": 0.32916666666666666 }, "keyword_collision": { "requests": 240, "accuracy": 0.5166666666666667 }, "public_services": { "requests": 96, "accuracy": 0.59375 }, "quoted_distraction": { "requests": 240, "accuracy": 0.5375 }, "retail_and_support": { "requests": 96, "accuracy": 0.40625 }, "science_and_environment": { "requests": 96, "accuracy": 0.5 }, "software_and_data": { "requests": 96, "accuracy": 0.4479166666666667 }, "sports_and_leisure": { "requests": 96, "accuracy": 0.4583333333333333 }, "workplace_and_business": { "requests": 96, "accuracy": 0.46875 } }, "wrong_cases": [ { "id": "19c9492dc7a815f38232", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "explicit" ], "error": null }, { "id": "67ed7847bffa3a948ba8", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "public_services" ], "error": null }, { "id": "efa47373a3b287b3172f", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "53ab818b9bbe70e5041a", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "science_and_environment" ], "error": null }, { "id": "5eca30c84bed105d5917", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "science_and_environment" ], "error": null }, { "id": "a802af8581cb2dccc564", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "education_and_history", "english", "explicit" ], "error": null }, { "id": "8a542db17025c945d82a", "expected": "math", "predicted": "coding", "confidence": 0.9999997777778271, "tags": [ "english", "explicit", "software_and_data" ], "error": null }, { "id": "65fd4a24e7f479643b44", "expected": "coding", "predicted": "reasoning", "confidence": 0.8181816694215146, "tags": [ "english", "keyword_collision", "software_and_data" ], "error": null }, { "id": "934ba026d1e3ce74b569", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "home_and_travel" ], "error": null }, { "id": "89a663ddbd3b116f7dad", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "education_and_history", "english", "quoted_distraction" ], "error": null }, { "id": "ab2cd20424545503e8c8", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "workplace_and_business" ], "error": null }, { "id": "160acc6f66c6c44a5c1a", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "science_and_environment" ], "error": null }, { "id": "139d4a1f04b5766ec6d7", "expected": "general", "predicted": "coding", "confidence": 0.8333331944444675, "tags": [ "arts_and_media", "english", "keyword_collision" ], "error": null }, { "id": "910408689532d0bab01a", "expected": "math", "predicted": "summarization", "confidence": 0.9999997500000625, "tags": [ "english", "quoted_distraction", "sports_and_leisure" ], "error": null }, { "id": "6d4b96eec9d06ff276a6", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "health_and_wellbeing", "implicit" ], "error": null }, { "id": "23f2e504dd641aef0c07", "expected": "summarization", "predicted": "creative_writing", "confidence": 0.99999950000025, "tags": [ "arts_and_media", "english", "implicit" ], "error": null }, { "id": "78c269d427439f8111dd", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "health_and_wellbeing", "quoted_distraction" ], "error": null }, { "id": "f6e9e9fca12fd3609488", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "software_and_data" ], "error": null }, { "id": "184fe3bd6431e4cdd3cf", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "education_and_history", "english", "explicit" ], "error": null }, { "id": "8482e766de147bc66518", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "sports_and_leisure" ], "error": null }, { "id": "983b29e2012f4f52a18a", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "retail_and_support" ], "error": null }, { "id": "1af9541d3e5169671ed9", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "science_and_environment" ], "error": null }, { "id": "47c82e72705c0ebb8538", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "261d5019720fdb62c020", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "public_services" ], "error": null }, { "id": "b20f7e3d48ad50c3f2aa", "expected": "reasoning", "predicted": "summarization", "confidence": 0.9999997500000625, "tags": [ "english", "explicit", "sports_and_leisure" ], "error": null }, { "id": "c3d5cf5bb4c50e290131", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "software_and_data" ], "error": null }, { "id": "0f1c4364cf65fa073901", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "sports_and_leisure" ], "error": null }, { "id": "bdf254ebb8b710451598", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "health_and_wellbeing", "quoted_distraction" ], "error": null }, { "id": "6954bf9b1ce86e9d9112", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "retail_and_support" ], "error": null }, { "id": "90fc2ef794829f7c8b72", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "public_services", "quoted_distraction" ], "error": null }, { "id": "2017324475c364638788", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "ea4eaa9694442865d139", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "health_and_wellbeing", "implicit" ], "error": null }, { "id": "3f7573575bb1e7f70119", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "education_and_history", "english", "quoted_distraction" ], "error": null }, { "id": "4688163023f1b2f24f91", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "education_and_history", "english", "keyword_collision" ], "error": null }, { "id": "c26dd697fbea4059c088", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "sports_and_leisure" ], "error": null }, { "id": "1c5fcb14d2f1cb6316dc", "expected": "math", "predicted": "reasoning", "confidence": 0.9999997142857959, "tags": [ "english", "health_and_wellbeing", "keyword_collision" ], "error": null }, { "id": "1cb124621368496ebd28", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "science_and_environment" ], "error": null }, { "id": "aaaa66d2f78f7efa4206", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "workplace_and_business" ], "error": null }, { "id": "11a493e5b6c2f9665457", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "quoted_distraction" ], "error": null }, { "id": "fd91948f511a05c56916", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "cb95c73725be1e468ad5", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "sports_and_leisure" ], "error": null }, { "id": "73bae6342993f8d7d7bc", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "health_and_wellbeing", "implicit" ], "error": null }, { "id": "9ce3d131bea9511add54", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "software_and_data" ], "error": null }, { "id": "f5cdf734f0cf32071198", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "education_and_history", "english", "quoted_distraction" ], "error": null }, { "id": "0a3dde9b7138a4b8df44", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "science_and_environment" ], "error": null }, { "id": "241efe3ded101ae619cb", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "quoted_distraction" ], "error": null }, { "id": "2cce175108805f6828d3", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "keyword_collision" ], "error": null }, { "id": "11de30d98dc3356d6fc7", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "workplace_and_business" ], "error": null }, { "id": "ea41d7cd578344088523", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "public_services" ], "error": null }, { "id": "e663ab1e1e9abb22015a", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "keyword_collision" ], "error": null }, { "id": "3ab5a7b2dc7b9498ee4d", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "software_and_data" ], "error": null }, { "id": "7a2b8fb4fae307bf2c3c", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "public_services", "quoted_distraction" ], "error": null }, { "id": "f7e6ca90bc77c4df7997", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "implicit" ], "error": null }, { "id": "9b55ab785d84812f4c8d", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "42568acd8cec8efe0b3c", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "workplace_and_business" ], "error": null }, { "id": "67bc3fceaa3417199229", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "workplace_and_business" ], "error": null }, { "id": "b05dff71c48c3deed906", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "workplace_and_business" ], "error": null }, { "id": "7563ef4bfbc2207133f4", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "health_and_wellbeing" ], "error": null }, { "id": "04a04e71fe6a88a707e2", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "software_and_data" ], "error": null }, { "id": "c3961f339329d5625ba3", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "public_services", "quoted_distraction" ], "error": null }, { "id": "6ddbd473835109740c1f", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "english", "health_and_wellbeing", "implicit" ], "error": null }, { "id": "5806547483dbfefd47d3", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "software_and_data" ], "error": null }, { "id": "c46a284ef6b548e06f66", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "9d9204511108f5a5c743", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "home_and_travel" ], "error": null }, { "id": "864e215dcedfae63f5bb", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "education_and_history", "english", "implicit" ], "error": null }, { "id": "e77a082d7223fe6e7f6f", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "sports_and_leisure" ], "error": null }, { "id": "75d04946b370bd8e8989", "expected": "creative_writing", "predicted": "coding", "confidence": 0.9999997777778271, "tags": [ "english", "home_and_travel", "keyword_collision" ], "error": null }, { "id": "073f663035685d8cd67b", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "education_and_history", "english", "quoted_distraction" ], "error": null }, { "id": "b76200e5d8a6a789b892", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "health_and_wellbeing" ], "error": null }, { "id": "679b70ec7f17e5bc7d1d", "expected": "reasoning", "predicted": "coding", "confidence": 0.8181816694215146, "tags": [ "english", "keyword_collision", "software_and_data" ], "error": null }, { "id": "1b4c6ec103665932ecc2", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "retail_and_support" ], "error": null }, { "id": "2079a105982416677b9f", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "science_and_environment" ], "error": null }, { "id": "04b0be7a016981756d27", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "science_and_environment" ], "error": null }, { "id": "64db9b9347702bdfd601", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "implicit" ], "error": null }, { "id": "82b2ae5b00aeb0a9b7d3", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "0f88f56700a25bdc5ade", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "974687e3a3903f925a47", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "sports_and_leisure" ], "error": null }, { "id": "28b24fee3d60f36e4262", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "education_and_history", "english", "implicit" ], "error": null }, { "id": "3f1152b7a9865caff6d1", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "sports_and_leisure" ], "error": null }, { "id": "f793586c3f2f013b94db", "expected": "summarization", "predicted": "creative_writing", "confidence": 0.8749998906250137, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "62dc7eca070859e9e8c2", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "99718a62d463917596b9", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "sports_and_leisure" ], "error": null }, { "id": "4f4983b8517fe7d8745e", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "health_and_wellbeing", "quoted_distraction" ], "error": null }, { "id": "0a51567358fa13497148", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "retail_and_support" ], "error": null }, { "id": "5c143d5a8da9ac05ecad", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "health_and_wellbeing", "quoted_distraction" ], "error": null }, { "id": "8dead2a7992843ded750", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "science_and_environment" ], "error": null }, { "id": "5b816623134b4f5a0feb", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "implicit" ], "error": null }, { "id": "bb0e2df9b40eb15ca542", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "education_and_history", "english", "keyword_collision" ], "error": null }, { "id": "80b8205ff85542f097c0", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "software_and_data" ], "error": null }, { "id": "0b0ce0600ea71df829a6", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "keyword_collision" ], "error": null }, { "id": "dd635f6d3e3bb1d28f7d", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "education_and_history", "english", "implicit" ], "error": null }, { "id": "e5613c647a6945127bd0", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "retail_and_support" ], "error": null }, { "id": "d67a4fa9a57e65ad8788", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "quoted_distraction" ], "error": null }, { "id": "c37d6544b1be4c5c9299", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "retail_and_support" ], "error": null }, { "id": "cb8e1da587bcc8b66819", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "education_and_history", "english", "quoted_distraction" ], "error": null }, { "id": "d872f9c9d393d43ae8fa", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "implicit" ], "error": null }, { "id": "071f1d05d4164434a6a4", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "home_and_travel" ], "error": null }, { "id": "f5f9687baa336d3b132d", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "science_and_environment" ], "error": null }, { "id": "fed121619d7d51b0c427", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "software_and_data" ], "error": null }, { "id": "a830cd5fe057362f5417", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "home_and_travel" ], "error": null }, { "id": "a7d0191cbcd6bd999a89", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "science_and_environment" ], "error": null }, { "id": "d41ad266a6d363f201ef", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "implicit" ], "error": null }, { "id": "fb8fe76e1701d45c3b01", "expected": "general", "predicted": "coding", "confidence": 0.6666664444445185, "tags": [ "english", "keyword_collision", "public_services" ], "error": null }, { "id": "026bdcd9add215dd7be9", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "sports_and_leisure" ], "error": null }, { "id": "2a3effdea00a1bc75a20", "expected": "summarization", "predicted": "creative_writing", "confidence": 0.6956521134215554, "tags": [ "education_and_history", "english", "quoted_distraction" ], "error": null }, { "id": "de575aee9eebda8643b2", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "keyword_collision" ], "error": null }, { "id": "582b7ecfc374940f96d0", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "public_services" ], "error": null }, { "id": "af6a5b62d658833cbd06", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "retail_and_support" ], "error": null }, { "id": "000ecfb9fee737216186", "expected": "general", "predicted": "summarization", "confidence": 0.9999997500000625, "tags": [ "education_and_history", "english", "explicit" ], "error": null }, { "id": "0034500228776613ce0b", "expected": "coding", "predicted": "reasoning", "confidence": 0.7777776049383099, "tags": [ "education_and_history", "english", "keyword_collision" ], "error": null }, { "id": "4406a6dad536185a39c8", "expected": "coding", "predicted": "reasoning", "confidence": 0.7142855102041399, "tags": [ "english", "implicit", "software_and_data" ], "error": null }, { "id": "1dd48cce991abd916f1f", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "e23237c7ab336cc55983", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "implicit" ], "error": null }, { "id": "23fbaf37fed861f555e9", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "education_and_history", "english", "keyword_collision" ], "error": null }, { "id": "935882fd9871a82b2f34", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "education_and_history", "english", "implicit" ], "error": null }, { "id": "f77cb06b5a2c5ae40043", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "science_and_environment" ], "error": null }, { "id": "8a50c06d02eed14f08d1", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "workplace_and_business" ], "error": null }, { "id": "cff40d04de4707191283", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "science_and_environment" ], "error": null }, { "id": "6179e53c41951b7f93da", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "software_and_data" ], "error": null }, { "id": "0d2ff52d4a210955f259", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "science_and_environment" ], "error": null }, { "id": "f01a9d70456a77248fce", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "software_and_data" ], "error": null }, { "id": "1d8702ec94a85453308b", "expected": "summarization", "predicted": "creative_writing", "confidence": 0.99999950000025, "tags": [ "arts_and_media", "english", "keyword_collision" ], "error": null }, { "id": "68986e70c506d42a3f24", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "implicit" ], "error": null }, { "id": "bdcd24b85ac8bdfb2c9b", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "quoted_distraction" ], "error": null }, { "id": "6d26a887ac42219196eb", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "workplace_and_business" ], "error": null }, { "id": "bcf2a162a6d48fe8a2d0", "expected": "coding", "predicted": "reasoning", "confidence": 0.7142855102041399, "tags": [ "english", "home_and_travel", "keyword_collision" ], "error": null }, { "id": "08660a25fde1ab51b43f", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "83cd435b7e45907f9716", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "f3db680483eda7ee8c5d", "expected": "coding", "predicted": "summarization", "confidence": 0.6666665555555741, "tags": [ "english", "keyword_collision", "retail_and_support" ], "error": null }, { "id": "c9db8b4932fd15a78052", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "health_and_wellbeing", "implicit" ], "error": null }, { "id": "34d153b59477f1da2301", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "retail_and_support" ], "error": null }, { "id": "be1621d2bffa14faa217", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "retail_and_support" ], "error": null }, { "id": "22fe52b70034d0630ef0", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "science_and_environment" ], "error": null }, { "id": "c4268afe94d4d0fa12ec", "expected": "coding", "predicted": "summarization", "confidence": 0.799999840000032, "tags": [ "english", "public_services", "quoted_distraction" ], "error": null }, { "id": "915570a621fff0121127", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "explicit" ], "error": null }, { "id": "168bd22a2f90cba24345", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "implicit" ], "error": null }, { "id": "a8735bd0cbf69521d0a0", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "education_and_history", "english", "implicit" ], "error": null }, { "id": "2a7f1edfefb2342759b8", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "843737d0c419959c8283", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "workplace_and_business" ], "error": null }, { "id": "c01ba1ff99d62992d182", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "e89c49f2408537936571", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "retail_and_support" ], "error": null }, { "id": "ad3181675a78f656f584", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "health_and_wellbeing" ], "error": null }, { "id": "41f55aa8dcd35313725f", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "science_and_environment" ], "error": null }, { "id": "e075dced2aad6ef2b9dc", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "public_services" ], "error": null }, { "id": "0290f779d565a08ae06f", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "sports_and_leisure" ], "error": null }, { "id": "3a668118adb6651a2024", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "health_and_wellbeing" ], "error": null }, { "id": "a492c8230050d71d31fb", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "926b0c374ff6bcc08b12", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "education_and_history", "english", "implicit" ], "error": null }, { "id": "2ee9da25c537bf08d45b", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "science_and_environment" ], "error": null }, { "id": "730c75001e1a75d8e1a1", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "science_and_environment" ], "error": null }, { "id": "137197c52518d04a1851", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "sports_and_leisure" ], "error": null }, { "id": "c5201b92e8eea97d0779", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "sports_and_leisure" ], "error": null }, { "id": "356d606db5145c551091", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "software_and_data" ], "error": null }, { "id": "ef84d8c26c15a0137291", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "explicit" ], "error": null }, { "id": "2626eb3c60b7409e1b94", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "science_and_environment" ], "error": null }, { "id": "a3d09721dc54c7aa23e5", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "software_and_data" ], "error": null }, { "id": "ad95925fc1de97bc70af", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "workplace_and_business" ], "error": null }, { "id": "f9a8cf3068d45c8f3eab", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "implicit" ], "error": null }, { "id": "fbc9291d970bdd23e823", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "software_and_data" ], "error": null }, { "id": "39c9c27b895d27f6ca67", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "c6ab4a0cb09ffd902062", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "education_and_history", "english", "quoted_distraction" ], "error": null }, { "id": "95fac4f60746f89097e3", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "retail_and_support" ], "error": null }, { "id": "cbc888cbb5bca1df0382", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "software_and_data" ], "error": null }, { "id": "2c9cb4ec13f386670ac1", "expected": "creative_writing", "predicted": "coding", "confidence": 0.6666664444445185, "tags": [ "education_and_history", "english", "keyword_collision" ], "error": null }, { "id": "95fe893d16587da4c304", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "workplace_and_business" ], "error": null }, { "id": "1145d80ad18cae992b8d", "expected": "creative_writing", "predicted": "coding", "confidence": 0.799999840000032, "tags": [ "english", "health_and_wellbeing", "keyword_collision" ], "error": null }, { "id": "b61ac98d9fc06327314f", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "b6f766624531547d3e51", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "education_and_history", "english", "implicit" ], "error": null }, { "id": "50cf5e4b45cbf4d15e0e", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "retail_and_support" ], "error": null }, { "id": "7c9f8dcb26520d798034", "expected": "math", "predicted": "summarization", "confidence": 0.9999997500000625, "tags": [ "english", "keyword_collision", "retail_and_support" ], "error": null }, { "id": "736d2c1b60f6d06eba0a", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "retail_and_support" ], "error": null }, { "id": "185f8c2743213b77eeef", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "dbc5b6eaaebd8ca86141", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "english", "public_services", "quoted_distraction" ], "error": null }, { "id": "ee7eae2defca05629587", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "workplace_and_business" ], "error": null }, { "id": "2e9f75b565c09e38ba99", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "health_and_wellbeing", "quoted_distraction" ], "error": null }, { "id": "d1bacc82210dcdddc82b", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "sports_and_leisure" ], "error": null }, { "id": "2fcf0f95bafa8322107f", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "keyword_collision" ], "error": null }, { "id": "0753da9d8314a486f9b0", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "workplace_and_business" ], "error": null }, { "id": "28f5c4eb8ecac426e8a1", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "public_services", "quoted_distraction" ], "error": null }, { "id": "0195b8853a9ac1946077", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "public_services" ], "error": null }, { "id": "106c9f6d3683a357a317", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "implicit" ], "error": null }, { "id": "4b9de10b8409d8a18158", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "workplace_and_business" ], "error": null }, { "id": "9af9c98619b1dd1c7938", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "education_and_history", "english", "quoted_distraction" ], "error": null }, { "id": "203b5ecd78c4ae28b73d", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "workplace_and_business" ], "error": null }, { "id": "9b95af368c8da504cf36", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "implicit" ], "error": null }, { "id": "46f712e5440d46598940", "expected": "math", "predicted": "summarization", "confidence": 0.9999997500000625, "tags": [ "english", "keyword_collision", "public_services" ], "error": null }, { "id": "3bef78cba49297d29eb4", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "sports_and_leisure" ], "error": null }, { "id": "e5fe576cda7fbe1fc492", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "sports_and_leisure" ], "error": null }, { "id": "42ac07c6888290378b2d", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "workplace_and_business" ], "error": null }, { "id": "1893c74cd232241fd977", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "public_services" ], "error": null }, { "id": "ceb333586f1863996b2c", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "workplace_and_business" ], "error": null }, { "id": "09b263482370f5ee454e", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "implicit" ], "error": null }, { "id": "83aaab476c8f6417dbc5", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "science_and_environment" ], "error": null }, { "id": "07900988cff1fe92f5c2", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "sports_and_leisure" ], "error": null }, { "id": "119d90ed93df45ee84e3", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "home_and_travel" ], "error": null }, { "id": "e510d4b2a901c6fea727", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "health_and_wellbeing", "implicit" ], "error": null }, { "id": "1c2d6e66e6467c5357d4", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "health_and_wellbeing", "implicit" ], "error": null }, { "id": "9f4cba614b2eb428c1b9", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "44317bf66d948e161221", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "implicit" ], "error": null }, { "id": "5cee93858561dd2c7bc0", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "software_and_data" ], "error": null }, { "id": "beb5f2c57b6fd758b862", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "implicit" ], "error": null }, { "id": "6f3f88ca5c2bcebaa348", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "public_services" ], "error": null }, { "id": "8ace2a5095414dac9e3d", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "science_and_environment" ], "error": null }, { "id": "5ea75201f8d3ee977091", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "public_services" ], "error": null }, { "id": "1bc087bec252f279b524", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "workplace_and_business" ], "error": null }, { "id": "b67731b5aa5aa8bf12c4", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "health_and_wellbeing", "quoted_distraction" ], "error": null }, { "id": "5a056b8838064b85e000", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "software_and_data" ], "error": null }, { "id": "71f380cf26c386f7c4c9", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "science_and_environment" ], "error": null }, { "id": "7c24b468d9684cc50901", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "keyword_collision" ], "error": null }, { "id": "e58631029d8f261abac3", "expected": "creative_writing", "predicted": "coding", "confidence": 0.6666664444445185, "tags": [ "arts_and_media", "english", "keyword_collision" ], "error": null }, { "id": "8b6ffad6d552ae1f1619", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "software_and_data" ], "error": null }, { "id": "f76d90605fb8d23c5c25", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "public_services" ], "error": null }, { "id": "a6950e8a97382e1e0dc2", "expected": "general", "predicted": "math", "confidence": 0.7499998125000469, "tags": [ "english", "health_and_wellbeing", "keyword_collision" ], "error": null }, { "id": "ab0171a79853c028a5c9", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "70feaa32e350132f93a3", "expected": "creative_writing", "predicted": "coding", "confidence": 0.7142855102041399, "tags": [ "english", "quoted_distraction", "software_and_data" ], "error": null }, { "id": "0fe5cf11594460deced5", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "workplace_and_business" ], "error": null }, { "id": "9b6c9257e4e4e1a25dad", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "software_and_data" ], "error": null }, { "id": "835e4a244a5fa09e02d4", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "workplace_and_business" ], "error": null }, { "id": "f6e19ef742e837419fae", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "public_services" ], "error": null }, { "id": "09585255255364a13cd0", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "41be605dcd05dd815139", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "f6ab0ff7558938e68cb2", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "health_and_wellbeing", "keyword_collision" ], "error": null }, { "id": "798852db56402a369fbc", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "retail_and_support" ], "error": null }, { "id": "439ce0ab2b6ce969a6fd", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "public_services" ], "error": null }, { "id": "362e9048d6b5caa6fc80", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "implicit" ], "error": null }, { "id": "e089a2337b543aba9e6a", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "sports_and_leisure" ], "error": null }, { "id": "e3449496d500a2014736", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "science_and_environment" ], "error": null }, { "id": "e0389fd633c6d8c52e29", "expected": "summarization", "predicted": "coding", "confidence": 0.799999840000032, "tags": [ "english", "quoted_distraction", "retail_and_support" ], "error": null }, { "id": "56906d5a3796f985cd07", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "keyword_collision" ], "error": null }, { "id": "c51921b4bd404469e9ef", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "sports_and_leisure" ], "error": null }, { "id": "661ceb172a936831b94e", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "keyword_collision" ], "error": null }, { "id": "5431b08ebc1b0f08f88d", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "science_and_environment" ], "error": null }, { "id": "df01fb7ad4d4fe17b24b", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "c5ae42d1f6f9e09cc987", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "public_services" ], "error": null }, { "id": "c9e1b7308bf44be53926", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "sports_and_leisure" ], "error": null }, { "id": "c4973445663e2050358e", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "retail_and_support" ], "error": null }, { "id": "b50e762ed7f64d3516a0", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "quoted_distraction" ], "error": null }, { "id": "872df632207f40af3414", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "software_and_data" ], "error": null }, { "id": "42c985012850390119a3", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "explicit" ], "error": null }, { "id": "c2e87dd1077449d0d226", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "software_and_data" ], "error": null }, { "id": "47df61fe6ebf44aa7d74", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "implicit" ], "error": null }, { "id": "16c68065c512a78b06ea", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "retail_and_support" ], "error": null }, { "id": "6e1fb90a7937f6db1d34", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "keyword_collision" ], "error": null }, { "id": "cd3ad6e3b171cdbd9282", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "keyword_collision" ], "error": null }, { "id": "1a5b9e39b10f56af0a43", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "education_and_history", "english", "keyword_collision" ], "error": null }, { "id": "14949c784c7033eeaae6", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "workplace_and_business" ], "error": null }, { "id": "b2648eb34888d97af84b", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "public_services" ], "error": null }, { "id": "be1f42609b8856680837", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "health_and_wellbeing", "keyword_collision" ], "error": null }, { "id": "18e34fbe3bbc6b6637c4", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "science_and_environment" ], "error": null }, { "id": "94653e1b7b611651fda7", "expected": "general", "predicted": "coding", "confidence": 0.9999997777778271, "tags": [ "english", "keyword_collision", "workplace_and_business" ], "error": null }, { "id": "a5a0ad02a3a8e8f51474", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "implicit" ], "error": null }, { "id": "c4519f2cc08ab19a3541", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "home_and_travel" ], "error": null }, { "id": "0992496ed8d3f9ec1fb6", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "workplace_and_business" ], "error": null }, { "id": "7c047bd0819657f9af8f", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "workplace_and_business" ], "error": null }, { "id": "853850de7eade8e07116", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "home_and_travel" ], "error": null }, { "id": "caa46e4f7df207bfeb64", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "retail_and_support" ], "error": null }, { "id": "188ad75336e020ca44e2", "expected": "general", "predicted": "reasoning", "confidence": 0.99999960000016, "tags": [ "english", "explicit", "science_and_environment" ], "error": null }, { "id": "78b7dde5e1bc6c2ed5f3", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "implicit" ], "error": null }, { "id": "731ea10face531ad50ec", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "implicit" ], "error": null }, { "id": "3e98b22d7df9d0a13214", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "sports_and_leisure" ], "error": null }, { "id": "b1d17d0217d1f37964d1", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "retail_and_support" ], "error": null }, { "id": "be1d66a4d2ff1afcf414", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "education_and_history", "english", "quoted_distraction" ], "error": null }, { "id": "300f6b0675565ecc6d45", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "workplace_and_business" ], "error": null }, { "id": "ced5ae2c12e4b03c0a9e", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "education_and_history", "english", "keyword_collision" ], "error": null }, { "id": "c91d1b795453cef24072", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "workplace_and_business" ], "error": null }, { "id": "01c3890f0ace6b3783a0", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "science_and_environment" ], "error": null }, { "id": "0f3a5094d18273df4665", "expected": "creative_writing", "predicted": "summarization", "confidence": 0.7142855102041399, "tags": [ "english", "explicit", "workplace_and_business" ], "error": null }, { "id": "af7c7b95adf86fcfacde", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "public_services", "quoted_distraction" ], "error": null }, { "id": "590af5d2c38389a02f2a", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "public_services" ], "error": null }, { "id": "6ba2c15f9377fac89b4a", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "546db3471f77e0f59eaa", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "141551a6f71b9146567a", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "workplace_and_business" ], "error": null }, { "id": "fc50f462a50733f226bb", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "workplace_and_business" ], "error": null }, { "id": "6f940728b57d5f0f4b25", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "education_and_history", "english", "keyword_collision" ], "error": null }, { "id": "1896ec51636504b5700b", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "software_and_data" ], "error": null }, { "id": "3e1f6f0bcfb1ca54ecde", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "education_and_history", "english", "implicit" ], "error": null }, { "id": "796a2f6e93bee4409bce", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "software_and_data" ], "error": null }, { "id": "2f70ee51dbfefb489a80", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "science_and_environment" ], "error": null }, { "id": "1ba1ecac79192724fcc5", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "education_and_history", "english", "explicit" ], "error": null }, { "id": "2da59285c3bc72884c17", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "retail_and_support" ], "error": null }, { "id": "c6061d47eebee24ebb8d", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "6aa95a3d06108b020a26", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "science_and_environment" ], "error": null }, { "id": "118d25bc2c70f8e162a8", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "software_and_data" ], "error": null }, { "id": "d5764d525bd61e8d1b73", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "explicit" ], "error": null }, { "id": "f7c254a6e4f410cd4e6a", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "software_and_data" ], "error": null }, { "id": "0829d305f71eae0886cd", "expected": "coding", "predicted": "summarization", "confidence": 0.799999840000032, "tags": [ "education_and_history", "english", "keyword_collision" ], "error": null }, { "id": "20bf3adbf974f1aacb80", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "software_and_data" ], "error": null }, { "id": "ccab106ab907b66ab5e2", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "keyword_collision" ], "error": null }, { "id": "37e8b426c18514cd65b1", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "sports_and_leisure" ], "error": null }, { "id": "94e79d61f2500cd071f3", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "health_and_wellbeing", "implicit" ], "error": null }, { "id": "d24926e9a916c32751a0", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "science_and_environment" ], "error": null }, { "id": "23bf6c4ebeb9d6c37004", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "retail_and_support" ], "error": null }, { "id": "e5b82b29d27a8e485db1", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "workplace_and_business" ], "error": null }, { "id": "f65f15005f59610cb231", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "sports_and_leisure" ], "error": null }, { "id": "53bac73d24f3603f18f0", "expected": "coding", "predicted": "reasoning", "confidence": 0.7142855102041399, "tags": [ "arts_and_media", "english", "implicit" ], "error": null }, { "id": "baf75e9bc3cfd38d0296", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "health_and_wellbeing", "quoted_distraction" ], "error": null }, { "id": "c602467efebd5a38f7d7", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "quoted_distraction" ], "error": null }, { "id": "96bba5ad095c103bd28e", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "english", "health_and_wellbeing", "implicit" ], "error": null }, { "id": "20f3a91189afd5a16825", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "workplace_and_business" ], "error": null }, { "id": "b4ecc03b78cb71d0fa1d", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "dc7f8c5082a153c49621", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "public_services", "quoted_distraction" ], "error": null }, { "id": "866a22e24bc847837717", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "education_and_history", "english", "implicit" ], "error": null }, { "id": "4eeaa4706a2ec0c93654", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "science_and_environment" ], "error": null }, { "id": "b171cc036a497955621d", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "quoted_distraction" ], "error": null }, { "id": "7085f44b6a5ea931ae3d", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "science_and_environment" ], "error": null }, { "id": "ae7aff453f9ca2901a2e", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "science_and_environment" ], "error": null }, { "id": "bfaf32d0d5e2c4a3fb20", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "public_services" ], "error": null }, { "id": "20965d22670abba93100", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "implicit" ], "error": null }, { "id": "84f586d5f97e2def57cd", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "workplace_and_business" ], "error": null }, { "id": "1e7e6038c35305ad39cd", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "science_and_environment" ], "error": null }, { "id": "f0ecf4303cb0440a99da", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "implicit" ], "error": null }, { "id": "6c7b54d80cc64ed3bb08", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "implicit" ], "error": null }, { "id": "3628135c6b33fe7e6a60", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "retail_and_support" ], "error": null }, { "id": "7b24c403412b0b80a003", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "sports_and_leisure" ], "error": null }, { "id": "b99e61cfcea7b0917787", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "retail_and_support" ], "error": null }, { "id": "351c8d3edc2ec9ba1ee0", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "health_and_wellbeing" ], "error": null }, { "id": "0743eeb5413ea1274d85", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "public_services" ], "error": null }, { "id": "e91cd530fb33d1051772", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "040247cb6b64fe4ff373", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "workplace_and_business" ], "error": null }, { "id": "884036295bc6527dd26d", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "implicit" ], "error": null }, { "id": "28acb48403496cd12181", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "education_and_history", "english", "implicit" ], "error": null }, { "id": "46dabebfa9cf860d65b7", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "software_and_data" ], "error": null }, { "id": "2aa99f805e32e60ba81a", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "keyword_collision" ], "error": null }, { "id": "c715c42a3a8ec0dcda2c", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "workplace_and_business" ], "error": null }, { "id": "77d2e02f5ee18e1e1998", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "education_and_history", "english", "explicit" ], "error": null }, { "id": "87a7f4436b7438713f22", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "health_and_wellbeing", "keyword_collision" ], "error": null }, { "id": "088c9c71ce5c86c8fdd7", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "retail_and_support" ], "error": null }, { "id": "5fb517539877682f199e", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "explicit" ], "error": null }, { "id": "122eb6ae2cb690054796", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "software_and_data" ], "error": null }, { "id": "b334dc236e4afdc8dc1d", "expected": "creative_writing", "predicted": "reasoning", "confidence": 0.7499998125000469, "tags": [ "arts_and_media", "english", "keyword_collision" ], "error": null }, { "id": "7b7863705f9079319049", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "workplace_and_business" ], "error": null }, { "id": "ffa7fa26566ae7a73259", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "education_and_history", "english", "implicit" ], "error": null }, { "id": "4d7d78bde13039aeb22f", "expected": "creative_writing", "predicted": "summarization", "confidence": 0.9999997500000625, "tags": [ "english", "quoted_distraction", "software_and_data" ], "error": null }, { "id": "30dc803e3b2879054360", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "public_services" ], "error": null }, { "id": "5bf1a76a89ef1255b54e", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "public_services" ], "error": null }, { "id": "0c9421bdebab5df308d2", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "science_and_environment" ], "error": null }, { "id": "615fba136c6cb047f5bb", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "health_and_wellbeing", "keyword_collision" ], "error": null }, { "id": "90bff28e81552b97e438", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "public_services", "quoted_distraction" ], "error": null }, { "id": "38fa57e835cade3b7ee8", "expected": "math", "predicted": "summarization", "confidence": 0.9999997500000625, "tags": [ "english", "home_and_travel", "keyword_collision" ], "error": null }, { "id": "797e030743adca3b2bc4", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "education_and_history", "english", "quoted_distraction" ], "error": null }, { "id": "801c183238931341eef5", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "health_and_wellbeing", "implicit" ], "error": null }, { "id": "5303fce092eb5c606dc9", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "education_and_history", "english", "quoted_distraction" ], "error": null }, { "id": "637ad4b9121132512550", "expected": "coding", "predicted": "summarization", "confidence": 0.6666665555555741, "tags": [ "english", "keyword_collision", "sports_and_leisure" ], "error": null }, { "id": "26503efb9be312417162", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "software_and_data" ], "error": null }, { "id": "bea9e1f2e8f7427a1245", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "workplace_and_business" ], "error": null }, { "id": "23866ffb91c420cd9118", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "workplace_and_business" ], "error": null }, { "id": "493a179099fb1a021656", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "software_and_data" ], "error": null }, { "id": "dd6773d4373fc6a31670", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "retail_and_support" ], "error": null }, { "id": "77dbeb8016a419b481fa", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "public_services" ], "error": null }, { "id": "4c2ee79659cdb28c7539", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "english", "health_and_wellbeing", "keyword_collision" ], "error": null }, { "id": "8566945e918cf6d679ad", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "public_services" ], "error": null }, { "id": "e67d36f52445d6f7e187", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "workplace_and_business" ], "error": null }, { "id": "7d7ed7448c6bfd21e150", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "sports_and_leisure" ], "error": null }, { "id": "54644525861726786806", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "workplace_and_business" ], "error": null }, { "id": "761348beea7d6febb36e", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "education_and_history", "english", "implicit" ], "error": null }, { "id": "12272f9cde9cbaf8c39a", "expected": "coding", "predicted": "reasoning", "confidence": 0.7777776049383099, "tags": [ "english", "keyword_collision", "retail_and_support" ], "error": null }, { "id": "525dc384a3d430f54265", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "health_and_wellbeing", "keyword_collision" ], "error": null }, { "id": "bb27e1008c3991db1fb1", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "public_services" ], "error": null }, { "id": "b10a24a6dd3dff1fe67b", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "software_and_data" ], "error": null }, { "id": "37b99a80ddac444fea19", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "642db1bf76a8216db9e0", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "software_and_data" ], "error": null }, { "id": "5e0201ee96cebb29ab3c", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "software_and_data" ], "error": null }, { "id": "8010f179c8265849f665", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "software_and_data" ], "error": null }, { "id": "0cb3783931628bcd49b4", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "science_and_environment" ], "error": null }, { "id": "1ea1cb699c1759141306", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "46452eb03e331d38ecc9", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "workplace_and_business" ], "error": null }, { "id": "8e00883cb7168bd15bbc", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "english", "health_and_wellbeing", "quoted_distraction" ], "error": null }, { "id": "7b497d8c859c3410f194", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "health_and_wellbeing", "implicit" ], "error": null }, { "id": "1573b989b993b5f40975", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "e9031f07af843da769c9", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "health_and_wellbeing", "quoted_distraction" ], "error": null }, { "id": "1e8fb49c4fa09d6a00c2", "expected": "math", "predicted": "coding", "confidence": 0.6666665555555741, "tags": [ "arts_and_media", "english", "keyword_collision" ], "error": null }, { "id": "baf912a9f5483c9553f6", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "health_and_wellbeing", "implicit" ], "error": null }, { "id": "0a1bd4698457a8ac7261", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "sports_and_leisure" ], "error": null }, { "id": "2e6adb0dc3df4b196ef2", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "software_and_data" ], "error": null }, { "id": "7be90b4d5ac754b57948", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "sports_and_leisure" ], "error": null }, { "id": "9ec9ff018185d81d81cd", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "software_and_data" ], "error": null }, { "id": "7595aadd7f3cf8211971", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "quoted_distraction" ], "error": null }, { "id": "636850a6c66067943f76", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "health_and_wellbeing", "implicit" ], "error": null }, { "id": "0b483f7f02fff7f1685e", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "keyword_collision" ], "error": null }, { "id": "505c47960e52813d4745", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "quoted_distraction" ], "error": null }, { "id": "8099eb9c16304c5518f8", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "health_and_wellbeing", "quoted_distraction" ], "error": null }, { "id": "a850b6ce07bbb614a0b2", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "keyword_collision" ], "error": null }, { "id": "8e81bf855fc25beb3092", "expected": "math", "predicted": "summarization", "confidence": 0.9999997500000625, "tags": [ "english", "keyword_collision", "retail_and_support" ], "error": null }, { "id": "d2fd7b4f52c51e98cf89", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "workplace_and_business" ], "error": null }, { "id": "a061c5108e9501a72790", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "education_and_history", "english", "explicit" ], "error": null }, { "id": "dd4f2c1710dff5842eb9", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "health_and_wellbeing", "quoted_distraction" ], "error": null }, { "id": "d6e3ba53669ced31bbd9", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "retail_and_support" ], "error": null }, { "id": "e22c70f63feb13865cde", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "health_and_wellbeing", "keyword_collision" ], "error": null }, { "id": "cfbb60500a196c9f6a44", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "science_and_environment" ], "error": null }, { "id": "79b117c4b1d82376055f", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "eaa028f021dcb2a2eaf6", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "retail_and_support" ], "error": null }, { "id": "8377e539f21a6698238c", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "ca47a93477cb6f7bd54f", "expected": "creative_writing", "predicted": "coding", "confidence": 0.799999840000032, "tags": [ "english", "explicit", "health_and_wellbeing" ], "error": null }, { "id": "6d3199640c07f20fc662", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "keyword_collision" ], "error": null }, { "id": "c3d7039c6cb5fae2718e", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "sports_and_leisure" ], "error": null }, { "id": "20d016443e2f4f3ed61c", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "education_and_history", "english", "explicit" ], "error": null }, { "id": "5d1029eb9a244be44a89", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "retail_and_support" ], "error": null }, { "id": "3b1e4a2969e6812cd198", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "science_and_environment" ], "error": null }, { "id": "18420ee632bd0b8fa702", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "100a4655b778a61c15e1", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "workplace_and_business" ], "error": null }, { "id": "a89afa71a134defd0788", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "implicit" ], "error": null }, { "id": "f05f79295511ba0f0af6", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "home_and_travel" ], "error": null }, { "id": "bdf561c21847bd51fbde", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "retail_and_support" ], "error": null }, { "id": "3dd717cda4c5cf892608", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "education_and_history", "english", "implicit" ], "error": null }, { "id": "b91ece8e2df579cca066", "expected": "reasoning", "predicted": "coding", "confidence": 0.6153845207100738, "tags": [ "english", "keyword_collision", "science_and_environment" ], "error": null }, { "id": "3f5ea624e6b566bca6f6", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "sports_and_leisure" ], "error": null }, { "id": "46d42c4ccdd6ad81892c", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "science_and_environment" ], "error": null }, { "id": "d816071e7e93c04c8d5e", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "retail_and_support" ], "error": null }, { "id": "4e7d6cc93749288871b2", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "05ff4ae31bdc6a6bc29c", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "science_and_environment" ], "error": null }, { "id": "f3968d022ca681e65c6f", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "software_and_data" ], "error": null }, { "id": "04aa5647dad28657a678", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "implicit" ], "error": null }, { "id": "3cb971818df68aa9ff18", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "workplace_and_business" ], "error": null }, { "id": "10a8196ccde72c8e62b0", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "science_and_environment" ], "error": null }, { "id": "bcb507b7485b4b6b2247", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "workplace_and_business" ], "error": null }, { "id": "97aca115ee5a32995067", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "software_and_data" ], "error": null }, { "id": "750df5485ae642dc1c9d", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "education_and_history", "english", "keyword_collision" ], "error": null }, { "id": "a0041d473b6535db4467", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "sports_and_leisure" ], "error": null }, { "id": "fc84d7d52582336defe6", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "education_and_history", "english", "implicit" ], "error": null }, { "id": "5d833c9dc11a01825921", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "retail_and_support" ], "error": null }, { "id": "ad3705afb97c013d1ca0", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "health_and_wellbeing", "implicit" ], "error": null }, { "id": "8284106cf72f25d61a0a", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "home_and_travel" ], "error": null }, { "id": "d5637ed1653dd0959527", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "science_and_environment" ], "error": null }, { "id": "8775f2f8d78993ac26a6", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "workplace_and_business" ], "error": null }, { "id": "f84ac96f4cfab6c9e842", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "science_and_environment" ], "error": null }, { "id": "b3e958362c10654d49ed", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "public_services" ], "error": null }, { "id": "958ed1156f788ec1eb16", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "software_and_data" ], "error": null }, { "id": "8b68ee5bccd41cdd7159", "expected": "summarization", "predicted": "creative_writing", "confidence": 0.6666666111111158, "tags": [ "english", "quoted_distraction", "retail_and_support" ], "error": null }, { "id": "4a756cc5e9d8f76bdabe", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "retail_and_support" ], "error": null }, { "id": "c620157692f657cd865a", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "sports_and_leisure" ], "error": null }, { "id": "c794076af96fe4e47ddf", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "workplace_and_business" ], "error": null }, { "id": "f4eb3f9509b0a24ec979", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "9e01bf98fe9a9c01bac0", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "implicit" ], "error": null }, { "id": "ef21a309f0d30fcd33ea", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "workplace_and_business" ], "error": null }, { "id": "5d129c1fe8f22bdf3805", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "explicit" ], "error": null }, { "id": "fc7e69523e6845e159fb", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "education_and_history", "english", "explicit" ], "error": null }, { "id": "b5540b2515ee49df64b5", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "science_and_environment" ], "error": null }, { "id": "c8f730b74146f542d9a4", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "home_and_travel" ], "error": null }, { "id": "cc835c6b914c0cfb65f5", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "health_and_wellbeing", "keyword_collision" ], "error": null }, { "id": "57e42a1360ad2bdcccb1", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "education_and_history", "english", "implicit" ], "error": null }, { "id": "050eb91e7b0e1ac81f26", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "education_and_history", "english", "explicit" ], "error": null }, { "id": "943f2dfdbd0c64238750", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "health_and_wellbeing", "keyword_collision" ], "error": null }, { "id": "c76ff5f05fdc1f121d6b", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "public_services" ], "error": null }, { "id": "747a66c3a98538e88a17", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "software_and_data" ], "error": null }, { "id": "52ae73013b5f9168754f", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "retail_and_support" ], "error": null }, { "id": "646c4be145d5a4ab2e87", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "health_and_wellbeing" ], "error": null }, { "id": "69f4fb75ebae545f647d", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "sports_and_leisure" ], "error": null }, { "id": "ef335e214de78a2cc4a2", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "software_and_data" ], "error": null }, { "id": "5d7ddc81cfba149f5981", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "retail_and_support" ], "error": null }, { "id": "29f9e0e083e9c1f99c7e", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "public_services" ], "error": null }, { "id": "090f74989daec93e6901", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "software_and_data" ], "error": null }, { "id": "974d254b39db7ef31937", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "workplace_and_business" ], "error": null }, { "id": "87e7e947a2c2810115cc", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "science_and_environment" ], "error": null }, { "id": "186af0da14921c1d4475", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "science_and_environment" ], "error": null }, { "id": "ed7cf02e86191b90926d", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "explicit" ], "error": null }, { "id": "3b939131fa23e2122c19", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "software_and_data" ], "error": null }, { "id": "e050c1378797feb7a4fe", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "348e1c4ea62a66aff9ea", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "explicit" ], "error": null }, { "id": "53981090931e861724d4", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "implicit" ], "error": null }, { "id": "f40203ea7a442d906594", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "software_and_data" ], "error": null }, { "id": "dee22c306c6c40982d13", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "education_and_history", "english", "explicit" ], "error": null }, { "id": "9369d49991855abf8f00", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "science_and_environment" ], "error": null }, { "id": "4b0d564fa13e3b383c35", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "workplace_and_business" ], "error": null }, { "id": "b7289b1a3da136193a29", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "keyword_collision" ], "error": null }, { "id": "5a2d1fc65c149e714de8", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "software_and_data" ], "error": null }, { "id": "76ca9822e0f2df1710ca", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "sports_and_leisure" ], "error": null }, { "id": "2730e4d5e66f3bdcaf42", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "software_and_data" ], "error": null }, { "id": "312c8c0e101c1695122d", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "public_services" ], "error": null }, { "id": "77f78968efde0e9cc53d", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "workplace_and_business" ], "error": null }, { "id": "fbbcb0d9d03919e88376", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "public_services" ], "error": null }, { "id": "4c46d53e2860c4ef20ec", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "d65c6d2bc7bebd381ff1", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "87e8750a65c1ee973962", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "education_and_history", "english", "implicit" ], "error": null }, { "id": "857db3eb03b2d034ce92", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "quoted_distraction" ], "error": null }, { "id": "e616750c8c7fbdc2194b", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "keyword_collision" ], "error": null }, { "id": "a62561dc670369ed8ed3", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "bbfcef3f5958ab91ba99", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "software_and_data" ], "error": null }, { "id": "d8fe83298ba5099c71f0", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "workplace_and_business" ], "error": null }, { "id": "abe623e53fc0c8bf254c", "expected": "math", "predicted": "summarization", "confidence": 0.9999997500000625, "tags": [ "english", "keyword_collision", "public_services" ], "error": null }, { "id": "f04d6f595f26e5775002", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "sports_and_leisure" ], "error": null }, { "id": "0677ec5e47edea030d04", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "software_and_data" ], "error": null }, { "id": "82f620ed0a5d1d9006eb", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null } ], "accuracy_ci95": [ 0.46875, 0.5270833333333333 ], "macro_f1_ci95": [ 0.4705605514683423, 0.5411954761748519 ], "by_task_ci95": { "coding": [ 0.5625, 0.7125 ], "reasoning": [ 0.1625, 0.275 ], "creative_writing": [ 0.2625, 0.4625 ], "summarization": [ 0.575, 0.6875 ], "math": [ 0.09375, 0.2625 ], "general": [ 0.93125, 0.9875 ] } }, { "strategy": "laya", "requests": 960, "correct": 636, "accuracy": 0.6625, "macro_f1": 0.661615706852866, "errors": 0, "cache_hits": 0, "latency_ms": { "mean": 38.23760113218668, "p50": 37.625165998179, "p95": 46.52695384829712, "p99": 51.088239030432305 }, "latency_scope": "successful requests only", "all_attempts_latency_ms": { "p50": 37.625165998179, "p95": 46.52695384829712 }, "llm_calls": 0, "llm_call_rate": 0.0, "input_tokens": 0, "output_tokens": 0, "router_cost_usd": 0.0, "known_router_cost_usd": 0.0, "cost_coverage": 1.0, "cost_scope": "API charges only; local hardware and electricity excluded", "hardware_electricity_cost_usd": null, "hardware_electricity_cost_status": "unmeasured", "router_cost_per_1k_requests_usd": 0.0, "confusion_matrix": { "coding": { "coding": 131, "creative_writing": 4, "general": 11, "math": 8, "reasoning": 2, "summarization": 4 }, "reasoning": { "coding": 6, "creative_writing": 7, "general": 64, "reasoning": 77, "summarization": 6 }, "creative_writing": { "coding": 6, "creative_writing": 144, "general": 7, "summarization": 3 }, "summarization": { "coding": 9, "creative_writing": 27, "general": 34, "reasoning": 5, "summarization": 85 }, "math": { "coding": 9, "creative_writing": 2, "general": 54, "math": 74, "reasoning": 16, "summarization": 5 }, "general": { "coding": 14, "creative_writing": 14, "general": 125, "reasoning": 2, "summarization": 5 } }, "by_task": { "coding": { "requests": 160, "correct": 131, "accuracy": 0.81875 }, "reasoning": { "requests": 160, "correct": 77, "accuracy": 0.48125 }, "creative_writing": { "requests": 160, "correct": 144, "accuracy": 0.9 }, "summarization": { "requests": 160, "correct": 85, "accuracy": 0.53125 }, "math": { "requests": 160, "correct": 74, "accuracy": 0.4625 }, "general": { "requests": 160, "correct": 125, "accuracy": 0.78125 } }, "by_tag": { "arts_and_media": { "requests": 96, "accuracy": 0.65625 }, "education_and_history": { "requests": 96, "accuracy": 0.5520833333333334 }, "english": { "requests": 960, "accuracy": 0.6625 }, "explicit": { "requests": 240, "accuracy": 0.8375 }, "health_and_wellbeing": { "requests": 96, "accuracy": 0.6770833333333334 }, "home_and_travel": { "requests": 96, "accuracy": 0.6979166666666666 }, "implicit": { "requests": 240, "accuracy": 0.5583333333333333 }, "keyword_collision": { "requests": 240, "accuracy": 0.6416666666666667 }, "public_services": { "requests": 96, "accuracy": 0.6666666666666666 }, "quoted_distraction": { "requests": 240, "accuracy": 0.6125 }, "retail_and_support": { "requests": 96, "accuracy": 0.65625 }, "science_and_environment": { "requests": 96, "accuracy": 0.7083333333333334 }, "software_and_data": { "requests": 96, "accuracy": 0.5729166666666666 }, "sports_and_leisure": { "requests": 96, "accuracy": 0.6875 }, "workplace_and_business": { "requests": 96, "accuracy": 0.75 } }, "wrong_cases": [ { "id": "2ff6fc60ec5022d51e8d", "expected": "general", "predicted": "coding", "confidence": 0.4883, "tags": [ "english", "explicit", "sports_and_leisure" ], "error": null }, { "id": "67ed7847bffa3a948ba8", "expected": "math", "predicted": "reasoning", "confidence": 0.4165, "tags": [ "english", "explicit", "public_services" ], "error": null }, { "id": "efa47373a3b287b3172f", "expected": "math", "predicted": "reasoning", "confidence": 0.442, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "07df2439da4bcbfa95ba", "expected": "general", "predicted": "summarization", "confidence": 0.7621, "tags": [ "education_and_history", "english", "keyword_collision" ], "error": null }, { "id": "53ab818b9bbe70e5041a", "expected": "creative_writing", "predicted": "general", "confidence": 0.6959, "tags": [ "english", "quoted_distraction", "science_and_environment" ], "error": null }, { "id": "79a6cec2c9a0d3943095", "expected": "summarization", "predicted": "general", "confidence": 0.4072, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "8a542db17025c945d82a", "expected": "math", "predicted": "reasoning", "confidence": 0.2744, "tags": [ "english", "explicit", "software_and_data" ], "error": null }, { "id": "934ba026d1e3ce74b569", "expected": "reasoning", "predicted": "general", "confidence": 0.5893, "tags": [ "english", "explicit", "home_and_travel" ], "error": null }, { "id": "3ebcaca8935eb290fdd8", "expected": "summarization", "predicted": "creative_writing", "confidence": 0.5955, "tags": [ "arts_and_media", "english", "implicit" ], "error": null }, { "id": "89a663ddbd3b116f7dad", "expected": "creative_writing", "predicted": "coding", "confidence": 0.4145, "tags": [ "education_and_history", "english", "quoted_distraction" ], "error": null }, { "id": "7c173b42683fd641249f", "expected": "summarization", "predicted": "coding", "confidence": 0.6819, "tags": [ "english", "quoted_distraction", "software_and_data" ], "error": null }, { "id": "f975d1a7a27c593b5bdf", "expected": "coding", "predicted": "math", "confidence": 0.4215, "tags": [ "education_and_history", "english", "implicit" ], "error": null }, { "id": "6d4b96eec9d06ff276a6", "expected": "math", "predicted": "general", "confidence": 0.3642, "tags": [ "english", "health_and_wellbeing", "implicit" ], "error": null }, { "id": "23f2e504dd641aef0c07", "expected": "summarization", "predicted": "creative_writing", "confidence": 0.7853, "tags": [ "arts_and_media", "english", "implicit" ], "error": null }, { "id": "f6e9e9fca12fd3609488", "expected": "reasoning", "predicted": "coding", "confidence": 0.4562, "tags": [ "english", "implicit", "software_and_data" ], "error": null }, { "id": "c3d5cf5bb4c50e290131", "expected": "coding", "predicted": "summarization", "confidence": 0.4147, "tags": [ "english", "quoted_distraction", "software_and_data" ], "error": null }, { "id": "f9dc70a9a229bd10523c", "expected": "math", "predicted": "general", "confidence": 0.4058, "tags": [ "english", "public_services", "quoted_distraction" ], "error": null }, { "id": "5c270ed419d6d8259b05", "expected": "general", "predicted": "coding", "confidence": 0.4848, "tags": [ "english", "health_and_wellbeing", "quoted_distraction" ], "error": null }, { "id": "bdf254ebb8b710451598", "expected": "creative_writing", "predicted": "general", "confidence": 0.5823, "tags": [ "english", "health_and_wellbeing", "quoted_distraction" ], "error": null }, { "id": "6954bf9b1ce86e9d9112", "expected": "math", "predicted": "general", "confidence": 0.4783, "tags": [ "english", "quoted_distraction", "retail_and_support" ], "error": null }, { "id": "0868df1033be02f01a14", "expected": "summarization", "predicted": "coding", "confidence": 0.6612, "tags": [ "english", "implicit", "workplace_and_business" ], "error": null }, { "id": "90fc2ef794829f7c8b72", "expected": "math", "predicted": "coding", "confidence": 0.3795, "tags": [ "english", "public_services", "quoted_distraction" ], "error": null }, { "id": "2017324475c364638788", "expected": "math", "predicted": "reasoning", "confidence": 0.448, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "3f7573575bb1e7f70119", "expected": "math", "predicted": "general", "confidence": 0.3339, "tags": [ "education_and_history", "english", "quoted_distraction" ], "error": null }, { "id": "4688163023f1b2f24f91", "expected": "summarization", "predicted": "reasoning", "confidence": 0.2883, "tags": [ "education_and_history", "english", "keyword_collision" ], "error": null }, { "id": "c26dd697fbea4059c088", "expected": "reasoning", "predicted": "general", "confidence": 0.7859, "tags": [ "english", "quoted_distraction", "sports_and_leisure" ], "error": null }, { "id": "9132b51ad13574d4b4e5", "expected": "math", "predicted": "coding", "confidence": 0.4747, "tags": [ "english", "explicit", "software_and_data" ], "error": null }, { "id": "1c5fcb14d2f1cb6316dc", "expected": "math", "predicted": "reasoning", "confidence": 0.6197, "tags": [ "english", "health_and_wellbeing", "keyword_collision" ], "error": null }, { "id": "1cb124621368496ebd28", "expected": "summarization", "predicted": "general", "confidence": 0.4642, "tags": [ "english", "keyword_collision", "science_and_environment" ], "error": null }, { "id": "d29cb8254a1c42be9bab", "expected": "general", "predicted": "coding", "confidence": 0.3936, "tags": [ "english", "implicit", "software_and_data" ], "error": null }, { "id": "aaaa66d2f78f7efa4206", "expected": "summarization", "predicted": "reasoning", "confidence": 0.3762, "tags": [ "english", "implicit", "workplace_and_business" ], "error": null }, { "id": "58398efe336eecb1c8de", "expected": "summarization", "predicted": "creative_writing", "confidence": 0.5979, "tags": [ "education_and_history", "english", "keyword_collision" ], "error": null }, { "id": "fd91948f511a05c56916", "expected": "reasoning", "predicted": "summarization", "confidence": 0.3765, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "9ce3d131bea9511add54", "expected": "reasoning", "predicted": "coding", "confidence": 0.5314, "tags": [ "english", "quoted_distraction", "software_and_data" ], "error": null }, { "id": "f5cdf734f0cf32071198", "expected": "math", "predicted": "reasoning", "confidence": 0.3399, "tags": [ "education_and_history", "english", "quoted_distraction" ], "error": null }, { "id": "0be6f5bade803bbcdd3b", "expected": "general", "predicted": "coding", "confidence": 0.9148, "tags": [ "english", "quoted_distraction", "science_and_environment" ], "error": null }, { "id": "241efe3ded101ae619cb", "expected": "reasoning", "predicted": "summarization", "confidence": 0.5415, "tags": [ "english", "home_and_travel", "quoted_distraction" ], "error": null }, { "id": "2cce175108805f6828d3", "expected": "summarization", "predicted": "general", "confidence": 0.5657, "tags": [ "english", "home_and_travel", "keyword_collision" ], "error": null }, { "id": "11de30d98dc3356d6fc7", "expected": "reasoning", "predicted": "general", "confidence": 0.3528, "tags": [ "english", "keyword_collision", "workplace_and_business" ], "error": null }, { "id": "3ab5a7b2dc7b9498ee4d", "expected": "math", "predicted": "general", "confidence": 0.797, "tags": [ "english", "explicit", "software_and_data" ], "error": null }, { "id": "42568acd8cec8efe0b3c", "expected": "reasoning", "predicted": "general", "confidence": 0.7128, "tags": [ "english", "implicit", "workplace_and_business" ], "error": null }, { "id": "7563ef4bfbc2207133f4", "expected": "reasoning", "predicted": "general", "confidence": 0.4578, "tags": [ "english", "explicit", "health_and_wellbeing" ], "error": null }, { "id": "04a04e71fe6a88a707e2", "expected": "summarization", "predicted": "coding", "confidence": 0.926, "tags": [ "english", "keyword_collision", "software_and_data" ], "error": null }, { "id": "5806547483dbfefd47d3", "expected": "summarization", "predicted": "coding", "confidence": 0.3614, "tags": [ "english", "implicit", "software_and_data" ], "error": null }, { "id": "c46a284ef6b548e06f66", "expected": "math", "predicted": "creative_writing", "confidence": 0.4639, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "15a225ff569ecd16a83b", "expected": "general", "predicted": "creative_writing", "confidence": 0.9865, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "073f663035685d8cd67b", "expected": "math", "predicted": "general", "confidence": 0.4243, "tags": [ "education_and_history", "english", "quoted_distraction" ], "error": null }, { "id": "1b4c6ec103665932ecc2", "expected": "reasoning", "predicted": "general", "confidence": 0.3973, "tags": [ "english", "explicit", "retail_and_support" ], "error": null }, { "id": "2079a105982416677b9f", "expected": "reasoning", "predicted": "general", "confidence": 0.7584, "tags": [ "english", "implicit", "science_and_environment" ], "error": null }, { "id": "7a808fb2da45604b936b", "expected": "general", "predicted": "coding", "confidence": 0.6717, "tags": [ "english", "quoted_distraction", "science_and_environment" ], "error": null }, { "id": "64db9b9347702bdfd601", "expected": "reasoning", "predicted": "general", "confidence": 0.7207, "tags": [ "english", "home_and_travel", "implicit" ], "error": null }, { "id": "82b2ae5b00aeb0a9b7d3", "expected": "summarization", "predicted": "general", "confidence": 0.4329, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "0f88f56700a25bdc5ade", "expected": "math", "predicted": "general", "confidence": 0.3016, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "28b24fee3d60f36e4262", "expected": "summarization", "predicted": "creative_writing", "confidence": 0.3542, "tags": [ "education_and_history", "english", "implicit" ], "error": null }, { "id": "3f1152b7a9865caff6d1", "expected": "math", "predicted": "summarization", "confidence": 0.396, "tags": [ "english", "quoted_distraction", "sports_and_leisure" ], "error": null }, { "id": "99718a62d463917596b9", "expected": "math", "predicted": "coding", "confidence": 0.6337, "tags": [ "english", "keyword_collision", "sports_and_leisure" ], "error": null }, { "id": "0a51567358fa13497148", "expected": "reasoning", "predicted": "general", "confidence": 0.4746, "tags": [ "english", "quoted_distraction", "retail_and_support" ], "error": null }, { "id": "5c143d5a8da9ac05ecad", "expected": "math", "predicted": "reasoning", "confidence": 0.3583, "tags": [ "english", "health_and_wellbeing", "quoted_distraction" ], "error": null }, { "id": "8dead2a7992843ded750", "expected": "coding", "predicted": "math", "confidence": 0.5966, "tags": [ "english", "keyword_collision", "science_and_environment" ], "error": null }, { "id": "bb0e2df9b40eb15ca542", "expected": "reasoning", "predicted": "general", "confidence": 0.3776, "tags": [ "education_and_history", "english", "keyword_collision" ], "error": null }, { "id": "80b8205ff85542f097c0", "expected": "summarization", "predicted": "coding", "confidence": 0.5612, "tags": [ "english", "implicit", "software_and_data" ], "error": null }, { "id": "0b0ce0600ea71df829a6", "expected": "reasoning", "predicted": "general", "confidence": 0.7749, "tags": [ "english", "home_and_travel", "keyword_collision" ], "error": null }, { "id": "1b83610098aead6d79a8", "expected": "general", "predicted": "creative_writing", "confidence": 0.7566, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "e5613c647a6945127bd0", "expected": "reasoning", "predicted": "general", "confidence": 0.5398, "tags": [ "english", "quoted_distraction", "retail_and_support" ], "error": null }, { "id": "d67a4fa9a57e65ad8788", "expected": "math", "predicted": "general", "confidence": 0.4377, "tags": [ "english", "home_and_travel", "quoted_distraction" ], "error": null }, { "id": "d872f9c9d393d43ae8fa", "expected": "math", "predicted": "general", "confidence": 0.497, "tags": [ "english", "home_and_travel", "implicit" ], "error": null }, { "id": "a7d0191cbcd6bd999a89", "expected": "summarization", "predicted": "general", "confidence": 0.4408, "tags": [ "english", "implicit", "science_and_environment" ], "error": null }, { "id": "d41ad266a6d363f201ef", "expected": "math", "predicted": "reasoning", "confidence": 0.5758, "tags": [ "english", "home_and_travel", "implicit" ], "error": null }, { "id": "fb8fe76e1701d45c3b01", "expected": "general", "predicted": "coding", "confidence": 0.7698, "tags": [ "english", "keyword_collision", "public_services" ], "error": null }, { "id": "2a3effdea00a1bc75a20", "expected": "summarization", "predicted": "creative_writing", "confidence": 0.9443, "tags": [ "education_and_history", "english", "quoted_distraction" ], "error": null }, { "id": "582b7ecfc374940f96d0", "expected": "summarization", "predicted": "creative_writing", "confidence": 0.392, "tags": [ "english", "keyword_collision", "public_services" ], "error": null }, { "id": "0034500228776613ce0b", "expected": "coding", "predicted": "reasoning", "confidence": 0.5251, "tags": [ "education_and_history", "english", "keyword_collision" ], "error": null }, { "id": "88f682a0835b3dcd892d", "expected": "summarization", "predicted": "creative_writing", "confidence": 0.5068, "tags": [ "english", "quoted_distraction", "sports_and_leisure" ], "error": null }, { "id": "1dd48cce991abd916f1f", "expected": "reasoning", "predicted": "general", "confidence": 0.7341, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "e23237c7ab336cc55983", "expected": "summarization", "predicted": "general", "confidence": 0.3251, "tags": [ "english", "home_and_travel", "implicit" ], "error": null }, { "id": "af31a5f67eec66aaa29a", "expected": "coding", "predicted": "creative_writing", "confidence": 0.9167, "tags": [ "english", "health_and_wellbeing", "quoted_distraction" ], "error": null }, { "id": "746af810f07c8a200231", "expected": "math", "predicted": "general", "confidence": 0.6077, "tags": [ "education_and_history", "english", "implicit" ], "error": null }, { "id": "cff40d04de4707191283", "expected": "summarization", "predicted": "creative_writing", "confidence": 0.4418, "tags": [ "english", "implicit", "science_and_environment" ], "error": null }, { "id": "6179e53c41951b7f93da", "expected": "math", "predicted": "coding", "confidence": 0.839, "tags": [ "english", "keyword_collision", "software_and_data" ], "error": null }, { "id": "f01a9d70456a77248fce", "expected": "math", "predicted": "general", "confidence": 0.726, "tags": [ "english", "implicit", "software_and_data" ], "error": null }, { "id": "1d8702ec94a85453308b", "expected": "summarization", "predicted": "creative_writing", "confidence": 0.8648, "tags": [ "arts_and_media", "english", "keyword_collision" ], "error": null }, { "id": "e11ed35a9751357a5913", "expected": "summarization", "predicted": "creative_writing", "confidence": 0.955, "tags": [ "english", "quoted_distraction", "retail_and_support" ], "error": null }, { "id": "bcf2a162a6d48fe8a2d0", "expected": "coding", "predicted": "math", "confidence": 0.5158, "tags": [ "english", "home_and_travel", "keyword_collision" ], "error": null }, { "id": "83cd435b7e45907f9716", "expected": "math", "predicted": "general", "confidence": 0.3952, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "c9db8b4932fd15a78052", "expected": "math", "predicted": "reasoning", "confidence": 0.392, "tags": [ "english", "health_and_wellbeing", "implicit" ], "error": null }, { "id": "9eafd0ce696b2167119d", "expected": "math", "predicted": "general", "confidence": 0.4398, "tags": [ "arts_and_media", "english", "implicit" ], "error": null }, { "id": "d3af9cdb773a1957795e", "expected": "general", "predicted": "summarization", "confidence": 0.9778, "tags": [ "english", "quoted_distraction", "software_and_data" ], "error": null }, { "id": "dd751f0f2d5bd15a2c18", "expected": "reasoning", "predicted": "creative_writing", "confidence": 0.3595, "tags": [ "english", "health_and_wellbeing", "keyword_collision" ], "error": null }, { "id": "9993d1d5aa1248a8f422", "expected": "math", "predicted": "general", "confidence": 0.5065, "tags": [ "english", "keyword_collision", "software_and_data" ], "error": null }, { "id": "22fe52b70034d0630ef0", "expected": "summarization", "predicted": "general", "confidence": 0.3549, "tags": [ "english", "keyword_collision", "science_and_environment" ], "error": null }, { "id": "c75b289c656134938367", "expected": "general", "predicted": "creative_writing", "confidence": 0.8303, "tags": [ "english", "keyword_collision", "science_and_environment" ], "error": null }, { "id": "168bd22a2f90cba24345", "expected": "math", "predicted": "general", "confidence": 0.5403, "tags": [ "arts_and_media", "english", "implicit" ], "error": null }, { "id": "a8735bd0cbf69521d0a0", "expected": "coding", "predicted": "general", "confidence": 0.4182, "tags": [ "education_and_history", "english", "implicit" ], "error": null }, { "id": "2a7f1edfefb2342759b8", "expected": "reasoning", "predicted": "general", "confidence": 0.7998, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "c01ba1ff99d62992d182", "expected": "reasoning", "predicted": "general", "confidence": 0.5157, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "ad3181675a78f656f584", "expected": "math", "predicted": "general", "confidence": 0.5431, "tags": [ "english", "explicit", "health_and_wellbeing" ], "error": null }, { "id": "7566ac716621c468f791", "expected": "summarization", "predicted": "coding", "confidence": 0.8873, "tags": [ "english", "quoted_distraction", "workplace_and_business" ], "error": null }, { "id": "c3f9e52afdfb30061f8c", "expected": "general", "predicted": "coding", "confidence": 0.6201, "tags": [ "english", "keyword_collision", "public_services" ], "error": null }, { "id": "a492c8230050d71d31fb", "expected": "math", "predicted": "general", "confidence": 0.3422, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "bd6b5f5d19f6da08bbff", "expected": "general", "predicted": "creative_writing", "confidence": 0.4541, "tags": [ "education_and_history", "english", "keyword_collision" ], "error": null }, { "id": "36b2aacfff42ac2fda3e", "expected": "summarization", "predicted": "creative_writing", "confidence": 0.9212, "tags": [ "english", "quoted_distraction", "science_and_environment" ], "error": null }, { "id": "2ee9da25c537bf08d45b", "expected": "reasoning", "predicted": "general", "confidence": 0.6358, "tags": [ "english", "explicit", "science_and_environment" ], "error": null }, { "id": "137197c52518d04a1851", "expected": "math", "predicted": "general", "confidence": 0.519, "tags": [ "english", "explicit", "sports_and_leisure" ], "error": null }, { "id": "356d606db5145c551091", "expected": "math", "predicted": "coding", "confidence": 0.6349, "tags": [ "english", "implicit", "software_and_data" ], "error": null }, { "id": "a3d09721dc54c7aa23e5", "expected": "summarization", "predicted": "general", "confidence": 0.5825, "tags": [ "english", "implicit", "software_and_data" ], "error": null }, { "id": "fbc9291d970bdd23e823", "expected": "reasoning", "predicted": "coding", "confidence": 0.4644, "tags": [ "english", "keyword_collision", "software_and_data" ], "error": null }, { "id": "39c9c27b895d27f6ca67", "expected": "reasoning", "predicted": "general", "confidence": 0.347, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "f3c735797b200b0ff388", "expected": "summarization", "predicted": "general", "confidence": 0.5947, "tags": [ "english", "home_and_travel", "implicit" ], "error": null }, { "id": "c6ab4a0cb09ffd902062", "expected": "creative_writing", "predicted": "summarization", "confidence": 0.7121, "tags": [ "education_and_history", "english", "quoted_distraction" ], "error": null }, { "id": "95fac4f60746f89097e3", "expected": "math", "predicted": "reasoning", "confidence": 0.4259, "tags": [ "english", "explicit", "retail_and_support" ], "error": null }, { "id": "2c9cb4ec13f386670ac1", "expected": "creative_writing", "predicted": "coding", "confidence": 0.8028, "tags": [ "education_and_history", "english", "keyword_collision" ], "error": null }, { "id": "95fe893d16587da4c304", "expected": "reasoning", "predicted": "general", "confidence": 0.5633, "tags": [ "english", "quoted_distraction", "workplace_and_business" ], "error": null }, { "id": "a07bc61d3330fd09c761", "expected": "summarization", "predicted": "creative_writing", "confidence": 0.4012, "tags": [ "arts_and_media", "english", "explicit" ], "error": null }, { "id": "b61ac98d9fc06327314f", "expected": "summarization", "predicted": "general", "confidence": 0.3465, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "a6e98cf3b4cacd4aa15f", "expected": "general", "predicted": "coding", "confidence": 0.8292, "tags": [ "english", "keyword_collision", "software_and_data" ], "error": null }, { "id": "ce2c8127d9919f54e516", "expected": "general", "predicted": "reasoning", "confidence": 0.5262, "tags": [ "english", "keyword_collision", "retail_and_support" ], "error": null }, { "id": "b6f766624531547d3e51", "expected": "math", "predicted": "general", "confidence": 0.4218, "tags": [ "education_and_history", "english", "implicit" ], "error": null }, { "id": "50cf5e4b45cbf4d15e0e", "expected": "coding", "predicted": "summarization", "confidence": 0.2384, "tags": [ "english", "explicit", "retail_and_support" ], "error": null }, { "id": "7c9f8dcb26520d798034", "expected": "math", "predicted": "summarization", "confidence": 0.9182, "tags": [ "english", "keyword_collision", "retail_and_support" ], "error": null }, { "id": "19ec8530610803bfa9ed", "expected": "summarization", "predicted": "creative_writing", "confidence": 0.4261, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "dbc5b6eaaebd8ca86141", "expected": "coding", "predicted": "creative_writing", "confidence": 0.9903, "tags": [ "english", "public_services", "quoted_distraction" ], "error": null }, { "id": "fb116115b5eaeb78b7d6", "expected": "reasoning", "predicted": "summarization", "confidence": 0.9681, "tags": [ "english", "quoted_distraction", "science_and_environment" ], "error": null }, { "id": "ee7eae2defca05629587", "expected": "summarization", "predicted": "general", "confidence": 0.4004, "tags": [ "english", "implicit", "workplace_and_business" ], "error": null }, { "id": "2e9f75b565c09e38ba99", "expected": "reasoning", "predicted": "general", "confidence": 0.5647, "tags": [ "english", "health_and_wellbeing", "quoted_distraction" ], "error": null }, { "id": "2fcf0f95bafa8322107f", "expected": "summarization", "predicted": "general", "confidence": 0.4009, "tags": [ "arts_and_media", "english", "keyword_collision" ], "error": null }, { "id": "82f634ae9419d72b41aa", "expected": "reasoning", "predicted": "general", "confidence": 0.6181, "tags": [ "education_and_history", "english", "implicit" ], "error": null }, { "id": "28f5c4eb8ecac426e8a1", "expected": "reasoning", "predicted": "creative_writing", "confidence": 0.9962, "tags": [ "english", "public_services", "quoted_distraction" ], "error": null }, { "id": "0195b8853a9ac1946077", "expected": "summarization", "predicted": "creative_writing", "confidence": 0.4196, "tags": [ "english", "keyword_collision", "public_services" ], "error": null }, { "id": "fbfa779d5c110511ae6d", "expected": "reasoning", "predicted": "creative_writing", "confidence": 0.5816, "tags": [ "education_and_history", "english", "quoted_distraction" ], "error": null }, { "id": "4b9de10b8409d8a18158", "expected": "math", "predicted": "general", "confidence": 0.3353, "tags": [ "english", "implicit", "workplace_and_business" ], "error": null }, { "id": "9af9c98619b1dd1c7938", "expected": "reasoning", "predicted": "creative_writing", "confidence": 0.5327, "tags": [ "education_and_history", "english", "quoted_distraction" ], "error": null }, { "id": "203b5ecd78c4ae28b73d", "expected": "math", "predicted": "general", "confidence": 0.2902, "tags": [ "english", "implicit", "workplace_and_business" ], "error": null }, { "id": "9b95af368c8da504cf36", "expected": "summarization", "predicted": "general", "confidence": 0.4706, "tags": [ "english", "home_and_travel", "implicit" ], "error": null }, { "id": "be87b66f1a4177f7b2d1", "expected": "general", "predicted": "coding", "confidence": 0.3358, "tags": [ "english", "keyword_collision", "software_and_data" ], "error": null }, { "id": "46f712e5440d46598940", "expected": "math", "predicted": "summarization", "confidence": 0.604, "tags": [ "english", "keyword_collision", "public_services" ], "error": null }, { "id": "3bef78cba49297d29eb4", "expected": "reasoning", "predicted": "general", "confidence": 0.325, "tags": [ "english", "keyword_collision", "sports_and_leisure" ], "error": null }, { "id": "42ac07c6888290378b2d", "expected": "reasoning", "predicted": "general", "confidence": 0.5602, "tags": [ "english", "explicit", "workplace_and_business" ], "error": null }, { "id": "1893c74cd232241fd977", "expected": "summarization", "predicted": "general", "confidence": 0.5348, "tags": [ "english", "implicit", "public_services" ], "error": null }, { "id": "09b263482370f5ee454e", "expected": "coding", "predicted": "general", "confidence": 0.4114, "tags": [ "arts_and_media", "english", "implicit" ], "error": null }, { "id": "119d90ed93df45ee84e3", "expected": "reasoning", "predicted": "general", "confidence": 0.6363, "tags": [ "english", "explicit", "home_and_travel" ], "error": null }, { "id": "1c2d6e66e6467c5357d4", "expected": "summarization", "predicted": "creative_writing", "confidence": 0.2699, "tags": [ "english", "health_and_wellbeing", "implicit" ], "error": null }, { "id": "83012f85dabaaec55095", "expected": "general", "predicted": "creative_writing", "confidence": 0.9993, "tags": [ "english", "quoted_distraction", "sports_and_leisure" ], "error": null }, { "id": "beb5f2c57b6fd758b862", "expected": "reasoning", "predicted": "general", "confidence": 0.7754, "tags": [ "english", "home_and_travel", "implicit" ], "error": null }, { "id": "6f3f88ca5c2bcebaa348", "expected": "math", "predicted": "general", "confidence": 0.6245, "tags": [ "english", "implicit", "public_services" ], "error": null }, { "id": "8ace2a5095414dac9e3d", "expected": "reasoning", "predicted": "general", "confidence": 0.4434, "tags": [ "english", "implicit", "science_and_environment" ], "error": null }, { "id": "cf6723ce775961f3a0b7", "expected": "summarization", "predicted": "general", "confidence": 0.46, "tags": [ "english", "health_and_wellbeing", "keyword_collision" ], "error": null }, { "id": "7c24b468d9684cc50901", "expected": "summarization", "predicted": "general", "confidence": 0.9282, "tags": [ "english", "home_and_travel", "keyword_collision" ], "error": null }, { "id": "7f7dc9444db87e6ee067", "expected": "math", "predicted": "general", "confidence": 0.3595, "tags": [ "english", "explicit", "health_and_wellbeing" ], "error": null }, { "id": "0fe5cf11594460deced5", "expected": "math", "predicted": "creative_writing", "confidence": 0.9946, "tags": [ "english", "quoted_distraction", "workplace_and_business" ], "error": null }, { "id": "9b6c9257e4e4e1a25dad", "expected": "reasoning", "predicted": "general", "confidence": 0.4373, "tags": [ "english", "explicit", "software_and_data" ], "error": null }, { "id": "835e4a244a5fa09e02d4", "expected": "reasoning", "predicted": "general", "confidence": 0.618, "tags": [ "english", "implicit", "workplace_and_business" ], "error": null }, { "id": "19941a330ebfc696035a", "expected": "general", "predicted": "creative_writing", "confidence": 0.9907, "tags": [ "english", "health_and_wellbeing", "quoted_distraction" ], "error": null }, { "id": "f6e19ef742e837419fae", "expected": "summarization", "predicted": "general", "confidence": 0.4688, "tags": [ "english", "keyword_collision", "public_services" ], "error": null }, { "id": "09585255255364a13cd0", "expected": "coding", "predicted": "general", "confidence": 0.3492, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "41be605dcd05dd815139", "expected": "reasoning", "predicted": "general", "confidence": 0.5346, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "f6ab0ff7558938e68cb2", "expected": "reasoning", "predicted": "creative_writing", "confidence": 0.7482, "tags": [ "english", "health_and_wellbeing", "keyword_collision" ], "error": null }, { "id": "439ce0ab2b6ce969a6fd", "expected": "summarization", "predicted": "general", "confidence": 0.625, "tags": [ "english", "implicit", "public_services" ], "error": null }, { "id": "1da8c489eb382f741929", "expected": "summarization", "predicted": "general", "confidence": 0.4738, "tags": [ "english", "health_and_wellbeing", "quoted_distraction" ], "error": null }, { "id": "2665fa859232307228fe", "expected": "general", "predicted": "coding", "confidence": 0.8136, "tags": [ "english", "public_services", "quoted_distraction" ], "error": null }, { "id": "e3449496d500a2014736", "expected": "summarization", "predicted": "reasoning", "confidence": 0.3352, "tags": [ "english", "keyword_collision", "science_and_environment" ], "error": null }, { "id": "56906d5a3796f985cd07", "expected": "math", "predicted": "general", "confidence": 0.7888, "tags": [ "arts_and_media", "english", "keyword_collision" ], "error": null }, { "id": "c5ae42d1f6f9e09cc987", "expected": "reasoning", "predicted": "general", "confidence": 0.5028, "tags": [ "english", "implicit", "public_services" ], "error": null }, { "id": "c4973445663e2050358e", "expected": "reasoning", "predicted": "general", "confidence": 0.773, "tags": [ "english", "keyword_collision", "retail_and_support" ], "error": null }, { "id": "b50e762ed7f64d3516a0", "expected": "reasoning", "predicted": "general", "confidence": 0.8262, "tags": [ "english", "home_and_travel", "quoted_distraction" ], "error": null }, { "id": "872df632207f40af3414", "expected": "math", "predicted": "general", "confidence": 0.5582, "tags": [ "english", "keyword_collision", "software_and_data" ], "error": null }, { "id": "47df61fe6ebf44aa7d74", "expected": "coding", "predicted": "general", "confidence": 0.3629, "tags": [ "english", "home_and_travel", "implicit" ], "error": null }, { "id": "16c68065c512a78b06ea", "expected": "reasoning", "predicted": "general", "confidence": 0.5032, "tags": [ "english", "keyword_collision", "retail_and_support" ], "error": null }, { "id": "6e1fb90a7937f6db1d34", "expected": "summarization", "predicted": "general", "confidence": 0.3763, "tags": [ "english", "home_and_travel", "keyword_collision" ], "error": null }, { "id": "cd3ad6e3b171cdbd9282", "expected": "reasoning", "predicted": "general", "confidence": 0.4791, "tags": [ "english", "home_and_travel", "keyword_collision" ], "error": null }, { "id": "7f48179d7d548473a00f", "expected": "general", "predicted": "coding", "confidence": 0.6573, "tags": [ "english", "explicit", "software_and_data" ], "error": null }, { "id": "1a5b9e39b10f56af0a43", "expected": "reasoning", "predicted": "coding", "confidence": 0.8507, "tags": [ "education_and_history", "english", "keyword_collision" ], "error": null }, { "id": "40e07c71af0efd9b276d", "expected": "general", "predicted": "coding", "confidence": 0.6601, "tags": [ "english", "keyword_collision", "sports_and_leisure" ], "error": null }, { "id": "d9c03bdb09db3ee46782", "expected": "summarization", "predicted": "general", "confidence": 0.5267, "tags": [ "english", "keyword_collision", "science_and_environment" ], "error": null }, { "id": "188ad75336e020ca44e2", "expected": "general", "predicted": "creative_writing", "confidence": 0.7223, "tags": [ "english", "explicit", "science_and_environment" ], "error": null }, { "id": "fa3e209829382d0a0079", "expected": "math", "predicted": "general", "confidence": 0.3677, "tags": [ "education_and_history", "english", "explicit" ], "error": null }, { "id": "78b7dde5e1bc6c2ed5f3", "expected": "reasoning", "predicted": "general", "confidence": 0.5181, "tags": [ "english", "home_and_travel", "implicit" ], "error": null }, { "id": "731ea10face531ad50ec", "expected": "summarization", "predicted": "general", "confidence": 0.8795, "tags": [ "english", "home_and_travel", "implicit" ], "error": null }, { "id": "3e98b22d7df9d0a13214", "expected": "reasoning", "predicted": "coding", "confidence": 0.4488, "tags": [ "english", "keyword_collision", "sports_and_leisure" ], "error": null }, { "id": "be1d66a4d2ff1afcf414", "expected": "reasoning", "predicted": "creative_writing", "confidence": 0.9978, "tags": [ "education_and_history", "english", "quoted_distraction" ], "error": null }, { "id": "af7c7b95adf86fcfacde", "expected": "reasoning", "predicted": "general", "confidence": 0.4951, "tags": [ "english", "public_services", "quoted_distraction" ], "error": null }, { "id": "590af5d2c38389a02f2a", "expected": "reasoning", "predicted": "general", "confidence": 0.4273, "tags": [ "english", "explicit", "public_services" ], "error": null }, { "id": "63a81c73d0cc5fc350f1", "expected": "creative_writing", "predicted": "coding", "confidence": 0.5581, "tags": [ "english", "explicit", "software_and_data" ], "error": null }, { "id": "546db3471f77e0f59eaa", "expected": "math", "predicted": "reasoning", "confidence": 0.7661, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "141551a6f71b9146567a", "expected": "math", "predicted": "general", "confidence": 0.3734, "tags": [ "english", "keyword_collision", "workplace_and_business" ], "error": null }, { "id": "49dc8a421cf27db6796b", "expected": "coding", "predicted": "math", "confidence": 0.9145, "tags": [ "education_and_history", "english", "quoted_distraction" ], "error": null }, { "id": "1ba1ecac79192724fcc5", "expected": "math", "predicted": "general", "confidence": 0.6346, "tags": [ "education_and_history", "english", "explicit" ], "error": null }, { "id": "6aa95a3d06108b020a26", "expected": "reasoning", "predicted": "creative_writing", "confidence": 0.5247, "tags": [ "english", "quoted_distraction", "science_and_environment" ], "error": null }, { "id": "d5764d525bd61e8d1b73", "expected": "reasoning", "predicted": "general", "confidence": 0.4994, "tags": [ "arts_and_media", "english", "explicit" ], "error": null }, { "id": "0829d305f71eae0886cd", "expected": "coding", "predicted": "summarization", "confidence": 0.5128, "tags": [ "education_and_history", "english", "keyword_collision" ], "error": null }, { "id": "eee784ffdef19da3cd57", "expected": "general", "predicted": "summarization", "confidence": 0.7743, "tags": [ "english", "quoted_distraction", "workplace_and_business" ], "error": null }, { "id": "20bf3adbf974f1aacb80", "expected": "math", "predicted": "coding", "confidence": 0.3671, "tags": [ "english", "quoted_distraction", "software_and_data" ], "error": null }, { "id": "58f21bc291bc998b894b", "expected": "summarization", "predicted": "general", "confidence": 0.413, "tags": [ "english", "implicit", "public_services" ], "error": null }, { "id": "57b5e40b994d321b4145", "expected": "summarization", "predicted": "general", "confidence": 0.4244, "tags": [ "english", "keyword_collision", "retail_and_support" ], "error": null }, { "id": "37e8b426c18514cd65b1", "expected": "summarization", "predicted": "creative_writing", "confidence": 0.4364, "tags": [ "english", "keyword_collision", "sports_and_leisure" ], "error": null }, { "id": "23bf6c4ebeb9d6c37004", "expected": "creative_writing", "predicted": "summarization", "confidence": 0.5299, "tags": [ "english", "quoted_distraction", "retail_and_support" ], "error": null }, { "id": "e5b82b29d27a8e485db1", "expected": "creative_writing", "predicted": "general", "confidence": 0.3999, "tags": [ "english", "implicit", "workplace_and_business" ], "error": null }, { "id": "29d416ba1b10c8cf3493", "expected": "reasoning", "predicted": "general", "confidence": 0.5702, "tags": [ "arts_and_media", "english", "implicit" ], "error": null }, { "id": "b4ecc03b78cb71d0fa1d", "expected": "reasoning", "predicted": "general", "confidence": 0.6245, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "dc7f8c5082a153c49621", "expected": "reasoning", "predicted": "summarization", "confidence": 0.6607, "tags": [ "english", "public_services", "quoted_distraction" ], "error": null }, { "id": "866a22e24bc847837717", "expected": "summarization", "predicted": "reasoning", "confidence": 0.3032, "tags": [ "education_and_history", "english", "implicit" ], "error": null }, { "id": "b171cc036a497955621d", "expected": "reasoning", "predicted": "general", "confidence": 0.7646, "tags": [ "english", "home_and_travel", "quoted_distraction" ], "error": null }, { "id": "7085f44b6a5ea931ae3d", "expected": "summarization", "predicted": "creative_writing", "confidence": 0.8662, "tags": [ "english", "implicit", "science_and_environment" ], "error": null }, { "id": "ae7aff453f9ca2901a2e", "expected": "math", "predicted": "reasoning", "confidence": 0.4974, "tags": [ "english", "implicit", "science_and_environment" ], "error": null }, { "id": "bfaf32d0d5e2c4a3fb20", "expected": "math", "predicted": "general", "confidence": 0.3102, "tags": [ "english", "explicit", "public_services" ], "error": null }, { "id": "8012c138de05e430061e", "expected": "summarization", "predicted": "creative_writing", "confidence": 0.5647, "tags": [ "english", "quoted_distraction", "sports_and_leisure" ], "error": null }, { "id": "84f586d5f97e2def57cd", "expected": "math", "predicted": "general", "confidence": 0.3774, "tags": [ "english", "implicit", "workplace_and_business" ], "error": null }, { "id": "6c7b54d80cc64ed3bb08", "expected": "math", "predicted": "general", "confidence": 0.5204, "tags": [ "english", "home_and_travel", "implicit" ], "error": null }, { "id": "7b24c403412b0b80a003", "expected": "reasoning", "predicted": "general", "confidence": 0.5033, "tags": [ "english", "keyword_collision", "sports_and_leisure" ], "error": null }, { "id": "0743eeb5413ea1274d85", "expected": "math", "predicted": "general", "confidence": 0.4712, "tags": [ "english", "explicit", "public_services" ], "error": null }, { "id": "e91cd530fb33d1051772", "expected": "coding", "predicted": "general", "confidence": 0.4798, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "46dabebfa9cf860d65b7", "expected": "math", "predicted": "general", "confidence": 0.4982, "tags": [ "english", "implicit", "software_and_data" ], "error": null }, { "id": "2aa99f805e32e60ba81a", "expected": "math", "predicted": "general", "confidence": 0.235, "tags": [ "arts_and_media", "english", "keyword_collision" ], "error": null }, { "id": "5fb517539877682f199e", "expected": "math", "predicted": "general", "confidence": 0.3882, "tags": [ "arts_and_media", "english", "explicit" ], "error": null }, { "id": "8347fecebb46b58e98cc", "expected": "reasoning", "predicted": "general", "confidence": 0.4577, "tags": [ "education_and_history", "english", "explicit" ], "error": null }, { "id": "ffa7fa26566ae7a73259", "expected": "summarization", "predicted": "general", "confidence": 0.3778, "tags": [ "education_and_history", "english", "implicit" ], "error": null }, { "id": "874b17d4150dbdbef38d", "expected": "math", "predicted": "general", "confidence": 0.4409, "tags": [ "english", "implicit", "public_services" ], "error": null }, { "id": "200a8ac01d48f005eaa2", "expected": "summarization", "predicted": "coding", "confidence": 0.5194, "tags": [ "english", "explicit", "software_and_data" ], "error": null }, { "id": "30dc803e3b2879054360", "expected": "summarization", "predicted": "general", "confidence": 0.6591, "tags": [ "english", "implicit", "public_services" ], "error": null }, { "id": "8688f0e21e450eef2de5", "expected": "math", "predicted": "general", "confidence": 0.4067, "tags": [ "english", "health_and_wellbeing", "quoted_distraction" ], "error": null }, { "id": "4ff11dc008db970d1f08", "expected": "coding", "predicted": "general", "confidence": 0.4246, "tags": [ "english", "implicit", "software_and_data" ], "error": null }, { "id": "90bff28e81552b97e438", "expected": "math", "predicted": "general", "confidence": 0.6021, "tags": [ "english", "public_services", "quoted_distraction" ], "error": null }, { "id": "7a54e4ec7e62e159203c", "expected": "summarization", "predicted": "creative_writing", "confidence": 0.4912, "tags": [ "english", "keyword_collision", "sports_and_leisure" ], "error": null }, { "id": "38fa57e835cade3b7ee8", "expected": "math", "predicted": "summarization", "confidence": 0.653, "tags": [ "english", "home_and_travel", "keyword_collision" ], "error": null }, { "id": "801c183238931341eef5", "expected": "summarization", "predicted": "general", "confidence": 0.3762, "tags": [ "english", "health_and_wellbeing", "implicit" ], "error": null }, { "id": "5303fce092eb5c606dc9", "expected": "coding", "predicted": "summarization", "confidence": 0.5383, "tags": [ "education_and_history", "english", "quoted_distraction" ], "error": null }, { "id": "bea9e1f2e8f7427a1245", "expected": "coding", "predicted": "general", "confidence": 0.4984, "tags": [ "english", "quoted_distraction", "workplace_and_business" ], "error": null }, { "id": "7935c5ca2dcd865f96d4", "expected": "math", "predicted": "general", "confidence": 0.7173, "tags": [ "english", "implicit", "public_services" ], "error": null }, { "id": "5309d54fb82a34748587", "expected": "summarization", "predicted": "creative_writing", "confidence": 0.4639, "tags": [ "education_and_history", "english", "keyword_collision" ], "error": null }, { "id": "fcbbeafc82200cb94800", "expected": "math", "predicted": "coding", "confidence": 0.6815, "tags": [ "english", "keyword_collision", "software_and_data" ], "error": null }, { "id": "23866ffb91c420cd9118", "expected": "reasoning", "predicted": "general", "confidence": 0.7141, "tags": [ "english", "implicit", "workplace_and_business" ], "error": null }, { "id": "92348d63c89724bf9bde", "expected": "general", "predicted": "creative_writing", "confidence": 0.6836, "tags": [ "education_and_history", "english", "quoted_distraction" ], "error": null }, { "id": "5b7265b1fe7caadc5985", "expected": "math", "predicted": "general", "confidence": 0.3211, "tags": [ "education_and_history", "english", "quoted_distraction" ], "error": null }, { "id": "8566945e918cf6d679ad", "expected": "reasoning", "predicted": "general", "confidence": 0.4395, "tags": [ "english", "implicit", "public_services" ], "error": null }, { "id": "7d7ed7448c6bfd21e150", "expected": "summarization", "predicted": "reasoning", "confidence": 0.6562, "tags": [ "english", "keyword_collision", "sports_and_leisure" ], "error": null }, { "id": "12272f9cde9cbaf8c39a", "expected": "coding", "predicted": "math", "confidence": 0.6755, "tags": [ "english", "keyword_collision", "retail_and_support" ], "error": null }, { "id": "c74695ea244f3b80a3c0", "expected": "math", "predicted": "general", "confidence": 0.2891, "tags": [ "english", "quoted_distraction", "sports_and_leisure" ], "error": null }, { "id": "525dc384a3d430f54265", "expected": "math", "predicted": "coding", "confidence": 0.5542, "tags": [ "english", "health_and_wellbeing", "keyword_collision" ], "error": null }, { "id": "bb27e1008c3991db1fb1", "expected": "reasoning", "predicted": "general", "confidence": 0.552, "tags": [ "english", "implicit", "public_services" ], "error": null }, { "id": "581527a786d856c205fe", "expected": "coding", "predicted": "math", "confidence": 0.7238, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "642db1bf76a8216db9e0", "expected": "summarization", "predicted": "coding", "confidence": 0.7878, "tags": [ "english", "keyword_collision", "software_and_data" ], "error": null }, { "id": "8010f179c8265849f665", "expected": "math", "predicted": "general", "confidence": 0.588, "tags": [ "english", "explicit", "software_and_data" ], "error": null }, { "id": "377cfee96123d41d6332", "expected": "general", "predicted": "coding", "confidence": 0.5772, "tags": [ "english", "explicit", "retail_and_support" ], "error": null }, { "id": "1ea1cb699c1759141306", "expected": "summarization", "predicted": "general", "confidence": 0.6425, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "8e00883cb7168bd15bbc", "expected": "coding", "predicted": "general", "confidence": 0.4793, "tags": [ "english", "health_and_wellbeing", "quoted_distraction" ], "error": null }, { "id": "beb1a86dd8353cd5ca72", "expected": "reasoning", "predicted": "general", "confidence": 0.5096, "tags": [ "english", "keyword_collision", "public_services" ], "error": null }, { "id": "dac639aad51a840e6bcd", "expected": "coding", "predicted": "math", "confidence": 0.5597, "tags": [ "education_and_history", "english", "keyword_collision" ], "error": null }, { "id": "1573b989b993b5f40975", "expected": "summarization", "predicted": "general", "confidence": 0.4906, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "e9031f07af843da769c9", "expected": "math", "predicted": "general", "confidence": 0.438, "tags": [ "english", "health_and_wellbeing", "quoted_distraction" ], "error": null }, { "id": "21f9df87dbeaf71e1425", "expected": "summarization", "predicted": "creative_writing", "confidence": 0.5769, "tags": [ "english", "quoted_distraction", "science_and_environment" ], "error": null }, { "id": "2e6adb0dc3df4b196ef2", "expected": "summarization", "predicted": "coding", "confidence": 0.3758, "tags": [ "english", "implicit", "software_and_data" ], "error": null }, { "id": "9ec9ff018185d81d81cd", "expected": "reasoning", "predicted": "coding", "confidence": 0.3969, "tags": [ "english", "quoted_distraction", "software_and_data" ], "error": null }, { "id": "505c47960e52813d4745", "expected": "reasoning", "predicted": "general", "confidence": 0.6236, "tags": [ "english", "home_and_travel", "quoted_distraction" ], "error": null }, { "id": "8099eb9c16304c5518f8", "expected": "reasoning", "predicted": "general", "confidence": 0.664, "tags": [ "english", "health_and_wellbeing", "quoted_distraction" ], "error": null }, { "id": "a850b6ce07bbb614a0b2", "expected": "summarization", "predicted": "general", "confidence": 0.6765, "tags": [ "english", "home_and_travel", "keyword_collision" ], "error": null }, { "id": "8e81bf855fc25beb3092", "expected": "math", "predicted": "summarization", "confidence": 0.5022, "tags": [ "english", "keyword_collision", "retail_and_support" ], "error": null }, { "id": "dd4f2c1710dff5842eb9", "expected": "math", "predicted": "general", "confidence": 0.7352, "tags": [ "english", "health_and_wellbeing", "quoted_distraction" ], "error": null }, { "id": "a21fa96a5fbf89d71674", "expected": "general", "predicted": "creative_writing", "confidence": 0.5561, "tags": [ "english", "quoted_distraction", "workplace_and_business" ], "error": null }, { "id": "79b117c4b1d82376055f", "expected": "math", "predicted": "reasoning", "confidence": 0.5762, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "8377e539f21a6698238c", "expected": "coding", "predicted": "general", "confidence": 0.4044, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "fe2b5f3f39025145182a", "expected": "summarization", "predicted": "creative_writing", "confidence": 0.3157, "tags": [ "arts_and_media", "english", "keyword_collision" ], "error": null }, { "id": "737d199ccf4093ae6489", "expected": "creative_writing", "predicted": "general", "confidence": 0.523, "tags": [ "education_and_history", "english", "keyword_collision" ], "error": null }, { "id": "c3d7039c6cb5fae2718e", "expected": "creative_writing", "predicted": "coding", "confidence": 0.5825, "tags": [ "english", "quoted_distraction", "sports_and_leisure" ], "error": null }, { "id": "abe834e997ffa29864f1", "expected": "reasoning", "predicted": "general", "confidence": 0.3651, "tags": [ "english", "home_and_travel", "keyword_collision" ], "error": null }, { "id": "18420ee632bd0b8fa702", "expected": "math", "predicted": "general", "confidence": 0.4761, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "a89afa71a134defd0788", "expected": "reasoning", "predicted": "general", "confidence": 0.7811, "tags": [ "arts_and_media", "english", "implicit" ], "error": null }, { "id": "3dd717cda4c5cf892608", "expected": "reasoning", "predicted": "general", "confidence": 0.4634, "tags": [ "education_and_history", "english", "implicit" ], "error": null }, { "id": "5729a36e9816ea1e8e6c", "expected": "general", "predicted": "summarization", "confidence": 0.9117, "tags": [ "english", "quoted_distraction", "software_and_data" ], "error": null }, { "id": "3f5ea624e6b566bca6f6", "expected": "summarization", "predicted": "general", "confidence": 0.6646, "tags": [ "english", "keyword_collision", "sports_and_leisure" ], "error": null }, { "id": "46d42c4ccdd6ad81892c", "expected": "summarization", "predicted": "general", "confidence": 0.6153, "tags": [ "english", "implicit", "science_and_environment" ], "error": null }, { "id": "4e7d6cc93749288871b2", "expected": "reasoning", "predicted": "general", "confidence": 0.7123, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "26dd089128c0098e666f", "expected": "general", "predicted": "summarization", "confidence": 0.8514, "tags": [ "english", "public_services", "quoted_distraction" ], "error": null }, { "id": "05ff4ae31bdc6a6bc29c", "expected": "math", "predicted": "general", "confidence": 0.4574, "tags": [ "english", "implicit", "science_and_environment" ], "error": null }, { "id": "04aa5647dad28657a678", "expected": "creative_writing", "predicted": "general", "confidence": 0.4762, "tags": [ "arts_and_media", "english", "implicit" ], "error": null }, { "id": "10a8196ccde72c8e62b0", "expected": "reasoning", "predicted": "summarization", "confidence": 0.9871, "tags": [ "english", "quoted_distraction", "science_and_environment" ], "error": null }, { "id": "8df72a24169482e98f16", "expected": "coding", "predicted": "reasoning", "confidence": 0.3378, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "01e9bc55ce014f64ace9", "expected": "summarization", "predicted": "creative_writing", "confidence": 0.473, "tags": [ "arts_and_media", "english", "implicit" ], "error": null }, { "id": "bcb507b7485b4b6b2247", "expected": "coding", "predicted": "general", "confidence": 0.7016, "tags": [ "english", "quoted_distraction", "workplace_and_business" ], "error": null }, { "id": "97aca115ee5a32995067", "expected": "creative_writing", "predicted": "coding", "confidence": 0.4144, "tags": [ "english", "keyword_collision", "software_and_data" ], "error": null }, { "id": "750df5485ae642dc1c9d", "expected": "math", "predicted": "reasoning", "confidence": 0.3897, "tags": [ "education_and_history", "english", "keyword_collision" ], "error": null }, { "id": "fc84d7d52582336defe6", "expected": "math", "predicted": "reasoning", "confidence": 0.3036, "tags": [ "education_and_history", "english", "implicit" ], "error": null }, { "id": "ad3705afb97c013d1ca0", "expected": "summarization", "predicted": "general", "confidence": 0.5994, "tags": [ "english", "health_and_wellbeing", "implicit" ], "error": null }, { "id": "f84ac96f4cfab6c9e842", "expected": "reasoning", "predicted": "general", "confidence": 0.5861, "tags": [ "english", "explicit", "science_and_environment" ], "error": null }, { "id": "b3e958362c10654d49ed", "expected": "math", "predicted": "general", "confidence": 0.355, "tags": [ "english", "keyword_collision", "public_services" ], "error": null }, { "id": "958ed1156f788ec1eb16", "expected": "summarization", "predicted": "creative_writing", "confidence": 0.5013, "tags": [ "english", "keyword_collision", "software_and_data" ], "error": null }, { "id": "8b68ee5bccd41cdd7159", "expected": "summarization", "predicted": "creative_writing", "confidence": 0.838, "tags": [ "english", "quoted_distraction", "retail_and_support" ], "error": null }, { "id": "ca8a1428304372ae0e63", "expected": "summarization", "predicted": "creative_writing", "confidence": 0.3326, "tags": [ "english", "quoted_distraction", "science_and_environment" ], "error": null }, { "id": "c794076af96fe4e47ddf", "expected": "summarization", "predicted": "general", "confidence": 0.7878, "tags": [ "english", "implicit", "workplace_and_business" ], "error": null }, { "id": "e002489e78030d58d6c5", "expected": "general", "predicted": "creative_writing", "confidence": 0.9777, "tags": [ "education_and_history", "english", "quoted_distraction" ], "error": null }, { "id": "07da174977fc72b8d80b", "expected": "general", "predicted": "creative_writing", "confidence": 0.3904, "tags": [ "education_and_history", "english", "quoted_distraction" ], "error": null }, { "id": "f4eb3f9509b0a24ec979", "expected": "reasoning", "predicted": "general", "confidence": 0.6298, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "9e01bf98fe9a9c01bac0", "expected": "coding", "predicted": "general", "confidence": 0.3391, "tags": [ "arts_and_media", "english", "implicit" ], "error": null }, { "id": "5d129c1fe8f22bdf3805", "expected": "reasoning", "predicted": "general", "confidence": 0.4963, "tags": [ "arts_and_media", "english", "explicit" ], "error": null }, { "id": "b5540b2515ee49df64b5", "expected": "reasoning", "predicted": "general", "confidence": 0.5605, "tags": [ "english", "implicit", "science_and_environment" ], "error": null }, { "id": "cc835c6b914c0cfb65f5", "expected": "math", "predicted": "general", "confidence": 0.4552, "tags": [ "english", "health_and_wellbeing", "keyword_collision" ], "error": null }, { "id": "215182d4142f75695a9c", "expected": "coding", "predicted": "creative_writing", "confidence": 0.8411, "tags": [ "english", "quoted_distraction", "sports_and_leisure" ], "error": null }, { "id": "943f2dfdbd0c64238750", "expected": "math", "predicted": "general", "confidence": 0.5586, "tags": [ "english", "health_and_wellbeing", "keyword_collision" ], "error": null }, { "id": "747a66c3a98538e88a17", "expected": "creative_writing", "predicted": "coding", "confidence": 0.7743, "tags": [ "english", "keyword_collision", "software_and_data" ], "error": null }, { "id": "3fdc5b750dc8d215d582", "expected": "general", "predicted": "creative_writing", "confidence": 0.4685, "tags": [ "english", "health_and_wellbeing", "keyword_collision" ], "error": null }, { "id": "646c4be145d5a4ab2e87", "expected": "math", "predicted": "general", "confidence": 0.4832, "tags": [ "english", "explicit", "health_and_wellbeing" ], "error": null }, { "id": "69f4fb75ebae545f647d", "expected": "reasoning", "predicted": "general", "confidence": 0.9054, "tags": [ "english", "explicit", "sports_and_leisure" ], "error": null }, { "id": "ef335e214de78a2cc4a2", "expected": "math", "predicted": "reasoning", "confidence": 0.3489, "tags": [ "english", "implicit", "software_and_data" ], "error": null }, { "id": "91557e431618077c451c", "expected": "reasoning", "predicted": "general", "confidence": 0.3893, "tags": [ "arts_and_media", "english", "implicit" ], "error": null }, { "id": "c8ea33baf2fdbb8883eb", "expected": "general", "predicted": "creative_writing", "confidence": 0.5915, "tags": [ "arts_and_media", "english", "keyword_collision" ], "error": null }, { "id": "87e7e947a2c2810115cc", "expected": "reasoning", "predicted": "general", "confidence": 0.4602, "tags": [ "english", "explicit", "science_and_environment" ], "error": null }, { "id": "e050c1378797feb7a4fe", "expected": "creative_writing", "predicted": "general", "confidence": 0.4501, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "348e1c4ea62a66aff9ea", "expected": "math", "predicted": "general", "confidence": 0.5373, "tags": [ "arts_and_media", "english", "explicit" ], "error": null }, { "id": "d377da940f2180cb84d3", "expected": "reasoning", "predicted": "general", "confidence": 0.503, "tags": [ "english", "health_and_wellbeing", "keyword_collision" ], "error": null }, { "id": "c4fe609649957c21b2cf", "expected": "reasoning", "predicted": "general", "confidence": 0.6648, "tags": [ "english", "home_and_travel", "implicit" ], "error": null }, { "id": "74f9745ec3b37e63acf1", "expected": "math", "predicted": "general", "confidence": 0.3369, "tags": [ "education_and_history", "english", "keyword_collision" ], "error": null }, { "id": "23ac0693f4928a753ea6", "expected": "general", "predicted": "creative_writing", "confidence": 0.3953, "tags": [ "english", "health_and_wellbeing", "quoted_distraction" ], "error": null }, { "id": "4b0d564fa13e3b383c35", "expected": "reasoning", "predicted": "general", "confidence": 0.6968, "tags": [ "english", "explicit", "workplace_and_business" ], "error": null }, { "id": "5a2d1fc65c149e714de8", "expected": "reasoning", "predicted": "general", "confidence": 0.516, "tags": [ "english", "quoted_distraction", "software_and_data" ], "error": null }, { "id": "504733447d6f30d0efc9", "expected": "general", "predicted": "reasoning", "confidence": 0.4166, "tags": [ "english", "keyword_collision", "retail_and_support" ], "error": null }, { "id": "1875271e260b09b397b5", "expected": "coding", "predicted": "math", "confidence": 0.5304, "tags": [ "arts_and_media", "english", "keyword_collision" ], "error": null }, { "id": "d65c6d2bc7bebd381ff1", "expected": "reasoning", "predicted": "summarization", "confidence": 0.9341, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "87e8750a65c1ee973962", "expected": "summarization", "predicted": "creative_writing", "confidence": 0.4041, "tags": [ "education_and_history", "english", "implicit" ], "error": null }, { "id": "857db3eb03b2d034ce92", "expected": "coding", "predicted": "creative_writing", "confidence": 0.9828, "tags": [ "english", "home_and_travel", "quoted_distraction" ], "error": null }, { "id": "e616750c8c7fbdc2194b", "expected": "math", "predicted": "general", "confidence": 0.4199, "tags": [ "arts_and_media", "english", "keyword_collision" ], "error": null }, { "id": "a62561dc670369ed8ed3", "expected": "reasoning", "predicted": "general", "confidence": 0.3599, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "f5b97da8ef961e833639", "expected": "general", "predicted": "coding", "confidence": 0.4752, "tags": [ "english", "quoted_distraction", "workplace_and_business" ], "error": null }, { "id": "bbfcef3f5958ab91ba99", "expected": "creative_writing", "predicted": "general", "confidence": 0.5194, "tags": [ "english", "implicit", "software_and_data" ], "error": null }, { "id": "d8fe83298ba5099c71f0", "expected": "creative_writing", "predicted": "summarization", "confidence": 0.5921, "tags": [ "english", "quoted_distraction", "workplace_and_business" ], "error": null }, { "id": "abe623e53fc0c8bf254c", "expected": "math", "predicted": "general", "confidence": 0.4883, "tags": [ "english", "keyword_collision", "public_services" ], "error": null }, { "id": "0677ec5e47edea030d04", "expected": "math", "predicted": "coding", "confidence": 0.4741, "tags": [ "english", "quoted_distraction", "software_and_data" ], "error": null } ], "accuracy_ci95": [ 0.6333333333333333, 0.6916666666666667 ], "macro_f1_ci95": [ 0.630049600779701, 0.6914101220473432 ], "by_task_ci95": { "coding": [ 0.75, 0.88125 ], "reasoning": [ 0.40625, 0.55625 ], "creative_writing": [ 0.84375, 0.94375 ], "summarization": [ 0.475, 0.5875 ], "math": [ 0.35625, 0.5625 ], "general": [ 0.7125, 0.84375 ] } }, { "strategy": "strands", "requests": 960, "correct": 877, "accuracy": 0.9135416666666667, "macro_f1": 0.9136092021280621, "errors": 0, "cache_hits": 0, "latency_ms": { "mean": 167.93462332927143, "p50": 164.17612550139893, "p95": 207.51637739740545, "p99": 224.5762084727903 }, "latency_scope": "successful requests only", "all_attempts_latency_ms": { "p50": 164.17612550139893, "p95": 207.51637739740545 }, "llm_calls": 0, "llm_call_rate": 0.0, "input_tokens": 0, "output_tokens": 0, "router_cost_usd": 0.0, "known_router_cost_usd": 0.0, "cost_coverage": 1.0, "cost_scope": "API charges only; local hardware and electricity excluded", "hardware_electricity_cost_usd": null, "hardware_electricity_cost_status": "unmeasured", "router_cost_per_1k_requests_usd": 0.0, "confusion_matrix": { "coding": { "coding": 158, "math": 2 }, "reasoning": { "coding": 12, "creative_writing": 2, "general": 16, "math": 1, "reasoning": 127, "summarization": 2 }, "creative_writing": { "coding": 3, "creative_writing": 152, "general": 5 }, "summarization": { "coding": 5, "creative_writing": 1, "general": 1, "summarization": 153 }, "math": { "coding": 4, "general": 2, "math": 153, "summarization": 1 }, "general": { "coding": 15, "creative_writing": 2, "general": 134, "reasoning": 1, "summarization": 8 } }, "by_task": { "coding": { "requests": 160, "correct": 158, "accuracy": 0.9875 }, "reasoning": { "requests": 160, "correct": 127, "accuracy": 0.79375 }, "creative_writing": { "requests": 160, "correct": 152, "accuracy": 0.95 }, "summarization": { "requests": 160, "correct": 153, "accuracy": 0.95625 }, "math": { "requests": 160, "correct": 153, "accuracy": 0.95625 }, "general": { "requests": 160, "correct": 134, "accuracy": 0.8375 } }, "by_tag": { "arts_and_media": { "requests": 96, "accuracy": 0.9375 }, "education_and_history": { "requests": 96, "accuracy": 0.9583333333333334 }, "english": { "requests": 960, "accuracy": 0.9135416666666667 }, "explicit": { "requests": 240, "accuracy": 0.9666666666666667 }, "health_and_wellbeing": { "requests": 96, "accuracy": 0.9375 }, "home_and_travel": { "requests": 96, "accuracy": 0.9479166666666666 }, "implicit": { "requests": 240, "accuracy": 0.9166666666666666 }, "keyword_collision": { "requests": 240, "accuracy": 0.8833333333333333 }, "public_services": { "requests": 96, "accuracy": 0.9270833333333334 }, "quoted_distraction": { "requests": 240, "accuracy": 0.8875 }, "retail_and_support": { "requests": 96, "accuracy": 0.9375 }, "science_and_environment": { "requests": 96, "accuracy": 0.9375 }, "software_and_data": { "requests": 96, "accuracy": 0.6875 }, "sports_and_leisure": { "requests": 96, "accuracy": 0.9166666666666666 }, "workplace_and_business": { "requests": 96, "accuracy": 0.9479166666666666 } }, "wrong_cases": [ { "id": "07df2439da4bcbfa95ba", "expected": "general", "predicted": "summarization", "confidence": 0.7966, "tags": [ "education_and_history", "english", "keyword_collision" ], "error": null }, { "id": "5eca30c84bed105d5917", "expected": "coding", "predicted": "math", "confidence": 0.5083, "tags": [ "english", "implicit", "science_and_environment" ], "error": null }, { "id": "934ba026d1e3ce74b569", "expected": "reasoning", "predicted": "general", "confidence": 0.4822, "tags": [ "english", "explicit", "home_and_travel" ], "error": null }, { "id": "f6e9e9fca12fd3609488", "expected": "reasoning", "predicted": "coding", "confidence": 0.7168, "tags": [ "english", "implicit", "software_and_data" ], "error": null }, { "id": "bdf254ebb8b710451598", "expected": "creative_writing", "predicted": "general", "confidence": 0.6649, "tags": [ "english", "health_and_wellbeing", "quoted_distraction" ], "error": null }, { "id": "d29cb8254a1c42be9bab", "expected": "general", "predicted": "coding", "confidence": 0.7737, "tags": [ "english", "implicit", "software_and_data" ], "error": null }, { "id": "fd91948f511a05c56916", "expected": "reasoning", "predicted": "general", "confidence": 0.3548, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "9ce3d131bea9511add54", "expected": "reasoning", "predicted": "coding", "confidence": 0.5137, "tags": [ "english", "quoted_distraction", "software_and_data" ], "error": null }, { "id": "241efe3ded101ae619cb", "expected": "reasoning", "predicted": "general", "confidence": 0.4574, "tags": [ "english", "home_and_travel", "quoted_distraction" ], "error": null }, { "id": "04a04e71fe6a88a707e2", "expected": "summarization", "predicted": "coding", "confidence": 0.9339, "tags": [ "english", "keyword_collision", "software_and_data" ], "error": null }, { "id": "93f50458ad2a4a9e3a5d", "expected": "general", "predicted": "coding", "confidence": 0.5426, "tags": [ "english", "implicit", "software_and_data" ], "error": null }, { "id": "15a225ff569ecd16a83b", "expected": "general", "predicted": "creative_writing", "confidence": 0.5047, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "7a808fb2da45604b936b", "expected": "general", "predicted": "summarization", "confidence": 0.7785, "tags": [ "english", "quoted_distraction", "science_and_environment" ], "error": null }, { "id": "64db9b9347702bdfd601", "expected": "reasoning", "predicted": "general", "confidence": 0.5259, "tags": [ "english", "home_and_travel", "implicit" ], "error": null }, { "id": "99718a62d463917596b9", "expected": "math", "predicted": "coding", "confidence": 0.5949, "tags": [ "english", "keyword_collision", "sports_and_leisure" ], "error": null }, { "id": "1b83610098aead6d79a8", "expected": "general", "predicted": "summarization", "confidence": 0.5821, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "dc7d41480b40eaa51ddd", "expected": "coding", "predicted": "math", "confidence": 0.6097, "tags": [ "english", "keyword_collision", "science_and_environment" ], "error": null }, { "id": "04127c823546eec8c5e1", "expected": "general", "predicted": "coding", "confidence": 0.8663, "tags": [ "english", "keyword_collision", "public_services" ], "error": null }, { "id": "fb8fe76e1701d45c3b01", "expected": "general", "predicted": "coding", "confidence": 0.8719, "tags": [ "english", "keyword_collision", "public_services" ], "error": null }, { "id": "1dd48cce991abd916f1f", "expected": "reasoning", "predicted": "general", "confidence": 0.4979, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "6179e53c41951b7f93da", "expected": "math", "predicted": "coding", "confidence": 0.659, "tags": [ "english", "keyword_collision", "software_and_data" ], "error": null }, { "id": "d3af9cdb773a1957795e", "expected": "general", "predicted": "summarization", "confidence": 0.5924, "tags": [ "english", "quoted_distraction", "software_and_data" ], "error": null }, { "id": "9993d1d5aa1248a8f422", "expected": "math", "predicted": "coding", "confidence": 0.5539, "tags": [ "english", "keyword_collision", "software_and_data" ], "error": null }, { "id": "2a7f1edfefb2342759b8", "expected": "reasoning", "predicted": "general", "confidence": 0.5258, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "652abaf894ca582b8f3d", "expected": "summarization", "predicted": "coding", "confidence": 0.801, "tags": [ "english", "quoted_distraction", "software_and_data" ], "error": null }, { "id": "e075dced2aad6ef2b9dc", "expected": "reasoning", "predicted": "general", "confidence": 0.4421, "tags": [ "english", "implicit", "public_services" ], "error": null }, { "id": "c3f9e52afdfb30061f8c", "expected": "general", "predicted": "coding", "confidence": 0.5486, "tags": [ "english", "keyword_collision", "public_services" ], "error": null }, { "id": "4743676703472be059d9", "expected": "general", "predicted": "coding", "confidence": 0.7561, "tags": [ "english", "explicit", "software_and_data" ], "error": null }, { "id": "ef84d8c26c15a0137291", "expected": "reasoning", "predicted": "general", "confidence": 0.3804, "tags": [ "arts_and_media", "english", "explicit" ], "error": null }, { "id": "fbc9291d970bdd23e823", "expected": "reasoning", "predicted": "coding", "confidence": 0.8316, "tags": [ "english", "keyword_collision", "software_and_data" ], "error": null }, { "id": "39c9c27b895d27f6ca67", "expected": "reasoning", "predicted": "general", "confidence": 0.5549, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "cbc888cbb5bca1df0382", "expected": "reasoning", "predicted": "coding", "confidence": 0.5006, "tags": [ "english", "implicit", "software_and_data" ], "error": null }, { "id": "be0307f78ded5d0c42a0", "expected": "reasoning", "predicted": "coding", "confidence": 0.577, "tags": [ "english", "explicit", "software_and_data" ], "error": null }, { "id": "a6e98cf3b4cacd4aa15f", "expected": "general", "predicted": "coding", "confidence": 0.7153, "tags": [ "english", "keyword_collision", "software_and_data" ], "error": null }, { "id": "ce2c8127d9919f54e516", "expected": "general", "predicted": "reasoning", "confidence": 0.3392, "tags": [ "english", "keyword_collision", "retail_and_support" ], "error": null }, { "id": "2e9f75b565c09e38ba99", "expected": "reasoning", "predicted": "general", "confidence": 0.4603, "tags": [ "english", "health_and_wellbeing", "quoted_distraction" ], "error": null }, { "id": "be87b66f1a4177f7b2d1", "expected": "general", "predicted": "coding", "confidence": 0.8068, "tags": [ "english", "keyword_collision", "software_and_data" ], "error": null }, { "id": "5b39faae7199e7fcf54f", "expected": "reasoning", "predicted": "coding", "confidence": 0.4645, "tags": [ "english", "quoted_distraction", "software_and_data" ], "error": null }, { "id": "09c2383d7c1e38200243", "expected": "general", "predicted": "coding", "confidence": 0.3485, "tags": [ "english", "explicit", "software_and_data" ], "error": null }, { "id": "b30dd7009255b9d366de", "expected": "general", "predicted": "summarization", "confidence": 0.4963, "tags": [ "english", "keyword_collision", "science_and_environment" ], "error": null }, { "id": "5cee93858561dd2c7bc0", "expected": "summarization", "predicted": "coding", "confidence": 0.5002, "tags": [ "english", "quoted_distraction", "software_and_data" ], "error": null }, { "id": "beb5f2c57b6fd758b862", "expected": "reasoning", "predicted": "general", "confidence": 0.5254, "tags": [ "english", "home_and_travel", "implicit" ], "error": null }, { "id": "8b6ffad6d552ae1f1619", "expected": "reasoning", "predicted": "coding", "confidence": 0.5372, "tags": [ "english", "explicit", "software_and_data" ], "error": null }, { "id": "41be605dcd05dd815139", "expected": "reasoning", "predicted": "general", "confidence": 0.5835, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "7f48179d7d548473a00f", "expected": "general", "predicted": "coding", "confidence": 0.7351, "tags": [ "english", "explicit", "software_and_data" ], "error": null }, { "id": "40e07c71af0efd9b276d", "expected": "general", "predicted": "coding", "confidence": 0.6408, "tags": [ "english", "keyword_collision", "sports_and_leisure" ], "error": null }, { "id": "94653e1b7b611651fda7", "expected": "general", "predicted": "coding", "confidence": 0.6756, "tags": [ "english", "keyword_collision", "workplace_and_business" ], "error": null }, { "id": "dc5d9921cacf189ea3a7", "expected": "general", "predicted": "coding", "confidence": 0.4791, "tags": [ "english", "keyword_collision", "public_services" ], "error": null }, { "id": "78b7dde5e1bc6c2ed5f3", "expected": "reasoning", "predicted": "general", "confidence": 0.7495, "tags": [ "english", "home_and_travel", "implicit" ], "error": null }, { "id": "be1d66a4d2ff1afcf414", "expected": "reasoning", "predicted": "creative_writing", "confidence": 0.6882, "tags": [ "education_and_history", "english", "quoted_distraction" ], "error": null }, { "id": "118d25bc2c70f8e162a8", "expected": "reasoning", "predicted": "coding", "confidence": 0.5266, "tags": [ "english", "implicit", "software_and_data" ], "error": null }, { "id": "20bf3adbf974f1aacb80", "expected": "math", "predicted": "coding", "confidence": 0.6714, "tags": [ "english", "quoted_distraction", "software_and_data" ], "error": null }, { "id": "94e79d61f2500cd071f3", "expected": "reasoning", "predicted": "general", "confidence": 0.4814, "tags": [ "english", "health_and_wellbeing", "implicit" ], "error": null }, { "id": "dc7f8c5082a153c49621", "expected": "reasoning", "predicted": "summarization", "confidence": 0.6808, "tags": [ "english", "public_services", "quoted_distraction" ], "error": null }, { "id": "19b36d3302e3b1a40897", "expected": "general", "predicted": "coding", "confidence": 0.79, "tags": [ "english", "quoted_distraction", "software_and_data" ], "error": null }, { "id": "c715c42a3a8ec0dcda2c", "expected": "reasoning", "predicted": "coding", "confidence": 0.4718, "tags": [ "english", "quoted_distraction", "workplace_and_business" ], "error": null }, { "id": "642db1bf76a8216db9e0", "expected": "summarization", "predicted": "coding", "confidence": 0.7752, "tags": [ "english", "keyword_collision", "software_and_data" ], "error": null }, { "id": "1ea1cb699c1759141306", "expected": "summarization", "predicted": "general", "confidence": 0.4143, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "9ec9ff018185d81d81cd", "expected": "reasoning", "predicted": "coding", "confidence": 0.644, "tags": [ "english", "quoted_distraction", "software_and_data" ], "error": null }, { "id": "8e81bf855fc25beb3092", "expected": "math", "predicted": "summarization", "confidence": 0.5385, "tags": [ "english", "keyword_collision", "retail_and_support" ], "error": null }, { "id": "e22c70f63feb13865cde", "expected": "creative_writing", "predicted": "general", "confidence": 0.6832, "tags": [ "english", "health_and_wellbeing", "keyword_collision" ], "error": null }, { "id": "c3d7039c6cb5fae2718e", "expected": "creative_writing", "predicted": "coding", "confidence": 0.4639, "tags": [ "english", "quoted_distraction", "sports_and_leisure" ], "error": null }, { "id": "18420ee632bd0b8fa702", "expected": "math", "predicted": "general", "confidence": 0.4033, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "100a4655b778a61c15e1", "expected": "creative_writing", "predicted": "general", "confidence": 0.4346, "tags": [ "english", "keyword_collision", "workplace_and_business" ], "error": null }, { "id": "a89afa71a134defd0788", "expected": "reasoning", "predicted": "general", "confidence": 0.5181, "tags": [ "arts_and_media", "english", "implicit" ], "error": null }, { "id": "5729a36e9816ea1e8e6c", "expected": "general", "predicted": "summarization", "confidence": 0.5439, "tags": [ "english", "quoted_distraction", "software_and_data" ], "error": null }, { "id": "b91ece8e2df579cca066", "expected": "reasoning", "predicted": "creative_writing", "confidence": 0.532, "tags": [ "english", "keyword_collision", "science_and_environment" ], "error": null }, { "id": "26dd089128c0098e666f", "expected": "general", "predicted": "summarization", "confidence": 0.7455, "tags": [ "english", "public_services", "quoted_distraction" ], "error": null }, { "id": "04da87ada9dc1b30a5cf", "expected": "general", "predicted": "summarization", "confidence": 0.5058, "tags": [ "education_and_history", "english", "keyword_collision" ], "error": null }, { "id": "958ed1156f788ec1eb16", "expected": "summarization", "predicted": "coding", "confidence": 0.5152, "tags": [ "english", "keyword_collision", "software_and_data" ], "error": null }, { "id": "8b68ee5bccd41cdd7159", "expected": "summarization", "predicted": "creative_writing", "confidence": 0.664, "tags": [ "english", "quoted_distraction", "retail_and_support" ], "error": null }, { "id": "e002489e78030d58d6c5", "expected": "general", "predicted": "creative_writing", "confidence": 0.4024, "tags": [ "education_and_history", "english", "quoted_distraction" ], "error": null }, { "id": "5d129c1fe8f22bdf3805", "expected": "reasoning", "predicted": "general", "confidence": 0.4243, "tags": [ "arts_and_media", "english", "explicit" ], "error": null }, { "id": "943f2dfdbd0c64238750", "expected": "math", "predicted": "general", "confidence": 0.5708, "tags": [ "english", "health_and_wellbeing", "keyword_collision" ], "error": null }, { "id": "747a66c3a98538e88a17", "expected": "creative_writing", "predicted": "coding", "confidence": 0.4046, "tags": [ "english", "keyword_collision", "software_and_data" ], "error": null }, { "id": "5c5d7b9832913f37f1ee", "expected": "reasoning", "predicted": "coding", "confidence": 0.5675, "tags": [ "english", "keyword_collision", "software_and_data" ], "error": null }, { "id": "e050c1378797feb7a4fe", "expected": "creative_writing", "predicted": "general", "confidence": 0.4552, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "d377da940f2180cb84d3", "expected": "reasoning", "predicted": "math", "confidence": 0.3971, "tags": [ "english", "health_and_wellbeing", "keyword_collision" ], "error": null }, { "id": "9369d49991855abf8f00", "expected": "creative_writing", "predicted": "coding", "confidence": 0.3996, "tags": [ "english", "quoted_distraction", "science_and_environment" ], "error": null }, { "id": "5a2d1fc65c149e714de8", "expected": "reasoning", "predicted": "coding", "confidence": 0.4216, "tags": [ "english", "quoted_distraction", "software_and_data" ], "error": null }, { "id": "d65c6d2bc7bebd381ff1", "expected": "reasoning", "predicted": "summarization", "confidence": 0.5172, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "f5b97da8ef961e833639", "expected": "general", "predicted": "coding", "confidence": 0.4727, "tags": [ "english", "quoted_distraction", "workplace_and_business" ], "error": null }, { "id": "d8fe83298ba5099c71f0", "expected": "creative_writing", "predicted": "general", "confidence": 0.4618, "tags": [ "english", "quoted_distraction", "workplace_and_business" ], "error": null } ], "accuracy_ci95": [ 0.8916666666666667, 0.9333333333333333 ], "macro_f1_ci95": [ 0.892239342531565, 0.9336232979006062 ], "by_task_ci95": { "coding": [ 0.9625, 1.0 ], "reasoning": [ 0.71875, 0.8625 ], "creative_writing": [ 0.90625, 0.9875 ], "summarization": [ 0.9125, 0.9875 ], "math": [ 0.91875, 0.9875 ], "general": [ 0.75625, 0.90625 ] } }, { "strategy": "llm-router", "requests": 960, "correct": 876, "accuracy": 0.9125, "macro_f1": 0.9114505927313409, "errors": 0, "cache_hits": 0, "latency_ms": { "mean": 777.4775501396865, "p50": 736.2221669973223, "p95": 966.0432875043624, "p99": 1361.0550436870808 }, "latency_scope": "successful requests only", "all_attempts_latency_ms": { "p50": 736.2221669973223, "p95": 966.0432875043624 }, "llm_calls": 960, "llm_call_rate": 1.0, "input_tokens": 278037, "output_tokens": 12416, "router_cost_usd": 0.049155149999999995, "known_router_cost_usd": 0.049155149999999995, "cost_coverage": 1.0, "cost_scope": "API charges only; local hardware and electricity excluded", "hardware_electricity_cost_usd": null, "hardware_electricity_cost_status": "unmeasured", "router_cost_per_1k_requests_usd": 0.05120328124999999, "confusion_matrix": { "coding": { "coding": 147, "reasoning": 11, "summarization": 2 }, "reasoning": { "creative_writing": 2, "general": 1, "reasoning": 155, "summarization": 2 }, "creative_writing": { "coding": 1, "creative_writing": 157, "math": 1, "summarization": 1 }, "summarization": { "general": 1, "reasoning": 2, "summarization": 157 }, "math": { "coding": 1, "creative_writing": 1, "math": 150, "reasoning": 5, "summarization": 3 }, "general": { "general": 110, "math": 1, "reasoning": 25, "summarization": 24 } }, "by_task": { "coding": { "requests": 160, "correct": 147, "accuracy": 0.91875 }, "reasoning": { "requests": 160, "correct": 155, "accuracy": 0.96875 }, "creative_writing": { "requests": 160, "correct": 157, "accuracy": 0.98125 }, "summarization": { "requests": 160, "correct": 157, "accuracy": 0.98125 }, "math": { "requests": 160, "correct": 150, "accuracy": 0.9375 }, "general": { "requests": 160, "correct": 110, "accuracy": 0.6875 } }, "by_tag": { "arts_and_media": { "requests": 96, "accuracy": 0.90625 }, "education_and_history": { "requests": 96, "accuracy": 0.8229166666666666 }, "english": { "requests": 960, "accuracy": 0.9125 }, "explicit": { "requests": 240, "accuracy": 0.95 }, "health_and_wellbeing": { "requests": 96, "accuracy": 0.9270833333333334 }, "home_and_travel": { "requests": 96, "accuracy": 0.9583333333333334 }, "implicit": { "requests": 240, "accuracy": 0.9375 }, "keyword_collision": { "requests": 240, "accuracy": 0.8583333333333333 }, "public_services": { "requests": 96, "accuracy": 0.9479166666666666 }, "quoted_distraction": { "requests": 240, "accuracy": 0.9041666666666667 }, "retail_and_support": { "requests": 96, "accuracy": 0.8958333333333334 }, "science_and_environment": { "requests": 96, "accuracy": 0.8333333333333334 }, "software_and_data": { "requests": 96, "accuracy": 0.8958333333333334 }, "sports_and_leisure": { "requests": 96, "accuracy": 0.9479166666666666 }, "workplace_and_business": { "requests": 96, "accuracy": 0.9895833333333334 } }, "wrong_cases": [ { "id": "07df2439da4bcbfa95ba", "expected": "general", "predicted": "summarization", "confidence": 0.9, "tags": [ "education_and_history", "english", "keyword_collision" ], "error": null }, { "id": "65fd4a24e7f479643b44", "expected": "coding", "predicted": "reasoning", "confidence": 0.9, "tags": [ "english", "keyword_collision", "software_and_data" ], "error": null }, { "id": "ece20e79ca05b460150a", "expected": "coding", "predicted": "summarization", "confidence": 0.9, "tags": [ "english", "keyword_collision", "retail_and_support" ], "error": null }, { "id": "261d5019720fdb62c020", "expected": "coding", "predicted": "reasoning", "confidence": 0.9, "tags": [ "english", "implicit", "public_services" ], "error": null }, { "id": "6954bf9b1ce86e9d9112", "expected": "math", "predicted": "reasoning", "confidence": 0.9, "tags": [ "english", "quoted_distraction", "retail_and_support" ], "error": null }, { "id": "1c5fcb14d2f1cb6316dc", "expected": "math", "predicted": "reasoning", "confidence": 0.9, "tags": [ "english", "health_and_wellbeing", "keyword_collision" ], "error": null }, { "id": "58398efe336eecb1c8de", "expected": "summarization", "predicted": "reasoning", "confidence": 0.9, "tags": [ "education_and_history", "english", "keyword_collision" ], "error": null }, { "id": "36d677b0f72d2ef75c8a", "expected": "general", "predicted": "summarization", "confidence": 0.9, "tags": [ "english", "implicit", "science_and_environment" ], "error": null }, { "id": "93f50458ad2a4a9e3a5d", "expected": "general", "predicted": "reasoning", "confidence": 0.9, "tags": [ "english", "implicit", "software_and_data" ], "error": null }, { "id": "7a808fb2da45604b936b", "expected": "general", "predicted": "summarization", "confidence": 0.9, "tags": [ "english", "quoted_distraction", "science_and_environment" ], "error": null }, { "id": "99718a62d463917596b9", "expected": "math", "predicted": "coding", "confidence": 0.9, "tags": [ "english", "keyword_collision", "sports_and_leisure" ], "error": null }, { "id": "1b83610098aead6d79a8", "expected": "general", "predicted": "reasoning", "confidence": 0.0, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "5db8c00f18fdaa6598a3", "expected": "coding", "predicted": "reasoning", "confidence": 0.9, "tags": [ "arts_and_media", "english", "explicit" ], "error": null }, { "id": "5f55f0fe538771b2854e", "expected": "general", "predicted": "summarization", "confidence": 0.9, "tags": [ "english", "explicit", "software_and_data" ], "error": null }, { "id": "000ecfb9fee737216186", "expected": "general", "predicted": "reasoning", "confidence": 0.9, "tags": [ "education_and_history", "english", "explicit" ], "error": null }, { "id": "0034500228776613ce0b", "expected": "coding", "predicted": "reasoning", "confidence": 0.9, "tags": [ "education_and_history", "english", "keyword_collision" ], "error": null }, { "id": "252f8b27acee6acfe8ed", "expected": "general", "predicted": "reasoning", "confidence": 0.0, "tags": [ "arts_and_media", "english", "explicit" ], "error": null }, { "id": "d3af9cdb773a1957795e", "expected": "general", "predicted": "summarization", "confidence": 0.9, "tags": [ "english", "quoted_distraction", "software_and_data" ], "error": null }, { "id": "c75b289c656134938367", "expected": "general", "predicted": "reasoning", "confidence": 0.9, "tags": [ "english", "keyword_collision", "science_and_environment" ], "error": null }, { "id": "a04a5df58f444918f529", "expected": "general", "predicted": "reasoning", "confidence": 0.9, "tags": [ "arts_and_media", "english", "implicit" ], "error": null }, { "id": "2b7a8968b5bbe84c5b2e", "expected": "coding", "predicted": "reasoning", "confidence": 0.9, "tags": [ "english", "health_and_wellbeing", "keyword_collision" ], "error": null }, { "id": "bd6b5f5d19f6da08bbff", "expected": "general", "predicted": "summarization", "confidence": 0.9, "tags": [ "education_and_history", "english", "keyword_collision" ], "error": null }, { "id": "4743676703472be059d9", "expected": "general", "predicted": "reasoning", "confidence": 0.9, "tags": [ "english", "explicit", "software_and_data" ], "error": null }, { "id": "ce2c8127d9919f54e516", "expected": "general", "predicted": "reasoning", "confidence": 0.9, "tags": [ "english", "keyword_collision", "retail_and_support" ], "error": null }, { "id": "7c9f8dcb26520d798034", "expected": "math", "predicted": "summarization", "confidence": 1.0, "tags": [ "english", "keyword_collision", "retail_and_support" ], "error": null }, { "id": "b39e861cb0162feb1138", "expected": "coding", "predicted": "reasoning", "confidence": 0.9, "tags": [ "english", "keyword_collision", "science_and_environment" ], "error": null }, { "id": "28f5c4eb8ecac426e8a1", "expected": "reasoning", "predicted": "creative_writing", "confidence": 0.9, "tags": [ "english", "public_services", "quoted_distraction" ], "error": null }, { "id": "580683b15c6dc76ee0a5", "expected": "general", "predicted": "reasoning", "confidence": 0.9, "tags": [ "english", "quoted_distraction", "science_and_environment" ], "error": null }, { "id": "4aa86afff32f2190f312", "expected": "general", "predicted": "summarization", "confidence": 0.9, "tags": [ "english", "keyword_collision", "software_and_data" ], "error": null }, { "id": "be87b66f1a4177f7b2d1", "expected": "general", "predicted": "reasoning", "confidence": 0.9, "tags": [ "english", "keyword_collision", "software_and_data" ], "error": null }, { "id": "46f712e5440d46598940", "expected": "math", "predicted": "summarization", "confidence": 0.9, "tags": [ "english", "keyword_collision", "public_services" ], "error": null }, { "id": "b30dd7009255b9d366de", "expected": "general", "predicted": "summarization", "confidence": 0.9, "tags": [ "english", "keyword_collision", "science_and_environment" ], "error": null }, { "id": "0ec4ea9e5d6280937f2d", "expected": "coding", "predicted": "reasoning", "confidence": 0.9, "tags": [ "education_and_history", "english", "implicit" ], "error": null }, { "id": "bc78376d6e7d0c8ddba5", "expected": "general", "predicted": "math", "confidence": 0.8, "tags": [ "english", "health_and_wellbeing", "keyword_collision" ], "error": null }, { "id": "d931f47be7271f7441e4", "expected": "general", "predicted": "summarization", "confidence": 0.9, "tags": [ "education_and_history", "english", "explicit" ], "error": null }, { "id": "056e742c437dfa38e9b9", "expected": "general", "predicted": "summarization", "confidence": 0.9, "tags": [ "education_and_history", "english", "explicit" ], "error": null }, { "id": "a6950e8a97382e1e0dc2", "expected": "general", "predicted": "reasoning", "confidence": 0.9, "tags": [ "english", "health_and_wellbeing", "keyword_collision" ], "error": null }, { "id": "798852db56402a369fbc", "expected": "math", "predicted": "creative_writing", "confidence": 0.9, "tags": [ "english", "quoted_distraction", "retail_and_support" ], "error": null }, { "id": "8f7fdc44e1778580a960", "expected": "general", "predicted": "reasoning", "confidence": 0.9, "tags": [ "english", "home_and_travel", "quoted_distraction" ], "error": null }, { "id": "40e07c71af0efd9b276d", "expected": "general", "predicted": "reasoning", "confidence": 0.9, "tags": [ "english", "keyword_collision", "sports_and_leisure" ], "error": null }, { "id": "49a0d90c13ef1a299e8a", "expected": "general", "predicted": "summarization", "confidence": 0.9, "tags": [ "english", "home_and_travel", "quoted_distraction" ], "error": null }, { "id": "188ad75336e020ca44e2", "expected": "general", "predicted": "reasoning", "confidence": 0.9, "tags": [ "english", "explicit", "science_and_environment" ], "error": null }, { "id": "78b7dde5e1bc6c2ed5f3", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "implicit" ], "error": null }, { "id": "546db3471f77e0f59eaa", "expected": "math", "predicted": "reasoning", "confidence": 0.9, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "8e8a6eecbe221ea5450b", "expected": "general", "predicted": "summarization", "confidence": 0.9, "tags": [ "english", "keyword_collision", "science_and_environment" ], "error": null }, { "id": "0829d305f71eae0886cd", "expected": "coding", "predicted": "summarization", "confidence": 1.0, "tags": [ "education_and_history", "english", "keyword_collision" ], "error": null }, { "id": "ebb1bb67daa12233ed20", "expected": "general", "predicted": "reasoning", "confidence": 0.9, "tags": [ "english", "quoted_distraction", "sports_and_leisure" ], "error": null }, { "id": "53bac73d24f3603f18f0", "expected": "coding", "predicted": "reasoning", "confidence": 0.9, "tags": [ "arts_and_media", "english", "implicit" ], "error": null }, { "id": "d0847b2fc3712dc61d25", "expected": "general", "predicted": "summarization", "confidence": 0.9, "tags": [ "education_and_history", "english", "explicit" ], "error": null }, { "id": "dc7f8c5082a153c49621", "expected": "reasoning", "predicted": "summarization", "confidence": 0.9, "tags": [ "english", "public_services", "quoted_distraction" ], "error": null }, { "id": "456b1dfbde33124796a1", "expected": "general", "predicted": "reasoning", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "science_and_environment" ], "error": null }, { "id": "a43b38f9c785b5ce6a1d", "expected": "general", "predicted": "reasoning", "confidence": 0.9, "tags": [ "english", "implicit", "science_and_environment" ], "error": null }, { "id": "aba3cecda283c4cf9362", "expected": "general", "predicted": "reasoning", "confidence": 0.9, "tags": [ "english", "health_and_wellbeing", "implicit" ], "error": null }, { "id": "f866fcbe55f107de4d81", "expected": "coding", "predicted": "reasoning", "confidence": 0.9, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "92348d63c89724bf9bde", "expected": "general", "predicted": "summarization", "confidence": 0.9, "tags": [ "education_and_history", "english", "quoted_distraction" ], "error": null }, { "id": "c1ebf9db635e32596d4e", "expected": "general", "predicted": "summarization", "confidence": 0.9, "tags": [ "english", "explicit", "science_and_environment" ], "error": null }, { "id": "12272f9cde9cbaf8c39a", "expected": "coding", "predicted": "reasoning", "confidence": 0.9, "tags": [ "english", "keyword_collision", "retail_and_support" ], "error": null }, { "id": "642db1bf76a8216db9e0", "expected": "summarization", "predicted": "reasoning", "confidence": 0.9, "tags": [ "english", "keyword_collision", "software_and_data" ], "error": null }, { "id": "15ca6ca795d1016becf3", "expected": "coding", "predicted": "reasoning", "confidence": 0.9, "tags": [ "arts_and_media", "english", "keyword_collision" ], "error": null }, { "id": "1ea1cb699c1759141306", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "0b483f7f02fff7f1685e", "expected": "creative_writing", "predicted": "summarization", "confidence": 0.9, "tags": [ "english", "home_and_travel", "keyword_collision" ], "error": null }, { "id": "8e81bf855fc25beb3092", "expected": "math", "predicted": "summarization", "confidence": 0.9, "tags": [ "english", "keyword_collision", "retail_and_support" ], "error": null }, { "id": "a21fa96a5fbf89d71674", "expected": "general", "predicted": "summarization", "confidence": 0.9, "tags": [ "english", "quoted_distraction", "workplace_and_business" ], "error": null }, { "id": "ca0877c64afc5955fdb6", "expected": "general", "predicted": "reasoning", "confidence": 0.9, "tags": [ "education_and_history", "english", "quoted_distraction" ], "error": null }, { "id": "79b117c4b1d82376055f", "expected": "math", "predicted": "reasoning", "confidence": 0.9, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "c3d7039c6cb5fae2718e", "expected": "creative_writing", "predicted": "coding", "confidence": 0.9, "tags": [ "english", "quoted_distraction", "sports_and_leisure" ], "error": null }, { "id": "5729a36e9816ea1e8e6c", "expected": "general", "predicted": "summarization", "confidence": 0.9, "tags": [ "english", "quoted_distraction", "software_and_data" ], "error": null }, { "id": "b91ece8e2df579cca066", "expected": "reasoning", "predicted": "creative_writing", "confidence": 0.9, "tags": [ "english", "keyword_collision", "science_and_environment" ], "error": null }, { "id": "26dd089128c0098e666f", "expected": "general", "predicted": "summarization", "confidence": 0.9, "tags": [ "english", "public_services", "quoted_distraction" ], "error": null }, { "id": "f3968d022ca681e65c6f", "expected": "creative_writing", "predicted": "math", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "software_and_data" ], "error": null }, { "id": "04da87ada9dc1b30a5cf", "expected": "general", "predicted": "summarization", "confidence": 0.9, "tags": [ "education_and_history", "english", "keyword_collision" ], "error": null }, { "id": "51b90c09bceb80ca7d35", "expected": "general", "predicted": "summarization", "confidence": 0.9, "tags": [ "english", "explicit", "science_and_environment" ], "error": null }, { "id": "e002489e78030d58d6c5", "expected": "general", "predicted": "summarization", "confidence": 0.9, "tags": [ "education_and_history", "english", "quoted_distraction" ], "error": null }, { "id": "07da174977fc72b8d80b", "expected": "general", "predicted": "summarization", "confidence": 0.0, "tags": [ "education_and_history", "english", "quoted_distraction" ], "error": null }, { "id": "943f2dfdbd0c64238750", "expected": "math", "predicted": "reasoning", "confidence": 0.9, "tags": [ "english", "health_and_wellbeing", "keyword_collision" ], "error": null }, { "id": "3fdc5b750dc8d215d582", "expected": "general", "predicted": "reasoning", "confidence": 0.9, "tags": [ "english", "health_and_wellbeing", "keyword_collision" ], "error": null }, { "id": "c8ea33baf2fdbb8883eb", "expected": "general", "predicted": "reasoning", "confidence": 0.9, "tags": [ "arts_and_media", "english", "keyword_collision" ], "error": null }, { "id": "ab96097c1fcc369f0598", "expected": "general", "predicted": "summarization", "confidence": 0.9, "tags": [ "education_and_history", "english", "implicit" ], "error": null }, { "id": "d5266d5a0df7dfb41bcb", "expected": "general", "predicted": "reasoning", "confidence": 0.9, "tags": [ "english", "implicit", "science_and_environment" ], "error": null }, { "id": "ef4101ec661d1f9da2f3", "expected": "general", "predicted": "reasoning", "confidence": 0.9, "tags": [ "english", "explicit", "science_and_environment" ], "error": null }, { "id": "504733447d6f30d0efc9", "expected": "general", "predicted": "summarization", "confidence": 0.9, "tags": [ "english", "keyword_collision", "retail_and_support" ], "error": null }, { "id": "d65c6d2bc7bebd381ff1", "expected": "reasoning", "predicted": "summarization", "confidence": 0.9, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "f34dbbf72b10a606b663", "expected": "general", "predicted": "reasoning", "confidence": 0.0, "tags": [ "education_and_history", "english", "implicit" ], "error": null }, { "id": "72d4a621461880e7cbbe", "expected": "general", "predicted": "reasoning", "confidence": 0.9, "tags": [ "english", "keyword_collision", "science_and_environment" ], "error": null } ], "accuracy_ci95": [ 0.8885416666666667, 0.9354166666666667 ], "macro_f1_ci95": [ 0.8857677654300063, 0.9345618880925706 ], "by_task_ci95": { "coding": [ 0.85625, 0.96875 ], "reasoning": [ 0.94375, 0.99375 ], "creative_writing": [ 0.95625, 1.0 ], "summarization": [ 0.95625, 1.0 ], "math": [ 0.8875, 0.975 ], "general": [ 0.56875, 0.79375 ] } } ], "paired_comparisons": [ { "baseline": "axon-heuristic", "candidate": "laya", "accuracy_difference_pp": 16.458333333333332, "paired_ci95_pp": [ 12.604166666666671, 20.312499999999993 ], "familywise_ci95_pp": [ 11.354166666666664, 21.666666666666668 ], "separated_after_six_comparisons": true }, { "baseline": "axon-heuristic", "candidate": "strands", "accuracy_difference_pp": 41.5625, "paired_ci95_pp": [ 38.12500000000001, 44.99999999999999 ], "familywise_ci95_pp": [ 37.08333333333333, 46.14583333333333 ], "separated_after_six_comparisons": true }, { "baseline": "axon-heuristic", "candidate": "llm-router", "accuracy_difference_pp": 41.458333333333336, "paired_ci95_pp": [ 37.81249999999999, 45.104166666666664 ], "familywise_ci95_pp": [ 36.56250000000001, 46.35416666666667 ], "separated_after_six_comparisons": true }, { "baseline": "laya", "candidate": "strands", "accuracy_difference_pp": 25.104166666666668, "paired_ci95_pp": [ 21.875, 28.333333333333332 ], "familywise_ci95_pp": [ 20.833333333333332, 29.270833333333336 ], "separated_after_six_comparisons": true }, { "baseline": "laya", "candidate": "llm-router", "accuracy_difference_pp": 25.0, "paired_ci95_pp": [ 21.562500000000007, 28.333333333333332 ], "familywise_ci95_pp": [ 20.520833333333332, 29.47916666666667 ], "separated_after_six_comparisons": true }, { "baseline": "strands", "candidate": "llm-router", "accuracy_difference_pp": -0.10416666666666667, "paired_ci95_pp": [ -2.9166666666666674, 2.708333333333335 ], "familywise_ci95_pp": [ -4.062500000000002, 3.645833333333337 ], "separated_after_six_comparisons": false } ], "runtime": { "axon-heuristic": { "segments": [ { "started_at": "2026-10-02T19:09:01.280323+00:00", "first_record": 0, "initialization_ms": 0.008000002708286047, "warmup": [ { "strategy": "axon-heuristic", "task_type": "general", "confidence": 0.0, "latency_ms": 0.873417, "llm_calls": 0, "input_tokens": 0, "output_tokens": 0, "cost_usd": 0.0, "model": null, "cache_hit": false, "error": null, "heuristic_task": null, "heuristic_confidence": null, "confidence_kind": "heuristic_score", "probabilities": null, "native_confidence": null }, { "strategy": "axon-heuristic", "task_type": "coding", "confidence": 0.8823528373702545, "latency_ms": 0.091, "llm_calls": 0, "input_tokens": 0, "output_tokens": 0, "cost_usd": 0.0, "model": null, "cache_hit": false, "error": null, "heuristic_task": null, "heuristic_confidence": null, "confidence_kind": "heuristic_score", "probabilities": null, "native_confidence": null }, { "strategy": "axon-heuristic", "task_type": "general", "confidence": 0.0, "latency_ms": 0.062625, "llm_calls": 0, "input_tokens": 0, "output_tokens": 0, "cost_usd": 0.0, "model": null, "cache_hit": false, "error": null, "heuristic_task": null, "heuristic_confidence": null, "confidence_kind": "heuristic_score", "probabilities": null, "native_confidence": null } ], "completed_at": "2026-10-02T19:09:01.799776+00:00" } ], "initialization_ms": 0.008000002708286047, "warmup": [ { "strategy": "axon-heuristic", "task_type": "general", "confidence": 0.0, "latency_ms": 0.873417, "llm_calls": 0, "input_tokens": 0, "output_tokens": 0, "cost_usd": 0.0, "model": null, "cache_hit": false, "error": null, "heuristic_task": null, "heuristic_confidence": null, "confidence_kind": "heuristic_score", "probabilities": null, "native_confidence": null }, { "strategy": "axon-heuristic", "task_type": "coding", "confidence": 0.8823528373702545, "latency_ms": 0.091, "llm_calls": 0, "input_tokens": 0, "output_tokens": 0, "cost_usd": 0.0, "model": null, "cache_hit": false, "error": null, "heuristic_task": null, "heuristic_confidence": null, "confidence_kind": "heuristic_score", "probabilities": null, "native_confidence": null }, { "strategy": "axon-heuristic", "task_type": "general", "confidence": 0.0, "latency_ms": 0.062625, "llm_calls": 0, "input_tokens": 0, "output_tokens": 0, "cost_usd": 0.0, "model": null, "cache_hit": false, "error": null, "heuristic_task": null, "heuristic_confidence": null, "confidence_kind": "heuristic_score", "probabilities": null, "native_confidence": null } ] }, "laya": { "segments": [ { "started_at": "2026-10-02T19:09:05.846277+00:00", "first_record": 0, "initialization_ms": 4045.2739159954945, "warmup": [ { "strategy": "laya", "task_type": "general", "confidence": 0.9471, "latency_ms": 189.08916700456757, "llm_calls": 0, "input_tokens": 0, "output_tokens": 0, "cost_usd": 0.0, "model": "convaiinnovations/laya", "cache_hit": false, "error": null, "heuristic_task": null, "heuristic_confidence": null, "confidence_kind": "top_option_probability_uncalibrated_for_this_corpus", "probabilities": { "coding": 0.0019, "reasoning": 0.0063, "creative_writing": 0.0262, "summarization": 0.0169, "math": 0.0016, "general": 0.9471 }, "native_confidence": 0.8494 }, { "strategy": "laya", "task_type": "coding", "confidence": 0.9746, "latency_ms": 36.34754200174939, "llm_calls": 0, "input_tokens": 0, "output_tokens": 0, "cost_usd": 0.0, "model": "convaiinnovations/laya", "cache_hit": false, "error": null, "heuristic_task": null, "heuristic_confidence": null, "confidence_kind": "top_option_probability_uncalibrated_for_this_corpus", "probabilities": { "coding": 0.9746, "reasoning": 0.004, "creative_writing": 0.0032, "summarization": 0.0018, "math": 0.008, "general": 0.0083 }, "native_confidence": 0.9131 }, { "strategy": "laya", "task_type": "math", "confidence": 0.9759, "latency_ms": 34.65637500630692, "llm_calls": 0, "input_tokens": 0, "output_tokens": 0, "cost_usd": 0.0, "model": "convaiinnovations/laya", "cache_hit": false, "error": null, "heuristic_task": null, "heuristic_confidence": null, "confidence_kind": "top_option_probability_uncalibrated_for_this_corpus", "probabilities": { "coding": 0.0025, "reasoning": 0.0135, "creative_writing": 0.0021, "summarization": 0.0019, "math": 0.9759, "general": 0.0041 }, "native_confidence": 0.9194 } ], "completed_at": "2026-10-02T19:09:42.990455+00:00" } ], "model": "convaiinnovations/laya", "revision": "55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851", "code_revision": "fa9a2a7070b1789912a49ae24603bbfb1a78b001", "context_tokens": 512, "head_tokens": 192, "checkpoint": "english", "device": "mps", "packages": { "torch": "2.14.1", "transformers": "5.18.0", "huggingface-hub": "1.33.0", "laya": "0.3.24", "strands-decider": "0.0.1.dev4+gddd11994b" }, "compile": false, "seed": 0, "torch_threads": 10, "dtype": "torch.float32", "inference_dtype": "torch.float32", "checkpoint_routing": "disabled", "effective_temperatures": { "default": [ 1.6369030475616455, 1.2514300346374512, 1.983399510383606 ], "by_options": { "choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.5, "choice:2": 1.9063563346862793 } }, "temperature_note": "Upstream clamps the checkpoint's choice:11+ temperature. This benchmark uses six options, so that bucket is not used.", "initialization_ms": 4045.2739159954945, "warmup": [ { "strategy": "laya", "task_type": "general", "confidence": 0.9471, "latency_ms": 189.08916700456757, "llm_calls": 0, "input_tokens": 0, "output_tokens": 0, "cost_usd": 0.0, "model": "convaiinnovations/laya", "cache_hit": false, "error": null, "heuristic_task": null, "heuristic_confidence": null, "confidence_kind": "top_option_probability_uncalibrated_for_this_corpus", "probabilities": { "coding": 0.0019, "reasoning": 0.0063, "creative_writing": 0.0262, "summarization": 0.0169, "math": 0.0016, "general": 0.9471 }, "native_confidence": 0.8494 }, { "strategy": "laya", "task_type": "coding", "confidence": 0.9746, "latency_ms": 36.34754200174939, "llm_calls": 0, "input_tokens": 0, "output_tokens": 0, "cost_usd": 0.0, "model": "convaiinnovations/laya", "cache_hit": false, "error": null, "heuristic_task": null, "heuristic_confidence": null, "confidence_kind": "top_option_probability_uncalibrated_for_this_corpus", "probabilities": { "coding": 0.9746, "reasoning": 0.004, "creative_writing": 0.0032, "summarization": 0.0018, "math": 0.008, "general": 0.0083 }, "native_confidence": 0.9131 }, { "strategy": "laya", "task_type": "math", "confidence": 0.9759, "latency_ms": 34.65637500630692, "llm_calls": 0, "input_tokens": 0, "output_tokens": 0, "cost_usd": 0.0, "model": "convaiinnovations/laya", "cache_hit": false, "error": null, "heuristic_task": null, "heuristic_confidence": null, "confidence_kind": "top_option_probability_uncalibrated_for_this_corpus", "probabilities": { "coding": 0.0025, "reasoning": 0.0135, "creative_writing": 0.0021, "summarization": 0.0019, "math": 0.9759, "general": 0.0041 }, "native_confidence": 0.9194 } ] }, "strands": { "segments": [ { "started_at": "2026-10-02T19:09:44.807326+00:00", "first_record": 0, "initialization_ms": 1627.9327919983189, "warmup": [ { "strategy": "strands", "task_type": "general", "confidence": 0.7819, "latency_ms": 199.29116700222949, "llm_calls": 0, "input_tokens": 0, "output_tokens": 0, "cost_usd": 0.0, "model": "StrandsAgents/strands-decider-2B-hobson-v19", "cache_hit": false, "error": null, "heuristic_task": null, "heuristic_confidence": null, "confidence_kind": "top_option_probability_uncalibrated_for_this_corpus", "probabilities": { "coding": 0.0544, "reasoning": 0.0292, "creative_writing": 0.0872, "summarization": 0.0346, "math": 0.0126, "general": 0.7819 }, "native_confidence": 0.7383 }, { "strategy": "strands", "task_type": "coding", "confidence": 0.8878, "latency_ms": 148.69195900246268, "llm_calls": 0, "input_tokens": 0, "output_tokens": 0, "cost_usd": 0.0, "model": "StrandsAgents/strands-decider-2B-hobson-v19", "cache_hit": false, "error": null, "heuristic_task": null, "heuristic_confidence": null, "confidence_kind": "top_option_probability_uncalibrated_for_this_corpus", "probabilities": { "coding": 0.8878, "reasoning": 0.0077, "creative_writing": 0.0021, "summarization": 0.0031, "math": 0.0725, "general": 0.0268 }, "native_confidence": 0.8654 }, { "strategy": "strands", "task_type": "math", "confidence": 0.9578, "latency_ms": 148.38841700111516, "llm_calls": 0, "input_tokens": 0, "output_tokens": 0, "cost_usd": 0.0, "model": "StrandsAgents/strands-decider-2B-hobson-v19", "cache_hit": false, "error": null, "heuristic_task": null, "heuristic_confidence": null, "confidence_kind": "top_option_probability_uncalibrated_for_this_corpus", "probabilities": { "coding": 0.0042, "reasoning": 0.0078, "creative_writing": 0.0018, "summarization": 0.0033, "math": 0.9578, "general": 0.0251 }, "native_confidence": 0.9493 } ], "completed_at": "2026-10-02T19:12:26.709047+00:00" } ], "model": "StrandsAgents/strands-decider-2B-hobson-v19", "revision": "bb282d786bc251fd4e3068de3ada9ddbb38127cd", "code_revision": "ddd11994bc451ffc78fa30f43037c291fd4d44b6", "base_model": "Qwen/Qwen3.5-2B-Base", "base_revision": "b1485b2fa6dfa1287294f269f5fb618e03d52d7c", "context_tokens": 4096, "device": "mps", "packages": { "torch": "2.14.1", "transformers": "5.18.0", "huggingface-hub": "1.33.0", "laya": "0.3.24", "strands-decider": "0.0.1.dev4+gddd11994b" }, "compile": false, "seed": 0, "torch_threads": 10, "dtype": "torch.bfloat16", "strict_window": true, "shared_prefix_cache": "within request; no response cache", "optional_kernel_packages": { "causal-conv1d": null, "flash-linear-attention": null, "flash-attn": null }, "effective_temperatures": { "default": 0.9627721607677362, "by_kind": { "noul": 0.9107136998460428, "choice": 0.734189596436441, "score": 1.32780942142348 } }, "initialization_ms": 1627.9327919983189, "warmup": [ { "strategy": "strands", "task_type": "general", "confidence": 0.7819, "latency_ms": 199.29116700222949, "llm_calls": 0, "input_tokens": 0, "output_tokens": 0, "cost_usd": 0.0, "model": "StrandsAgents/strands-decider-2B-hobson-v19", "cache_hit": false, "error": null, "heuristic_task": null, "heuristic_confidence": null, "confidence_kind": "top_option_probability_uncalibrated_for_this_corpus", "probabilities": { "coding": 0.0544, "reasoning": 0.0292, "creative_writing": 0.0872, "summarization": 0.0346, "math": 0.0126, "general": 0.7819 }, "native_confidence": 0.7383 }, { "strategy": "strands", "task_type": "coding", "confidence": 0.8878, "latency_ms": 148.69195900246268, "llm_calls": 0, "input_tokens": 0, "output_tokens": 0, "cost_usd": 0.0, "model": "StrandsAgents/strands-decider-2B-hobson-v19", "cache_hit": false, "error": null, "heuristic_task": null, "heuristic_confidence": null, "confidence_kind": "top_option_probability_uncalibrated_for_this_corpus", "probabilities": { "coding": 0.8878, "reasoning": 0.0077, "creative_writing": 0.0021, "summarization": 0.0031, "math": 0.0725, "general": 0.0268 }, "native_confidence": 0.8654 }, { "strategy": "strands", "task_type": "math", "confidence": 0.9578, "latency_ms": 148.38841700111516, "llm_calls": 0, "input_tokens": 0, "output_tokens": 0, "cost_usd": 0.0, "model": "StrandsAgents/strands-decider-2B-hobson-v19", "cache_hit": false, "error": null, "heuristic_task": null, "heuristic_confidence": null, "confidence_kind": "top_option_probability_uncalibrated_for_this_corpus", "probabilities": { "coding": 0.0042, "reasoning": 0.0078, "creative_writing": 0.0018, "summarization": 0.0033, "math": 0.9578, "general": 0.0251 }, "native_confidence": 0.9493 } ] }, "llm-router": { "segments": [ { "started_at": "2026-10-02T19:12:27.075987+00:00", "first_record": 0, "initialization_ms": 0.08350000280188397, "warmup": [ { "strategy": "llm-router", "task_type": "general", "confidence": 1.0, "latency_ms": 1264.1230410008575, "llm_calls": 1, "input_tokens": 257, "output_tokens": 12, "cost_usd": 4.575e-05, "model": "gpt-4o-mini-2024-07-18", "cache_hit": false, "error": null, "heuristic_task": null, "heuristic_confidence": null, "confidence_kind": "llm_self_report", "probabilities": null, "native_confidence": null }, { "strategy": "llm-router", "task_type": "coding", "confidence": 1.0, "latency_ms": 997.9232090045116, "llm_calls": 1, "input_tokens": 263, "output_tokens": 12, "cost_usd": 4.664999999999999e-05, "model": "gpt-4o-mini-2024-07-18", "cache_hit": false, "error": null, "heuristic_task": null, "heuristic_confidence": null, "confidence_kind": "llm_self_report", "probabilities": null, "native_confidence": null }, { "strategy": "llm-router", "task_type": "math", "confidence": 1.0, "latency_ms": 909.2030409956351, "llm_calls": 1, "input_tokens": 261, "output_tokens": 12, "cost_usd": 4.6349999999999995e-05, "model": "gpt-4o-mini-2024-07-18", "cache_hit": false, "error": null, "heuristic_task": null, "heuristic_confidence": null, "confidence_kind": "llm_self_report", "probabilities": null, "native_confidence": null } ], "completed_at": "2026-10-02T19:24:56.998096+00:00" } ], "initialization_ms": 0.08350000280188397, "warmup": [ { "strategy": "llm-router", "task_type": "general", "confidence": 1.0, "latency_ms": 1264.1230410008575, "llm_calls": 1, "input_tokens": 257, "output_tokens": 12, "cost_usd": 4.575e-05, "model": "gpt-4o-mini-2024-07-18", "cache_hit": false, "error": null, "heuristic_task": null, "heuristic_confidence": null, "confidence_kind": "llm_self_report", "probabilities": null, "native_confidence": null }, { "strategy": "llm-router", "task_type": "coding", "confidence": 1.0, "latency_ms": 997.9232090045116, "llm_calls": 1, "input_tokens": 263, "output_tokens": 12, "cost_usd": 4.664999999999999e-05, "model": "gpt-4o-mini-2024-07-18", "cache_hit": false, "error": null, "heuristic_task": null, "heuristic_confidence": null, "confidence_kind": "llm_self_report", "probabilities": null, "native_confidence": null }, { "strategy": "llm-router", "task_type": "math", "confidence": 1.0, "latency_ms": 909.2030409956351, "llm_calls": 1, "input_tokens": 261, "output_tokens": 12, "cost_usd": 4.6349999999999995e-05, "model": "gpt-4o-mini-2024-07-18", "cache_hit": false, "error": null, "heuristic_task": null, "heuristic_confidence": null, "confidence_kind": "llm_self_report", "probabilities": null, "native_confidence": null } ] } }, "configuration": { "panel": "primary", "corpus_manifest_sha256": "ec68993165e9cbf24e63a3e09d561e6162edc784885acd9ccb9f22479044f53b", "case_order_sha256": "8931dd8d30fde19c8de568bfdb611fc836945962f66095ba3e34eb109d48bfb9", "device": "mps", "repetitions": 1, "local_files_only": true, "strategies": [ "axon-heuristic", "laya", "strands", "llm-router" ], "llm_model": "gpt-4o-mini-2024-07-18", "llm_json_schema": true, "llm_input_cost_per_million": 0.15, "llm_output_cost_per_million": 0.6, "llm_cache_bust": true, "llm_max_output_tokens": 256, "llm_timeout_seconds": 60, "concurrency": 1, "warmup_requests": 3, "seed": 775, "runtime_lock_sha256": "ea96150ead1f720130b6ebeac7dad70a1dcfea2d69a8989dab37a89ad27da1b0" }, "environment": { "python": "3.13.12", "system": "Darwin", "release": "25.6.0", "machine": "arm64" }, "repeatability": { "unique_cases": 120, "families": 30, "repetitions": 3, "strategies": [ { "strategy": "axon-heuristic", "unique_cases": 120, "cases_with_prediction_changes": 0 }, { "strategy": "laya", "unique_cases": 120, "cases_with_prediction_changes": 0 }, { "strategy": "strands", "unique_cases": 120, "cases_with_prediction_changes": 0 }, { "strategy": "llm-router", "unique_cases": 120, "cases_with_prediction_changes": 0 } ], "summaries": [ { "strategy": "axon-heuristic", "requests": 360, "correct": 204, "accuracy": 0.5666666666666667, "macro_f1": 0.5558051782189714, "errors": 0, "cache_hits": 0, "latency_ms": { "mean": 0.4785031222222222, "p50": 0.304958, "p95": 1.4355291500000003, "p99": 3.0612615300000012 }, "latency_scope": "successful requests only", "all_attempts_latency_ms": { "p50": 0.304958, "p95": 1.4355291500000003 }, "llm_calls": 0, "llm_call_rate": 0.0, "input_tokens": 0, "output_tokens": 0, "router_cost_usd": 0.0, "known_router_cost_usd": 0.0, "cost_coverage": 1.0, "cost_scope": "API charges only; local hardware and electricity excluded", "hardware_electricity_cost_usd": null, "hardware_electricity_cost_status": "unmeasured", "router_cost_per_1k_requests_usd": 0.0, "confusion_matrix": { "coding": { "coding": 54, "general": 6 }, "reasoning": { "general": 42, "reasoning": 15, "summarization": 3 }, "creative_writing": { "coding": 3, "creative_writing": 27, "general": 30 }, "summarization": { "general": 18, "summarization": 42 }, "math": { "coding": 3, "general": 48, "math": 6, "summarization": 3 }, "general": { "general": 60 } }, "by_task": { "coding": { "requests": 60, "correct": 54, "accuracy": 0.9 }, "reasoning": { "requests": 60, "correct": 15, "accuracy": 0.25 }, "creative_writing": { "requests": 60, "correct": 27, "accuracy": 0.45 }, "summarization": { "requests": 60, "correct": 42, "accuracy": 0.7 }, "math": { "requests": 60, "correct": 6, "accuracy": 0.1 }, "general": { "requests": 60, "correct": 60, "accuracy": 1.0 } }, "by_tag": { "arts_and_media": { "requests": 72, "accuracy": 0.375 }, "education_and_history": { "requests": 12, "accuracy": 0.5 }, "english": { "requests": 360, "accuracy": 0.5666666666666667 }, "explicit": { "requests": 90, "accuracy": 0.7 }, "health_and_wellbeing": { "requests": 36, "accuracy": 0.6666666666666666 }, "home_and_travel": { "requests": 36, "accuracy": 0.5 }, "implicit": { "requests": 90, "accuracy": 0.4666666666666667 }, "keyword_collision": { "requests": 90, "accuracy": 0.5666666666666667 }, "public_services": { "requests": 36, "accuracy": 0.75 }, "quoted_distraction": { "requests": 90, "accuracy": 0.5333333333333333 }, "retail_and_support": { "requests": 36, "accuracy": 0.6666666666666666 }, "science_and_environment": { "requests": 24, "accuracy": 0.375 }, "software_and_data": { "requests": 12, "accuracy": 1.0 }, "sports_and_leisure": { "requests": 60, "accuracy": 0.65 }, "workplace_and_business": { "requests": 36, "accuracy": 0.5 } }, "wrong_cases": [ { "id": "168bd22a2f90cba24345", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "implicit" ], "error": null }, { "id": "0f1c4364cf65fa073901", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "sports_and_leisure" ], "error": null }, { "id": "7b7863705f9079319049", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "workplace_and_business" ], "error": null }, { "id": "348e1c4ea62a66aff9ea", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "explicit" ], "error": null }, { "id": "0a3dde9b7138a4b8df44", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "science_and_environment" ], "error": null }, { "id": "be1d66a4d2ff1afcf414", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "education_and_history", "english", "quoted_distraction" ], "error": null }, { "id": "c715c42a3a8ec0dcda2c", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "workplace_and_business" ], "error": null }, { "id": "a061c5108e9501a72790", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "education_and_history", "english", "explicit" ], "error": null }, { "id": "38fa57e835cade3b7ee8", "expected": "math", "predicted": "summarization", "confidence": 0.9999997500000625, "tags": [ "english", "home_and_travel", "keyword_collision" ], "error": null }, { "id": "11a493e5b6c2f9665457", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "quoted_distraction" ], "error": null }, { "id": "3f5ea624e6b566bca6f6", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "sports_and_leisure" ], "error": null }, { "id": "884036295bc6527dd26d", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "implicit" ], "error": null }, { "id": "b61ac98d9fc06327314f", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "1e7e6038c35305ad39cd", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "science_and_environment" ], "error": null }, { "id": "0c9421bdebab5df308d2", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "science_and_environment" ], "error": null }, { "id": "a5a0ad02a3a8e8f51474", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "implicit" ], "error": null }, { "id": "83cd435b7e45907f9716", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "ed7cf02e86191b90926d", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "explicit" ], "error": null }, { "id": "e616750c8c7fbdc2194b", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "keyword_collision" ], "error": null }, { "id": "e89c49f2408537936571", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "retail_and_support" ], "error": null }, { "id": "c6061d47eebee24ebb8d", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "e22c70f63feb13865cde", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "health_and_wellbeing", "keyword_collision" ], "error": null }, { "id": "f05f79295511ba0f0af6", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "home_and_travel" ], "error": null }, { "id": "73bae6342993f8d7d7bc", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "health_and_wellbeing", "implicit" ], "error": null }, { "id": "87a7f4436b7438713f22", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "health_and_wellbeing", "keyword_collision" ], "error": null }, { "id": "4f4983b8517fe7d8745e", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "health_and_wellbeing", "quoted_distraction" ], "error": null }, { "id": "2cce175108805f6828d3", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "keyword_collision" ], "error": null }, { "id": "2017324475c364638788", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "835e4a244a5fa09e02d4", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "workplace_and_business" ], "error": null }, { "id": "cb95c73725be1e468ad5", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "sports_and_leisure" ], "error": null }, { "id": "974d254b39db7ef31937", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "workplace_and_business" ], "error": null }, { "id": "915570a621fff0121127", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "explicit" ], "error": null }, { "id": "4a756cc5e9d8f76bdabe", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "retail_and_support" ], "error": null }, { "id": "c3961f339329d5625ba3", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "public_services", "quoted_distraction" ], "error": null }, { "id": "582b7ecfc374940f96d0", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "public_services" ], "error": null }, { "id": "04b0be7a016981756d27", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "science_and_environment" ], "error": null }, { "id": "a89afa71a134defd0788", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "implicit" ], "error": null }, { "id": "fd91948f511a05c56916", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "5431b08ebc1b0f08f88d", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "science_and_environment" ], "error": null }, { "id": "362e9048d6b5caa6fc80", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "implicit" ], "error": null }, { "id": "5d129c1fe8f22bdf3805", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "explicit" ], "error": null }, { "id": "b20f7e3d48ad50c3f2aa", "expected": "reasoning", "predicted": "summarization", "confidence": 0.9999997500000625, "tags": [ "english", "explicit", "sports_and_leisure" ], "error": null }, { "id": "1e8fb49c4fa09d6a00c2", "expected": "math", "predicted": "coding", "confidence": 0.6666665555555741, "tags": [ "arts_and_media", "english", "keyword_collision" ], "error": null }, { "id": "d65c6d2bc7bebd381ff1", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "0992496ed8d3f9ec1fb6", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "workplace_and_business" ], "error": null }, { "id": "5d1029eb9a244be44a89", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "retail_and_support" ], "error": null }, { "id": "312c8c0e101c1695122d", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "public_services" ], "error": null }, { "id": "c01ba1ff99d62992d182", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "e5b82b29d27a8e485db1", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "workplace_and_business" ], "error": null }, { "id": "df01fb7ad4d4fe17b24b", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "d41ad266a6d363f201ef", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "implicit" ], "error": null }, { "id": "e58631029d8f261abac3", "expected": "creative_writing", "predicted": "coding", "confidence": 0.6666664444445185, "tags": [ "arts_and_media", "english", "keyword_collision" ], "error": null }, { "id": "168bd22a2f90cba24345", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "implicit" ], "error": null }, { "id": "0f1c4364cf65fa073901", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "sports_and_leisure" ], "error": null }, { "id": "7b7863705f9079319049", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "workplace_and_business" ], "error": null }, { "id": "348e1c4ea62a66aff9ea", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "explicit" ], "error": null }, { "id": "0a3dde9b7138a4b8df44", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "science_and_environment" ], "error": null }, { "id": "be1d66a4d2ff1afcf414", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "education_and_history", "english", "quoted_distraction" ], "error": null }, { "id": "c715c42a3a8ec0dcda2c", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "workplace_and_business" ], "error": null }, { "id": "a061c5108e9501a72790", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "education_and_history", "english", "explicit" ], "error": null }, { "id": "38fa57e835cade3b7ee8", "expected": "math", "predicted": "summarization", "confidence": 0.9999997500000625, "tags": [ "english", "home_and_travel", "keyword_collision" ], "error": null }, { "id": "11a493e5b6c2f9665457", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "quoted_distraction" ], "error": null }, { "id": "3f5ea624e6b566bca6f6", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "sports_and_leisure" ], "error": null }, { "id": "884036295bc6527dd26d", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "implicit" ], "error": null }, { "id": "b61ac98d9fc06327314f", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "1e7e6038c35305ad39cd", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "science_and_environment" ], "error": null }, { "id": "0c9421bdebab5df308d2", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "science_and_environment" ], "error": null }, { "id": "a5a0ad02a3a8e8f51474", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "implicit" ], "error": null }, { "id": "83cd435b7e45907f9716", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "ed7cf02e86191b90926d", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "explicit" ], "error": null }, { "id": "e616750c8c7fbdc2194b", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "keyword_collision" ], "error": null }, { "id": "e89c49f2408537936571", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "retail_and_support" ], "error": null }, { "id": "c6061d47eebee24ebb8d", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "e22c70f63feb13865cde", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "health_and_wellbeing", "keyword_collision" ], "error": null }, { "id": "f05f79295511ba0f0af6", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "home_and_travel" ], "error": null }, { "id": "73bae6342993f8d7d7bc", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "health_and_wellbeing", "implicit" ], "error": null }, { "id": "87a7f4436b7438713f22", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "health_and_wellbeing", "keyword_collision" ], "error": null }, { "id": "4f4983b8517fe7d8745e", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "health_and_wellbeing", "quoted_distraction" ], "error": null }, { "id": "2cce175108805f6828d3", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "keyword_collision" ], "error": null }, { "id": "2017324475c364638788", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "835e4a244a5fa09e02d4", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "workplace_and_business" ], "error": null }, { "id": "cb95c73725be1e468ad5", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "sports_and_leisure" ], "error": null }, { "id": "974d254b39db7ef31937", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "workplace_and_business" ], "error": null }, { "id": "915570a621fff0121127", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "explicit" ], "error": null }, { "id": "4a756cc5e9d8f76bdabe", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "retail_and_support" ], "error": null }, { "id": "c3961f339329d5625ba3", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "public_services", "quoted_distraction" ], "error": null }, { "id": "582b7ecfc374940f96d0", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "public_services" ], "error": null }, { "id": "04b0be7a016981756d27", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "science_and_environment" ], "error": null }, { "id": "a89afa71a134defd0788", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "implicit" ], "error": null }, { "id": "fd91948f511a05c56916", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "5431b08ebc1b0f08f88d", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "science_and_environment" ], "error": null }, { "id": "362e9048d6b5caa6fc80", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "implicit" ], "error": null }, { "id": "5d129c1fe8f22bdf3805", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "explicit" ], "error": null }, { "id": "b20f7e3d48ad50c3f2aa", "expected": "reasoning", "predicted": "summarization", "confidence": 0.9999997500000625, "tags": [ "english", "explicit", "sports_and_leisure" ], "error": null }, { "id": "1e8fb49c4fa09d6a00c2", "expected": "math", "predicted": "coding", "confidence": 0.6666665555555741, "tags": [ "arts_and_media", "english", "keyword_collision" ], "error": null }, { "id": "d65c6d2bc7bebd381ff1", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "0992496ed8d3f9ec1fb6", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "workplace_and_business" ], "error": null }, { "id": "5d1029eb9a244be44a89", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "retail_and_support" ], "error": null }, { "id": "312c8c0e101c1695122d", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "public_services" ], "error": null }, { "id": "c01ba1ff99d62992d182", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "e5b82b29d27a8e485db1", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "workplace_and_business" ], "error": null }, { "id": "df01fb7ad4d4fe17b24b", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "d41ad266a6d363f201ef", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "implicit" ], "error": null }, { "id": "e58631029d8f261abac3", "expected": "creative_writing", "predicted": "coding", "confidence": 0.6666664444445185, "tags": [ "arts_and_media", "english", "keyword_collision" ], "error": null }, { "id": "168bd22a2f90cba24345", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "implicit" ], "error": null }, { "id": "0f1c4364cf65fa073901", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "sports_and_leisure" ], "error": null }, { "id": "7b7863705f9079319049", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "workplace_and_business" ], "error": null }, { "id": "348e1c4ea62a66aff9ea", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "explicit" ], "error": null }, { "id": "0a3dde9b7138a4b8df44", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "science_and_environment" ], "error": null }, { "id": "be1d66a4d2ff1afcf414", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "education_and_history", "english", "quoted_distraction" ], "error": null }, { "id": "c715c42a3a8ec0dcda2c", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "workplace_and_business" ], "error": null }, { "id": "a061c5108e9501a72790", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "education_and_history", "english", "explicit" ], "error": null }, { "id": "38fa57e835cade3b7ee8", "expected": "math", "predicted": "summarization", "confidence": 0.9999997500000625, "tags": [ "english", "home_and_travel", "keyword_collision" ], "error": null }, { "id": "11a493e5b6c2f9665457", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "quoted_distraction" ], "error": null }, { "id": "3f5ea624e6b566bca6f6", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "sports_and_leisure" ], "error": null }, { "id": "884036295bc6527dd26d", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "implicit" ], "error": null }, { "id": "b61ac98d9fc06327314f", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "1e7e6038c35305ad39cd", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "science_and_environment" ], "error": null }, { "id": "0c9421bdebab5df308d2", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "science_and_environment" ], "error": null }, { "id": "a5a0ad02a3a8e8f51474", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "implicit" ], "error": null }, { "id": "83cd435b7e45907f9716", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "ed7cf02e86191b90926d", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "explicit" ], "error": null }, { "id": "e616750c8c7fbdc2194b", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "keyword_collision" ], "error": null }, { "id": "e89c49f2408537936571", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "quoted_distraction", "retail_and_support" ], "error": null }, { "id": "c6061d47eebee24ebb8d", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "e22c70f63feb13865cde", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "health_and_wellbeing", "keyword_collision" ], "error": null }, { "id": "f05f79295511ba0f0af6", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "home_and_travel" ], "error": null }, { "id": "73bae6342993f8d7d7bc", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "health_and_wellbeing", "implicit" ], "error": null }, { "id": "87a7f4436b7438713f22", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "health_and_wellbeing", "keyword_collision" ], "error": null }, { "id": "4f4983b8517fe7d8745e", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "health_and_wellbeing", "quoted_distraction" ], "error": null }, { "id": "2cce175108805f6828d3", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "keyword_collision" ], "error": null }, { "id": "2017324475c364638788", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "835e4a244a5fa09e02d4", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "workplace_and_business" ], "error": null }, { "id": "cb95c73725be1e468ad5", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "sports_and_leisure" ], "error": null }, { "id": "974d254b39db7ef31937", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "workplace_and_business" ], "error": null }, { "id": "915570a621fff0121127", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "explicit" ], "error": null }, { "id": "4a756cc5e9d8f76bdabe", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "retail_and_support" ], "error": null }, { "id": "c3961f339329d5625ba3", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "public_services", "quoted_distraction" ], "error": null }, { "id": "582b7ecfc374940f96d0", "expected": "summarization", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "public_services" ], "error": null }, { "id": "04b0be7a016981756d27", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "science_and_environment" ], "error": null }, { "id": "a89afa71a134defd0788", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "implicit" ], "error": null }, { "id": "fd91948f511a05c56916", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "5431b08ebc1b0f08f88d", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "science_and_environment" ], "error": null }, { "id": "362e9048d6b5caa6fc80", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "implicit" ], "error": null }, { "id": "5d129c1fe8f22bdf3805", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "explicit" ], "error": null }, { "id": "b20f7e3d48ad50c3f2aa", "expected": "reasoning", "predicted": "summarization", "confidence": 0.9999997500000625, "tags": [ "english", "explicit", "sports_and_leisure" ], "error": null }, { "id": "1e8fb49c4fa09d6a00c2", "expected": "math", "predicted": "coding", "confidence": 0.6666665555555741, "tags": [ "arts_and_media", "english", "keyword_collision" ], "error": null }, { "id": "d65c6d2bc7bebd381ff1", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "0992496ed8d3f9ec1fb6", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "english", "keyword_collision", "workplace_and_business" ], "error": null }, { "id": "5d1029eb9a244be44a89", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "explicit", "retail_and_support" ], "error": null }, { "id": "312c8c0e101c1695122d", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "public_services" ], "error": null }, { "id": "c01ba1ff99d62992d182", "expected": "reasoning", "predicted": "general", "confidence": 0.0, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "e5b82b29d27a8e485db1", "expected": "creative_writing", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "workplace_and_business" ], "error": null }, { "id": "df01fb7ad4d4fe17b24b", "expected": "coding", "predicted": "general", "confidence": 0.0, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "d41ad266a6d363f201ef", "expected": "math", "predicted": "general", "confidence": 0.0, "tags": [ "english", "home_and_travel", "implicit" ], "error": null }, { "id": "e58631029d8f261abac3", "expected": "creative_writing", "predicted": "coding", "confidence": 0.6666664444445185, "tags": [ "arts_and_media", "english", "keyword_collision" ], "error": null } ] }, { "strategy": "laya", "requests": 360, "correct": 273, "accuracy": 0.7583333333333333, "macro_f1": 0.7615880977714083, "errors": 0, "cache_hits": 0, "latency_ms": { "mean": 39.1654851749182, "p50": 38.279874501313316, "p95": 48.143129799791495, "p99": 51.09445425281592 }, "latency_scope": "successful requests only", "all_attempts_latency_ms": { "p50": 38.279874501313316, "p95": 48.143129799791495 }, "llm_calls": 0, "llm_call_rate": 0.0, "input_tokens": 0, "output_tokens": 0, "router_cost_usd": 0.0, "known_router_cost_usd": 0.0, "cost_coverage": 1.0, "cost_scope": "API charges only; local hardware and electricity excluded", "hardware_electricity_cost_usd": null, "hardware_electricity_cost_status": "unmeasured", "router_cost_per_1k_requests_usd": 0.0, "confusion_matrix": { "coding": { "coding": 57, "math": 3 }, "reasoning": { "creative_writing": 3, "general": 15, "reasoning": 36, "summarization": 6 }, "creative_writing": { "creative_writing": 57, "general": 3 }, "summarization": { "creative_writing": 3, "general": 18, "summarization": 39 }, "math": { "general": 12, "math": 39, "reasoning": 6, "summarization": 3 }, "general": { "coding": 9, "creative_writing": 6, "general": 45 } }, "by_task": { "coding": { "requests": 60, "correct": 57, "accuracy": 0.95 }, "reasoning": { "requests": 60, "correct": 36, "accuracy": 0.6 }, "creative_writing": { "requests": 60, "correct": 57, "accuracy": 0.95 }, "summarization": { "requests": 60, "correct": 39, "accuracy": 0.65 }, "math": { "requests": 60, "correct": 39, "accuracy": 0.65 }, "general": { "requests": 60, "correct": 45, "accuracy": 0.75 } }, "by_tag": { "arts_and_media": { "requests": 72, "accuracy": 0.625 }, "education_and_history": { "requests": 12, "accuracy": 0.5 }, "english": { "requests": 360, "accuracy": 0.7583333333333333 }, "explicit": { "requests": 90, "accuracy": 0.9 }, "health_and_wellbeing": { "requests": 36, "accuracy": 1.0 }, "home_and_travel": { "requests": 36, "accuracy": 0.6666666666666666 }, "implicit": { "requests": 90, "accuracy": 0.5333333333333333 }, "keyword_collision": { "requests": 90, "accuracy": 0.8 }, "public_services": { "requests": 36, "accuracy": 0.8333333333333334 }, "quoted_distraction": { "requests": 90, "accuracy": 0.8 }, "retail_and_support": { "requests": 36, "accuracy": 0.75 }, "science_and_environment": { "requests": 24, "accuracy": 1.0 }, "software_and_data": { "requests": 12, "accuracy": 0.25 }, "sports_and_leisure": { "requests": 60, "accuracy": 0.8 }, "workplace_and_business": { "requests": 36, "accuracy": 0.8333333333333334 } }, "wrong_cases": [ { "id": "168bd22a2f90cba24345", "expected": "math", "predicted": "general", "confidence": 0.5403, "tags": [ "arts_and_media", "english", "implicit" ], "error": null }, { "id": "82f634ae9419d72b41aa", "expected": "reasoning", "predicted": "general", "confidence": 0.6181, "tags": [ "education_and_history", "english", "implicit" ], "error": null }, { "id": "348e1c4ea62a66aff9ea", "expected": "math", "predicted": "general", "confidence": 0.5373, "tags": [ "arts_and_media", "english", "explicit" ], "error": null }, { "id": "be1d66a4d2ff1afcf414", "expected": "reasoning", "predicted": "creative_writing", "confidence": 0.9978, "tags": [ "education_and_history", "english", "quoted_distraction" ], "error": null }, { "id": "38fa57e835cade3b7ee8", "expected": "math", "predicted": "summarization", "confidence": 0.653, "tags": [ "english", "home_and_travel", "keyword_collision" ], "error": null }, { "id": "3f5ea624e6b566bca6f6", "expected": "summarization", "predicted": "general", "confidence": 0.6646, "tags": [ "english", "keyword_collision", "sports_and_leisure" ], "error": null }, { "id": "b61ac98d9fc06327314f", "expected": "summarization", "predicted": "general", "confidence": 0.3465, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "83cd435b7e45907f9716", "expected": "math", "predicted": "general", "confidence": 0.3952, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "e616750c8c7fbdc2194b", "expected": "math", "predicted": "general", "confidence": 0.4199, "tags": [ "arts_and_media", "english", "keyword_collision" ], "error": null }, { "id": "15a225ff569ecd16a83b", "expected": "general", "predicted": "creative_writing", "confidence": 0.9865, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "2cce175108805f6828d3", "expected": "summarization", "predicted": "general", "confidence": 0.5657, "tags": [ "english", "home_and_travel", "keyword_collision" ], "error": null }, { "id": "2017324475c364638788", "expected": "math", "predicted": "reasoning", "confidence": 0.448, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "835e4a244a5fa09e02d4", "expected": "reasoning", "predicted": "general", "confidence": 0.618, "tags": [ "english", "implicit", "workplace_and_business" ], "error": null }, { "id": "d29cb8254a1c42be9bab", "expected": "general", "predicted": "coding", "confidence": 0.3936, "tags": [ "english", "implicit", "software_and_data" ], "error": null }, { "id": "7f48179d7d548473a00f", "expected": "general", "predicted": "coding", "confidence": 0.6573, "tags": [ "english", "explicit", "software_and_data" ], "error": null }, { "id": "58f21bc291bc998b894b", "expected": "summarization", "predicted": "general", "confidence": 0.413, "tags": [ "english", "implicit", "public_services" ], "error": null }, { "id": "f3c735797b200b0ff388", "expected": "summarization", "predicted": "general", "confidence": 0.5947, "tags": [ "english", "home_and_travel", "implicit" ], "error": null }, { "id": "582b7ecfc374940f96d0", "expected": "summarization", "predicted": "creative_writing", "confidence": 0.392, "tags": [ "english", "keyword_collision", "public_services" ], "error": null }, { "id": "a89afa71a134defd0788", "expected": "reasoning", "predicted": "general", "confidence": 0.7811, "tags": [ "arts_and_media", "english", "implicit" ], "error": null }, { "id": "fd91948f511a05c56916", "expected": "reasoning", "predicted": "summarization", "confidence": 0.3765, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "79a6cec2c9a0d3943095", "expected": "summarization", "predicted": "general", "confidence": 0.4072, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "5d129c1fe8f22bdf3805", "expected": "reasoning", "predicted": "general", "confidence": 0.4963, "tags": [ "arts_and_media", "english", "explicit" ], "error": null }, { "id": "a6e98cf3b4cacd4aa15f", "expected": "general", "predicted": "coding", "confidence": 0.8292, "tags": [ "english", "keyword_collision", "software_and_data" ], "error": null }, { "id": "d65c6d2bc7bebd381ff1", "expected": "reasoning", "predicted": "summarization", "confidence": 0.9341, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "c01ba1ff99d62992d182", "expected": "reasoning", "predicted": "general", "confidence": 0.5157, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "e5b82b29d27a8e485db1", "expected": "creative_writing", "predicted": "general", "confidence": 0.3999, "tags": [ "english", "implicit", "workplace_and_business" ], "error": null }, { "id": "581527a786d856c205fe", "expected": "coding", "predicted": "math", "confidence": 0.7238, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "d41ad266a6d363f201ef", "expected": "math", "predicted": "reasoning", "confidence": 0.5758, "tags": [ "english", "home_and_travel", "implicit" ], "error": null }, { "id": "83012f85dabaaec55095", "expected": "general", "predicted": "creative_writing", "confidence": 0.9993, "tags": [ "english", "quoted_distraction", "sports_and_leisure" ], "error": null }, { "id": "168bd22a2f90cba24345", "expected": "math", "predicted": "general", "confidence": 0.5403, "tags": [ "arts_and_media", "english", "implicit" ], "error": null }, { "id": "82f634ae9419d72b41aa", "expected": "reasoning", "predicted": "general", "confidence": 0.6181, "tags": [ "education_and_history", "english", "implicit" ], "error": null }, { "id": "348e1c4ea62a66aff9ea", "expected": "math", "predicted": "general", "confidence": 0.5373, "tags": [ "arts_and_media", "english", "explicit" ], "error": null }, { "id": "be1d66a4d2ff1afcf414", "expected": "reasoning", "predicted": "creative_writing", "confidence": 0.9978, "tags": [ "education_and_history", "english", "quoted_distraction" ], "error": null }, { "id": "38fa57e835cade3b7ee8", "expected": "math", "predicted": "summarization", "confidence": 0.653, "tags": [ "english", "home_and_travel", "keyword_collision" ], "error": null }, { "id": "3f5ea624e6b566bca6f6", "expected": "summarization", "predicted": "general", "confidence": 0.6646, "tags": [ "english", "keyword_collision", "sports_and_leisure" ], "error": null }, { "id": "b61ac98d9fc06327314f", "expected": "summarization", "predicted": "general", "confidence": 0.3465, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "83cd435b7e45907f9716", "expected": "math", "predicted": "general", "confidence": 0.3952, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "e616750c8c7fbdc2194b", "expected": "math", "predicted": "general", "confidence": 0.4199, "tags": [ "arts_and_media", "english", "keyword_collision" ], "error": null }, { "id": "15a225ff569ecd16a83b", "expected": "general", "predicted": "creative_writing", "confidence": 0.9865, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "2cce175108805f6828d3", "expected": "summarization", "predicted": "general", "confidence": 0.5657, "tags": [ "english", "home_and_travel", "keyword_collision" ], "error": null }, { "id": "2017324475c364638788", "expected": "math", "predicted": "reasoning", "confidence": 0.448, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "835e4a244a5fa09e02d4", "expected": "reasoning", "predicted": "general", "confidence": 0.618, "tags": [ "english", "implicit", "workplace_and_business" ], "error": null }, { "id": "d29cb8254a1c42be9bab", "expected": "general", "predicted": "coding", "confidence": 0.3936, "tags": [ "english", "implicit", "software_and_data" ], "error": null }, { "id": "7f48179d7d548473a00f", "expected": "general", "predicted": "coding", "confidence": 0.6573, "tags": [ "english", "explicit", "software_and_data" ], "error": null }, { "id": "58f21bc291bc998b894b", "expected": "summarization", "predicted": "general", "confidence": 0.413, "tags": [ "english", "implicit", "public_services" ], "error": null }, { "id": "f3c735797b200b0ff388", "expected": "summarization", "predicted": "general", "confidence": 0.5947, "tags": [ "english", "home_and_travel", "implicit" ], "error": null }, { "id": "582b7ecfc374940f96d0", "expected": "summarization", "predicted": "creative_writing", "confidence": 0.392, "tags": [ "english", "keyword_collision", "public_services" ], "error": null }, { "id": "a89afa71a134defd0788", "expected": "reasoning", "predicted": "general", "confidence": 0.7811, "tags": [ "arts_and_media", "english", "implicit" ], "error": null }, { "id": "fd91948f511a05c56916", "expected": "reasoning", "predicted": "summarization", "confidence": 0.3765, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "79a6cec2c9a0d3943095", "expected": "summarization", "predicted": "general", "confidence": 0.4072, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "5d129c1fe8f22bdf3805", "expected": "reasoning", "predicted": "general", "confidence": 0.4963, "tags": [ "arts_and_media", "english", "explicit" ], "error": null }, { "id": "a6e98cf3b4cacd4aa15f", "expected": "general", "predicted": "coding", "confidence": 0.8292, "tags": [ "english", "keyword_collision", "software_and_data" ], "error": null }, { "id": "d65c6d2bc7bebd381ff1", "expected": "reasoning", "predicted": "summarization", "confidence": 0.9341, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "c01ba1ff99d62992d182", "expected": "reasoning", "predicted": "general", "confidence": 0.5157, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "e5b82b29d27a8e485db1", "expected": "creative_writing", "predicted": "general", "confidence": 0.3999, "tags": [ "english", "implicit", "workplace_and_business" ], "error": null }, { "id": "581527a786d856c205fe", "expected": "coding", "predicted": "math", "confidence": 0.7238, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "d41ad266a6d363f201ef", "expected": "math", "predicted": "reasoning", "confidence": 0.5758, "tags": [ "english", "home_and_travel", "implicit" ], "error": null }, { "id": "83012f85dabaaec55095", "expected": "general", "predicted": "creative_writing", "confidence": 0.9993, "tags": [ "english", "quoted_distraction", "sports_and_leisure" ], "error": null }, { "id": "168bd22a2f90cba24345", "expected": "math", "predicted": "general", "confidence": 0.5403, "tags": [ "arts_and_media", "english", "implicit" ], "error": null }, { "id": "82f634ae9419d72b41aa", "expected": "reasoning", "predicted": "general", "confidence": 0.6181, "tags": [ "education_and_history", "english", "implicit" ], "error": null }, { "id": "348e1c4ea62a66aff9ea", "expected": "math", "predicted": "general", "confidence": 0.5373, "tags": [ "arts_and_media", "english", "explicit" ], "error": null }, { "id": "be1d66a4d2ff1afcf414", "expected": "reasoning", "predicted": "creative_writing", "confidence": 0.9978, "tags": [ "education_and_history", "english", "quoted_distraction" ], "error": null }, { "id": "38fa57e835cade3b7ee8", "expected": "math", "predicted": "summarization", "confidence": 0.653, "tags": [ "english", "home_and_travel", "keyword_collision" ], "error": null }, { "id": "3f5ea624e6b566bca6f6", "expected": "summarization", "predicted": "general", "confidence": 0.6646, "tags": [ "english", "keyword_collision", "sports_and_leisure" ], "error": null }, { "id": "b61ac98d9fc06327314f", "expected": "summarization", "predicted": "general", "confidence": 0.3465, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "83cd435b7e45907f9716", "expected": "math", "predicted": "general", "confidence": 0.3952, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "e616750c8c7fbdc2194b", "expected": "math", "predicted": "general", "confidence": 0.4199, "tags": [ "arts_and_media", "english", "keyword_collision" ], "error": null }, { "id": "15a225ff569ecd16a83b", "expected": "general", "predicted": "creative_writing", "confidence": 0.9865, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "2cce175108805f6828d3", "expected": "summarization", "predicted": "general", "confidence": 0.5657, "tags": [ "english", "home_and_travel", "keyword_collision" ], "error": null }, { "id": "2017324475c364638788", "expected": "math", "predicted": "reasoning", "confidence": 0.448, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "835e4a244a5fa09e02d4", "expected": "reasoning", "predicted": "general", "confidence": 0.618, "tags": [ "english", "implicit", "workplace_and_business" ], "error": null }, { "id": "d29cb8254a1c42be9bab", "expected": "general", "predicted": "coding", "confidence": 0.3936, "tags": [ "english", "implicit", "software_and_data" ], "error": null }, { "id": "7f48179d7d548473a00f", "expected": "general", "predicted": "coding", "confidence": 0.6573, "tags": [ "english", "explicit", "software_and_data" ], "error": null }, { "id": "58f21bc291bc998b894b", "expected": "summarization", "predicted": "general", "confidence": 0.413, "tags": [ "english", "implicit", "public_services" ], "error": null }, { "id": "f3c735797b200b0ff388", "expected": "summarization", "predicted": "general", "confidence": 0.5947, "tags": [ "english", "home_and_travel", "implicit" ], "error": null }, { "id": "582b7ecfc374940f96d0", "expected": "summarization", "predicted": "creative_writing", "confidence": 0.392, "tags": [ "english", "keyword_collision", "public_services" ], "error": null }, { "id": "a89afa71a134defd0788", "expected": "reasoning", "predicted": "general", "confidence": 0.7811, "tags": [ "arts_and_media", "english", "implicit" ], "error": null }, { "id": "fd91948f511a05c56916", "expected": "reasoning", "predicted": "summarization", "confidence": 0.3765, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "79a6cec2c9a0d3943095", "expected": "summarization", "predicted": "general", "confidence": 0.4072, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "5d129c1fe8f22bdf3805", "expected": "reasoning", "predicted": "general", "confidence": 0.4963, "tags": [ "arts_and_media", "english", "explicit" ], "error": null }, { "id": "a6e98cf3b4cacd4aa15f", "expected": "general", "predicted": "coding", "confidence": 0.8292, "tags": [ "english", "keyword_collision", "software_and_data" ], "error": null }, { "id": "d65c6d2bc7bebd381ff1", "expected": "reasoning", "predicted": "summarization", "confidence": 0.9341, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "c01ba1ff99d62992d182", "expected": "reasoning", "predicted": "general", "confidence": 0.5157, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "e5b82b29d27a8e485db1", "expected": "creative_writing", "predicted": "general", "confidence": 0.3999, "tags": [ "english", "implicit", "workplace_and_business" ], "error": null }, { "id": "581527a786d856c205fe", "expected": "coding", "predicted": "math", "confidence": 0.7238, "tags": [ "english", "implicit", "retail_and_support" ], "error": null }, { "id": "d41ad266a6d363f201ef", "expected": "math", "predicted": "reasoning", "confidence": 0.5758, "tags": [ "english", "home_and_travel", "implicit" ], "error": null }, { "id": "83012f85dabaaec55095", "expected": "general", "predicted": "creative_writing", "confidence": 0.9993, "tags": [ "english", "quoted_distraction", "sports_and_leisure" ], "error": null } ] }, { "strategy": "strands", "requests": 360, "correct": 321, "accuracy": 0.8916666666666667, "macro_f1": 0.8887292931349019, "errors": 0, "cache_hits": 0, "latency_ms": { "mean": 170.5983440944288, "p50": 164.52731249955832, "p95": 214.93081874978088, "p99": 229.69755874488334 }, "latency_scope": "successful requests only", "all_attempts_latency_ms": { "p50": 164.52731249955832, "p95": 214.93081874978088 }, "llm_calls": 0, "llm_call_rate": 0.0, "input_tokens": 0, "output_tokens": 0, "router_cost_usd": 0.0, "known_router_cost_usd": 0.0, "cost_coverage": 1.0, "cost_scope": "API charges only; local hardware and electricity excluded", "hardware_electricity_cost_usd": null, "hardware_electricity_cost_status": "unmeasured", "router_cost_per_1k_requests_usd": 0.0, "confusion_matrix": { "coding": { "coding": 60 }, "reasoning": { "coding": 3, "creative_writing": 3, "general": 9, "reasoning": 42, "summarization": 3 }, "creative_writing": { "creative_writing": 57, "general": 3 }, "summarization": { "summarization": 60 }, "math": { "math": 60 }, "general": { "coding": 15, "creative_writing": 3, "general": 42 } }, "by_task": { "coding": { "requests": 60, "correct": 60, "accuracy": 1.0 }, "reasoning": { "requests": 60, "correct": 42, "accuracy": 0.7 }, "creative_writing": { "requests": 60, "correct": 57, "accuracy": 0.95 }, "summarization": { "requests": 60, "correct": 60, "accuracy": 1.0 }, "math": { "requests": 60, "correct": 60, "accuracy": 1.0 }, "general": { "requests": 60, "correct": 42, "accuracy": 0.7 } }, "by_tag": { "arts_and_media": { "requests": 72, "accuracy": 0.8333333333333334 }, "education_and_history": { "requests": 12, "accuracy": 0.75 }, "english": { "requests": 360, "accuracy": 0.8916666666666667 }, "explicit": { "requests": 90, "accuracy": 0.9333333333333333 }, "health_and_wellbeing": { "requests": 36, "accuracy": 0.9166666666666666 }, "home_and_travel": { "requests": 36, "accuracy": 1.0 }, "implicit": { "requests": 90, "accuracy": 0.9 }, "keyword_collision": { "requests": 90, "accuracy": 0.9 }, "public_services": { "requests": 36, "accuracy": 0.9166666666666666 }, "quoted_distraction": { "requests": 90, "accuracy": 0.8333333333333334 }, "retail_and_support": { "requests": 36, "accuracy": 1.0 }, "science_and_environment": { "requests": 24, "accuracy": 1.0 }, "software_and_data": { "requests": 12, "accuracy": 0.0 }, "sports_and_leisure": { "requests": 60, "accuracy": 0.95 }, "workplace_and_business": { "requests": 36, "accuracy": 0.9166666666666666 } }, "wrong_cases": [ { "id": "be1d66a4d2ff1afcf414", "expected": "reasoning", "predicted": "creative_writing", "confidence": 0.6882, "tags": [ "education_and_history", "english", "quoted_distraction" ], "error": null }, { "id": "c715c42a3a8ec0dcda2c", "expected": "reasoning", "predicted": "coding", "confidence": 0.4718, "tags": [ "english", "quoted_distraction", "workplace_and_business" ], "error": null }, { "id": "e22c70f63feb13865cde", "expected": "creative_writing", "predicted": "general", "confidence": 0.6832, "tags": [ "english", "health_and_wellbeing", "keyword_collision" ], "error": null }, { "id": "15a225ff569ecd16a83b", "expected": "general", "predicted": "creative_writing", "confidence": 0.5047, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "dc5d9921cacf189ea3a7", "expected": "general", "predicted": "coding", "confidence": 0.4791, "tags": [ "english", "keyword_collision", "public_services" ], "error": null }, { "id": "d29cb8254a1c42be9bab", "expected": "general", "predicted": "coding", "confidence": 0.7737, "tags": [ "english", "implicit", "software_and_data" ], "error": null }, { "id": "7f48179d7d548473a00f", "expected": "general", "predicted": "coding", "confidence": 0.7351, "tags": [ "english", "explicit", "software_and_data" ], "error": null }, { "id": "a89afa71a134defd0788", "expected": "reasoning", "predicted": "general", "confidence": 0.5181, "tags": [ "arts_and_media", "english", "implicit" ], "error": null }, { "id": "fd91948f511a05c56916", "expected": "reasoning", "predicted": "general", "confidence": 0.3548, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "5d129c1fe8f22bdf3805", "expected": "reasoning", "predicted": "general", "confidence": 0.4243, "tags": [ "arts_and_media", "english", "explicit" ], "error": null }, { "id": "a6e98cf3b4cacd4aa15f", "expected": "general", "predicted": "coding", "confidence": 0.7153, "tags": [ "english", "keyword_collision", "software_and_data" ], "error": null }, { "id": "d65c6d2bc7bebd381ff1", "expected": "reasoning", "predicted": "summarization", "confidence": 0.5172, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "19b36d3302e3b1a40897", "expected": "general", "predicted": "coding", "confidence": 0.79, "tags": [ "english", "quoted_distraction", "software_and_data" ], "error": null }, { "id": "be1d66a4d2ff1afcf414", "expected": "reasoning", "predicted": "creative_writing", "confidence": 0.6882, "tags": [ "education_and_history", "english", "quoted_distraction" ], "error": null }, { "id": "c715c42a3a8ec0dcda2c", "expected": "reasoning", "predicted": "coding", "confidence": 0.4718, "tags": [ "english", "quoted_distraction", "workplace_and_business" ], "error": null }, { "id": "e22c70f63feb13865cde", "expected": "creative_writing", "predicted": "general", "confidence": 0.6832, "tags": [ "english", "health_and_wellbeing", "keyword_collision" ], "error": null }, { "id": "15a225ff569ecd16a83b", "expected": "general", "predicted": "creative_writing", "confidence": 0.5047, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "dc5d9921cacf189ea3a7", "expected": "general", "predicted": "coding", "confidence": 0.4791, "tags": [ "english", "keyword_collision", "public_services" ], "error": null }, { "id": "d29cb8254a1c42be9bab", "expected": "general", "predicted": "coding", "confidence": 0.7737, "tags": [ "english", "implicit", "software_and_data" ], "error": null }, { "id": "7f48179d7d548473a00f", "expected": "general", "predicted": "coding", "confidence": 0.7351, "tags": [ "english", "explicit", "software_and_data" ], "error": null }, { "id": "a89afa71a134defd0788", "expected": "reasoning", "predicted": "general", "confidence": 0.5181, "tags": [ "arts_and_media", "english", "implicit" ], "error": null }, { "id": "fd91948f511a05c56916", "expected": "reasoning", "predicted": "general", "confidence": 0.3548, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "5d129c1fe8f22bdf3805", "expected": "reasoning", "predicted": "general", "confidence": 0.4243, "tags": [ "arts_and_media", "english", "explicit" ], "error": null }, { "id": "a6e98cf3b4cacd4aa15f", "expected": "general", "predicted": "coding", "confidence": 0.7153, "tags": [ "english", "keyword_collision", "software_and_data" ], "error": null }, { "id": "d65c6d2bc7bebd381ff1", "expected": "reasoning", "predicted": "summarization", "confidence": 0.5172, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "19b36d3302e3b1a40897", "expected": "general", "predicted": "coding", "confidence": 0.79, "tags": [ "english", "quoted_distraction", "software_and_data" ], "error": null }, { "id": "be1d66a4d2ff1afcf414", "expected": "reasoning", "predicted": "creative_writing", "confidence": 0.6882, "tags": [ "education_and_history", "english", "quoted_distraction" ], "error": null }, { "id": "c715c42a3a8ec0dcda2c", "expected": "reasoning", "predicted": "coding", "confidence": 0.4718, "tags": [ "english", "quoted_distraction", "workplace_and_business" ], "error": null }, { "id": "e22c70f63feb13865cde", "expected": "creative_writing", "predicted": "general", "confidence": 0.6832, "tags": [ "english", "health_and_wellbeing", "keyword_collision" ], "error": null }, { "id": "15a225ff569ecd16a83b", "expected": "general", "predicted": "creative_writing", "confidence": 0.5047, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "dc5d9921cacf189ea3a7", "expected": "general", "predicted": "coding", "confidence": 0.4791, "tags": [ "english", "keyword_collision", "public_services" ], "error": null }, { "id": "d29cb8254a1c42be9bab", "expected": "general", "predicted": "coding", "confidence": 0.7737, "tags": [ "english", "implicit", "software_and_data" ], "error": null }, { "id": "7f48179d7d548473a00f", "expected": "general", "predicted": "coding", "confidence": 0.7351, "tags": [ "english", "explicit", "software_and_data" ], "error": null }, { "id": "a89afa71a134defd0788", "expected": "reasoning", "predicted": "general", "confidence": 0.5181, "tags": [ "arts_and_media", "english", "implicit" ], "error": null }, { "id": "fd91948f511a05c56916", "expected": "reasoning", "predicted": "general", "confidence": 0.3548, "tags": [ "english", "implicit", "sports_and_leisure" ], "error": null }, { "id": "5d129c1fe8f22bdf3805", "expected": "reasoning", "predicted": "general", "confidence": 0.4243, "tags": [ "arts_and_media", "english", "explicit" ], "error": null }, { "id": "a6e98cf3b4cacd4aa15f", "expected": "general", "predicted": "coding", "confidence": 0.7153, "tags": [ "english", "keyword_collision", "software_and_data" ], "error": null }, { "id": "d65c6d2bc7bebd381ff1", "expected": "reasoning", "predicted": "summarization", "confidence": 0.5172, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "19b36d3302e3b1a40897", "expected": "general", "predicted": "coding", "confidence": 0.79, "tags": [ "english", "quoted_distraction", "software_and_data" ], "error": null } ] }, { "strategy": "llm-router", "requests": 360, "correct": 351, "accuracy": 0.975, "macro_f1": 0.9749582289055972, "errors": 0, "cache_hits": 0, "latency_ms": { "mean": 730.8187030164037, "p50": 575.5622085016512, "p95": 910.4565103963979, "p99": 1308.5456847190658 }, "latency_scope": "successful requests only", "all_attempts_latency_ms": { "p50": 575.5622085016512, "p95": 910.4565103963979 }, "llm_calls": 360, "llm_call_rate": 1.0, "input_tokens": 104432, "output_tokens": 4632, "router_cost_usd": 0.018444, "known_router_cost_usd": 0.018444, "cost_coverage": 1.0, "cost_scope": "API charges only; local hardware and electricity excluded", "hardware_electricity_cost_usd": null, "hardware_electricity_cost_status": "unmeasured", "router_cost_per_1k_requests_usd": 0.05123333333333333, "confusion_matrix": { "coding": { "coding": 54, "reasoning": 3, "summarization": 3 }, "reasoning": { "reasoning": 57, "summarization": 3 }, "creative_writing": { "creative_writing": 60 }, "summarization": { "summarization": 60 }, "math": { "math": 60 }, "general": { "general": 60 } }, "by_task": { "coding": { "requests": 60, "correct": 54, "accuracy": 0.9 }, "reasoning": { "requests": 60, "correct": 57, "accuracy": 0.95 }, "creative_writing": { "requests": 60, "correct": 60, "accuracy": 1.0 }, "summarization": { "requests": 60, "correct": 60, "accuracy": 1.0 }, "math": { "requests": 60, "correct": 60, "accuracy": 1.0 }, "general": { "requests": 60, "correct": 60, "accuracy": 1.0 } }, "by_tag": { "arts_and_media": { "requests": 72, "accuracy": 0.9583333333333334 }, "education_and_history": { "requests": 12, "accuracy": 1.0 }, "english": { "requests": 360, "accuracy": 0.975 }, "explicit": { "requests": 90, "accuracy": 1.0 }, "health_and_wellbeing": { "requests": 36, "accuracy": 0.9166666666666666 }, "home_and_travel": { "requests": 36, "accuracy": 1.0 }, "implicit": { "requests": 90, "accuracy": 1.0 }, "keyword_collision": { "requests": 90, "accuracy": 0.9333333333333333 }, "public_services": { "requests": 36, "accuracy": 1.0 }, "quoted_distraction": { "requests": 90, "accuracy": 0.9666666666666667 }, "retail_and_support": { "requests": 36, "accuracy": 0.9166666666666666 }, "science_and_environment": { "requests": 24, "accuracy": 1.0 }, "software_and_data": { "requests": 12, "accuracy": 1.0 }, "sports_and_leisure": { "requests": 60, "accuracy": 1.0 }, "workplace_and_business": { "requests": 36, "accuracy": 1.0 } }, "wrong_cases": [ { "id": "ece20e79ca05b460150a", "expected": "coding", "predicted": "summarization", "confidence": 0.9, "tags": [ "english", "keyword_collision", "retail_and_support" ], "error": null }, { "id": "2b7a8968b5bbe84c5b2e", "expected": "coding", "predicted": "reasoning", "confidence": 0.9, "tags": [ "english", "health_and_wellbeing", "keyword_collision" ], "error": null }, { "id": "d65c6d2bc7bebd381ff1", "expected": "reasoning", "predicted": "summarization", "confidence": 0.9, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "ece20e79ca05b460150a", "expected": "coding", "predicted": "summarization", "confidence": 0.9, "tags": [ "english", "keyword_collision", "retail_and_support" ], "error": null }, { "id": "2b7a8968b5bbe84c5b2e", "expected": "coding", "predicted": "reasoning", "confidence": 0.9, "tags": [ "english", "health_and_wellbeing", "keyword_collision" ], "error": null }, { "id": "d65c6d2bc7bebd381ff1", "expected": "reasoning", "predicted": "summarization", "confidence": 0.9, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null }, { "id": "ece20e79ca05b460150a", "expected": "coding", "predicted": "summarization", "confidence": 0.9, "tags": [ "english", "keyword_collision", "retail_and_support" ], "error": null }, { "id": "2b7a8968b5bbe84c5b2e", "expected": "coding", "predicted": "reasoning", "confidence": 0.9, "tags": [ "english", "health_and_wellbeing", "keyword_collision" ], "error": null }, { "id": "d65c6d2bc7bebd381ff1", "expected": "reasoning", "predicted": "summarization", "confidence": 0.9, "tags": [ "arts_and_media", "english", "quoted_distraction" ], "error": null } ] } ] } }