{ "schema_version": 2, "generated_at": "2026-08-20T22:35:56.157Z", "release_status": "release_ready", "release_qa": { "pass": true, "completed_at": "2026-08-20T22:30:57Z", "checks": [ { "id": "checkpoint-accounting", "pass": true, "detail": "All 8,640 latest direct-evaluation trial checkpoints resolve; retained terminal records are genuine agent or protocol outcomes." }, { "id": "reproducibility-suite", "pass": true, "detail": "Type checking, 47 automated tests, all 45 representative calculator checks, generated report, and Astro static build pass." }, { "id": "public-artifact-boundary", "pass": true, "detail": "Public status and score downloads expose only task-normalized economics; aggregate spend, planning forecasts, raw usage, credentials, and private reasoning are absent." }, { "id": "rendered-local-audit", "pass": true, "detail": "Desktop and mobile audits cover the benchmark and methodology routes, interaction labels, visual legends, route navigation, and horizontal table handling." }, { "id": "preview-integrity", "pass": true, "detail": "The protected Vercel preview is ready; four public benchmark artifacts return successfully and match the locally built prebuilt payload byte-for-byte." } ] }, "operation_count": 45, "independent_reference_ready_count": 45, "headline_eligible_count": 0, "credibility_summary": { "unverified": 1, "code_verified": 31, "reference_validated": 0, "empirically_validated": 0, "blocked": 13 }, "evidence_policy": { "mode": "automated_open_evidence", "peer_reviewed": false, "human_review_required": false }, "development": { "task_count": 32, "scripted_reference_rate": 1, "forced_runner_rate": 1, "automated_evidence_gate_pass": true, "pilot_gate_pass": true, "accounted_trials": 768, "expected_trials": 768, "primary_effect": { "comparison": "nemotron-full-skills minus nemotron-base", "endpoint": "task-level key-answer correctness", "n": 96, "operation_count": 4, "estimate": 0.25, "ci95": { "low": -0.02083333333333333, "high": 0.6041666666666666 }, "iterations": 10000, "hierarchy": "operation -> task -> repeat" }, "factorial": { "documentation_main_effect": -0.015625, "runner_main_effect": 0.26562499999999994, "interaction_effect": -0.09375, "cell_accuracy": { "base": 0.3854166666666667, "docs_only": 0.4166666666666667, "runner_only": 0.6979166666666666, "full_skills": 0.6354166666666666 } }, "replications": { "terra": { "n": 96, "operation_count": 4, "estimate": 0.41666666666666663, "ci95": { "low": 0.16666666666666666, "high": 0.7083333333333333 }, "iterations": 10000, "hierarchy": "operation -> task -> repeat" }, "sol": { "n": 96, "operation_count": 4, "estimate": 0.40625, "ci95": { "low": 0.14583333333333331, "high": 0.6875 }, "iterations": 10000, "hierarchy": "operation -> task -> repeat" } }, "arms": { "nemotron-base": { "n": 96, "task_accuracy": 0.3854166666666667, "pass_at_1": 0.34375, "pass_at_3": 0.46875, "pass_power_3": 0.28125, "terminal_failure_rate": 0.08333333333333333, "runner_invocation_rate_on_required_tasks": 0 }, "nemotron-docs-only": { "n": 96, "task_accuracy": 0.4166666666666667, "pass_at_1": 0.4375, "pass_at_3": 0.5, "pass_power_3": 0.34375, "terminal_failure_rate": 0.052083333333333336, "runner_invocation_rate_on_required_tasks": 0 }, "nemotron-runner-only": { "n": 96, "task_accuracy": 0.6979166666666666, "pass_at_1": 0.6875, "pass_at_3": 0.8125, "pass_power_3": 0.53125, "terminal_failure_rate": 0.020833333333333332, "runner_invocation_rate_on_required_tasks": 0.9814814814814815 }, "nemotron-full-skills": { "n": 96, "task_accuracy": 0.6354166666666666, "pass_at_1": 0.59375, "pass_at_3": 0.75, "pass_power_3": 0.5, "terminal_failure_rate": 0, "runner_invocation_rate_on_required_tasks": 1 }, "terra-base": { "n": 96, "task_accuracy": 0.4375, "pass_at_1": 0.4375, "pass_at_3": 0.5, "pass_power_3": 0.34375, "terminal_failure_rate": 0, "runner_invocation_rate_on_required_tasks": 0 }, "terra-full-skills": { "n": 96, "task_accuracy": 0.8541666666666666, "pass_at_1": 0.84375, "pass_at_3": 0.875, "pass_power_3": 0.84375, "terminal_failure_rate": 0, "runner_invocation_rate_on_required_tasks": 1 }, "sol-base": { "n": 96, "task_accuracy": 0.4375, "pass_at_1": 0.4375, "pass_at_3": 0.46875, "pass_power_3": 0.375, "terminal_failure_rate": 0, "runner_invocation_rate_on_required_tasks": 0 }, "sol-full-skills": { "n": 96, "task_accuracy": 0.84375, "pass_at_1": 0.78125, "pass_at_3": 0.90625, "pass_power_3": 0.78125, "terminal_failure_rate": 0, "runner_invocation_rate_on_required_tasks": 1 } }, "gate_checks": { "all_expected_trials_accounted_for": { "pass": true, "observed": 768, "required": 768 }, "forced_correct_runner_calls": { "pass": true, "observed": 1, "required": 0.99 }, "terminal_harness_failure_rate": { "pass": true, "observed": 0, "observed_count": 0, "required_less_than": 0.01, "reasons": {} }, "terminal_agent_failure_rate": { "informational": true, "observed": 0.01953125, "observed_count": 15, "reasons": { "invalid_final": 13, "turn_limit_exhausted": 1, "required_runner_not_invoked": 1 } }, "tool_required_runner_invocation": { "pass": true, "observed": 1, "required": 0.95 }, "five_point_uplift_power": { "pass": true, "observed": 0.8846, "required": 0.8, "simulation": { "power": 0.8846, "tasks": 360, "repeats": 3, "simulations": 5000, "baseline": 0.5, "uplift": 0.05, "intraclass_correlation_assumption": 0.2, "test": "two-sided normal approximation on paired task means", "seed": 5000005 } }, "zero_or_perfect_operation_audit": { "pass": true, "method": "deterministic artifact-integrity audit", "extreme_count": 3, "unresolved_count": 0 } } }, "evaluation": { "status": "frozen_automated_evidence", "task_count": 360, "expected_trials": 8640, "automated_evidence_gate_pass": true, "technical_preflight_pass": true, "technical_preflight_at": "2026-08-18T01:43:01.366Z", "scores": { "schema_version": 2, "generated_at": "2026-08-20T22:09:29.893Z", "split": "evaluation", "namespace": "direct", "task_count": 360, "expected_trials": 8640, "accounted_trials": 8640, "complete": true, "protocol_audit": { "excluded_attempt_count": 11, "excluded_unique_trial_count": 11, "by_arm": { "nemotron-base": { "attempt_count": 0, "unique_trial_count": 0, "violations": {} }, "nemotron-docs-only": { "attempt_count": 5, "unique_trial_count": 5, "violations": { "skill_read_limit_exceeded": 5 } }, "nemotron-runner-only": { "attempt_count": 2, "unique_trial_count": 2, "violations": { "runner_call_limit_exceeded": 2 } }, "nemotron-full-skills": { "attempt_count": 4, "unique_trial_count": 4, "violations": { "runner_call_limit_exceeded": 4 } }, "terra-base": { "attempt_count": 0, "unique_trial_count": 0, "violations": {} }, "terra-full-skills": { "attempt_count": 0, "unique_trial_count": 0, "violations": {} }, "sol-base": { "attempt_count": 0, "unique_trial_count": 0, "violations": {} }, "sol-full-skills": { "attempt_count": 0, "unique_trial_count": 0, "violations": {} } }, "note": "Attempts that exceeded preregistered model-turn or constrained-tool limits are retained in raw JSONL for auditability but excluded from resume completion and scoring." }, "preregistered_primary": { "comparison": "nemotron-full-skills minus nemotron-base", "endpoint": "task-level key-answer correctness", "n": 1080, "operation_count": 45, "estimate": 0.2907407407407407, "ci95": { "low": 0.20648148148148152, "high": 0.3768518518518519 }, "iterations": 10000, "hierarchy": "operation -> task -> repeat" }, "factorial": { "documentation_main_effect": -0.03564814814814811, "runner_main_effect": 0.32638888888888884, "interaction_effect": -0.06018518518518509, "cell_accuracy": { "base": 0.22870370370370371, "docs_only": 0.22314814814814815, "runner_only": 0.5851851851851851, "full_skills": 0.5194444444444445 } }, "replications": { "terra": { "n": 1080, "operation_count": 45, "estimate": 0.5259259259259259, "ci95": { "low": 0.41571759259259256, "high": 0.6296296296296297 }, "iterations": 10000, "hierarchy": "operation -> task -> repeat" }, "sol": { "n": 1080, "operation_count": 45, "estimate": 0.4083333333333333, "ci95": { "low": 0.3175925925925926, "high": 0.4990740740740739 }, "iterations": 10000, "hierarchy": "operation -> task -> repeat" } }, "arms": { "nemotron-base": { "n": 1080, "task_accuracy": 0.22870370370370371, "macro_operation_accuracy": 0.2287037037037037, "field_accuracy": 0.04135802469135803, "normalized_numerical_error": 24072411.683395404, "status_accuracy": 0.5787037037037037, "format_validity": 0.9416666666666667, "pass_at_1": 0.2361111111111111, "pass_at_3": 0.2833333333333333, "pass_power_3": 0.1638888888888889, "terminal_failure_rate": 0.058333333333333334, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "needs_clarification": { "needs_clarification": 123, "invalid_input": 1, "terminal_or_invalid": 11 }, "success": { "needs_clarification": 292, "success": 398, "out_of_domain": 23, "invalid_input": 47, "terminal_or_invalid": 50 }, "out_of_domain": { "out_of_domain": 104, "invalid_input": 29, "terminal_or_invalid": 2 } }, "cost_per_correct_task_usd": 0 }, "nemotron-docs-only": { "n": 1080, "task_accuracy": 0.22314814814814815, "macro_operation_accuracy": 0.22314814814814818, "field_accuracy": 0.040123456790123455, "normalized_numerical_error": 7821152.95466037, "status_accuracy": 0.6953703703703704, "format_validity": 0.9527777777777777, "pass_at_1": 0.21944444444444444, "pass_at_3": 0.2777777777777778, "pass_power_3": 0.1527777777777778, "terminal_failure_rate": 0.04722222222222222, "mean_runner_calls": 0, "mean_skill_reads": 1.262962962962963, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "needs_clarification": { "needs_clarification": 131, "terminal_or_invalid": 3, "invalid_input": 1 }, "success": { "needs_clarification": 205, "success": 528, "terminal_or_invalid": 45, "out_of_domain": 16, "invalid_input": 16 }, "out_of_domain": { "out_of_domain": 92, "invalid_input": 39, "terminal_or_invalid": 3, "needs_clarification": 1 } }, "cost_per_correct_task_usd": 0 }, "nemotron-runner-only": { "n": 1080, "task_accuracy": 0.5851851851851851, "macro_operation_accuracy": 0.585185185185185, "field_accuracy": 0.5117283950617284, "normalized_numerical_error": 1217809.8080047788, "status_accuracy": 0.9324074074074075, "format_validity": 0.9574074074074074, "pass_at_1": 0.5777777777777777, "pass_at_3": 0.7472222222222222, "pass_power_3": 0.3972222222222222, "terminal_failure_rate": 0.04259259259259259, "mean_runner_calls": 0.8425925925925926, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0.9861111111111112, "classification_matrix": { "needs_clarification": { "needs_clarification": 122, "terminal_or_invalid": 12, "invalid_input": 1 }, "success": { "success": 770, "terminal_or_invalid": 31, "out_of_domain": 3, "needs_clarification": 6 }, "out_of_domain": { "out_of_domain": 115, "invalid_input": 17, "terminal_or_invalid": 3 } }, "cost_per_correct_task_usd": 0 }, "nemotron-full-skills": { "n": 1080, "task_accuracy": 0.5194444444444445, "macro_operation_accuracy": 0.5194444444444445, "field_accuracy": 0.45308641975308644, "normalized_numerical_error": 2712117.5270400215, "status_accuracy": 0.8972222222222223, "format_validity": 0.9629629629629629, "pass_at_1": 0.5333333333333333, "pass_at_3": 0.7083333333333334, "pass_power_3": 0.30277777777777776, "terminal_failure_rate": 0.037037037037037035, "mean_runner_calls": 0.9648148148148148, "mean_skill_reads": 1.2712962962962964, "runner_invocation_rate_on_required_tasks": 0.9888888888888889, "classification_matrix": { "needs_clarification": { "needs_clarification": 130, "terminal_or_invalid": 5 }, "success": { "success": 757, "terminal_or_invalid": 32, "needs_clarification": 15, "out_of_domain": 6 }, "out_of_domain": { "invalid_input": 50, "out_of_domain": 82, "terminal_or_invalid": 3 } }, "cost_per_correct_task_usd": 0 }, "terra-base": { "n": 1080, "task_accuracy": 0.3101851851851852, "macro_operation_accuracy": 0.3101851851851852, "field_accuracy": 0.10740740740740741, "normalized_numerical_error": 7356.3261180986465, "status_accuracy": 0.45, "format_validity": 0.9981481481481481, "pass_at_1": 0.3111111111111111, "pass_at_3": 0.325, "pass_power_3": 0.28888888888888886, "terminal_failure_rate": 0.001851851851851852, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "needs_clarification": { "needs_clarification": 135 }, "success": { "needs_clarification": 582, "success": 218, "terminal_or_invalid": 2, "invalid_input": 7, "out_of_domain": 1 }, "out_of_domain": { "out_of_domain": 133, "invalid_input": 2 } }, "cost_per_correct_task_usd": 0.025327768656716425 }, "terra-full-skills": { "n": 1080, "task_accuracy": 0.8361111111111111, "macro_operation_accuracy": 0.8361111111111111, "field_accuracy": 0.7925925925925926, "normalized_numerical_error": 0.05047315385864652, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.8388888888888889, "pass_at_3": 0.85, "pass_power_3": 0.8194444444444444, "terminal_failure_rate": 0, "mean_runner_calls": 0.7574074074074074, "mean_skill_reads": 1.5712962962962962, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "needs_clarification": { "needs_clarification": 135 }, "success": { "success": 810 }, "out_of_domain": { "out_of_domain": 135 } }, "cost_per_correct_task_usd": 0.008981655315614612 }, "sol-base": { "n": 1080, "task_accuracy": 0.1962962962962963, "macro_operation_accuracy": 0.19629629629629627, "field_accuracy": 0.08518518518518518, "normalized_numerical_error": 1190701.5319339512, "status_accuracy": 0.33611111111111114, "format_validity": 1, "pass_at_1": 0.18888888888888888, "pass_at_3": 0.2222222222222222, "pass_power_3": 0.16666666666666666, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "needs_clarification": { "needs_clarification": 135 }, "success": { "needs_clarification": 598, "success": 206, "invalid_input": 6 }, "out_of_domain": { "invalid_input": 113, "out_of_domain": 22 } }, "cost_per_correct_task_usd": 0.08739573113207534 }, "sol-full-skills": { "n": 1080, "task_accuracy": 0.6046296296296296, "macro_operation_accuracy": 0.6046296296296297, "field_accuracy": 0.6080246913580247, "normalized_numerical_error": 0.07898204565304745, "status_accuracy": 0.9027777777777778, "format_validity": 1, "pass_at_1": 0.6166666666666667, "pass_at_3": 0.7361111111111112, "pass_power_3": 0.4666666666666667, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.8074074074074074, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "needs_clarification": { "needs_clarification": 135 }, "success": { "success": 800, "needs_clarification": 10 }, "out_of_domain": { "out_of_domain": 40, "invalid_input": 95 } }, "cost_per_correct_task_usd": 0.03194095405819294 } }, "by_skill": { "nemotron-base": { "flight-planning": { "n": 192, "task_accuracy": 0.203125, "macro_operation_accuracy": 0.20312499999999997, "field_accuracy": 0, "normalized_numerical_error": 58354286.35786516, "status_accuracy": 0.5989583333333334, "format_validity": 0.96875, "pass_at_1": 0.21875, "pass_at_3": 0.25, "pass_power_3": 0.140625, "terminal_failure_rate": 0.03125, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "out_of_domain": 8, "invalid_input": 22, "success": 76, "needs_clarification": 34, "terminal_or_invalid": 4 }, "out_of_domain": { "out_of_domain": 17, "invalid_input": 7 }, "needs_clarification": { "needs_clarification": 22, "terminal_or_invalid": 2 } }, "cost_per_correct_task_usd": 0 }, "atmosphere": { "n": 48, "task_accuracy": 0.22916666666666666, "macro_operation_accuracy": 0.22916666666666669, "field_accuracy": 0.027777777777777776, "normalized_numerical_error": 0, "status_accuracy": 0.7291666666666666, "format_validity": 0.8958333333333334, "pass_at_1": 0.1875, "pass_at_3": 0.3125, "pass_power_3": 0.125, "terminal_failure_rate": 0.10416666666666667, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "out_of_domain": { "out_of_domain": 5, "invalid_input": 1 }, "needs_clarification": { "needs_clarification": 5, "terminal_or_invalid": 1 }, "success": { "success": 25, "terminal_or_invalid": 4, "needs_clarification": 5, "out_of_domain": 2 } }, "cost_per_correct_task_usd": 0 }, "coordinate-frames": { "n": 72, "task_accuracy": 0.19444444444444445, "macro_operation_accuracy": 0.19444444444444445, "field_accuracy": 0, "normalized_numerical_error": 68943659.11612132, "status_accuracy": 0.75, "format_validity": 0.875, "pass_at_1": 0.16666666666666666, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0.125, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 40, "terminal_or_invalid": 7, "needs_clarification": 6, "invalid_input": 1 }, "out_of_domain": { "invalid_input": 2, "out_of_domain": 7 }, "needs_clarification": { "terminal_or_invalid": 2, "needs_clarification": 7 } }, "cost_per_correct_task_usd": 0 }, "aerodynamics": { "n": 96, "task_accuracy": 0.22916666666666666, "macro_operation_accuracy": 0.22916666666666669, "field_accuracy": 0.006944444444444444, "normalized_numerical_error": 65802.54972185315, "status_accuracy": 0.8020833333333334, "format_validity": 0.9791666666666666, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.1875, "terminal_failure_rate": 0.020833333333333332, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 55, "needs_clarification": 16, "invalid_input": 1 }, "needs_clarification": { "needs_clarification": 10, "terminal_or_invalid": 2 }, "out_of_domain": { "out_of_domain": 12 } }, "cost_per_correct_task_usd": 0 }, "propeller-systems": { "n": 72, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0, "normalized_numerical_error": 5757.775008651027, "status_accuracy": 0.375, "format_validity": 0.9166666666666666, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0.08333333333333333, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 32, "terminal_or_invalid": 3, "invalid_input": 5, "success": 12, "out_of_domain": 2 }, "out_of_domain": { "out_of_domain": 9 }, "needs_clarification": { "needs_clarification": 6, "terminal_or_invalid": 3 } }, "cost_per_correct_task_usd": 0 }, "rocket-analysis": { "n": 72, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0, "normalized_numerical_error": 8717415.611125799, "status_accuracy": 0.25, "format_validity": 0.9722222222222222, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0.027777777777777776, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "needs_clarification": { "needs_clarification": 8, "invalid_input": 1 }, "success": { "needs_clarification": 46, "success": 3, "invalid_input": 3, "terminal_or_invalid": 2 }, "out_of_domain": { "out_of_domain": 7, "invalid_input": 2 } }, "cost_per_correct_task_usd": 0 }, "orbital-mechanics": { "n": 168, "task_accuracy": 0.22023809523809523, "macro_operation_accuracy": 0.2202380952380953, "field_accuracy": 0.007936507936507936, "normalized_numerical_error": 13104210.273739578, "status_accuracy": 0.49404761904761907, "format_validity": 0.8988095238095238, "pass_at_1": 0.23214285714285715, "pass_at_3": 0.25, "pass_power_3": 0.19642857142857142, "terminal_failure_rate": 0.10119047619047619, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 46, "needs_clarification": 58, "terminal_or_invalid": 16, "invalid_input": 4, "out_of_domain": 2 }, "needs_clarification": { "needs_clarification": 21 }, "out_of_domain": { "out_of_domain": 16, "invalid_input": 4, "terminal_or_invalid": 1 } }, "cost_per_correct_task_usd": 0 }, "guidance-navigation-control": { "n": 48, "task_accuracy": 0.3333333333333333, "macro_operation_accuracy": 0.33333333333333337, "field_accuracy": 0.1388888888888889, "normalized_numerical_error": 2.2676127370386294, "status_accuracy": 0.9791666666666666, "format_validity": 1, "pass_at_1": 0.3125, "pass_at_3": 0.5, "pass_power_3": 0.1875, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "out_of_domain": { "out_of_domain": 5, "invalid_input": 1 }, "needs_clarification": { "needs_clarification": 6 }, "success": { "success": 36 } }, "cost_per_correct_task_usd": 0 }, "aircraft-performance": { "n": 168, "task_accuracy": 0.2857142857142857, "macro_operation_accuracy": 0.2857142857142857, "field_accuracy": 0.1865079365079365, "normalized_numerical_error": 8847.585703027515, "status_accuracy": 0.6428571428571429, "format_validity": 0.9404761904761905, "pass_at_1": 0.2857142857142857, "pass_at_3": 0.39285714285714285, "pass_power_3": 0.19642857142857142, "terminal_failure_rate": 0.05952380952380952, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 74, "terminal_or_invalid": 9, "needs_clarification": 41, "invalid_input": 2 }, "needs_clarification": { "needs_clarification": 20, "terminal_or_invalid": 1 }, "out_of_domain": { "invalid_input": 7, "out_of_domain": 14 } }, "cost_per_correct_task_usd": 0 }, "trajectory-optimization": { "n": 144, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0.023148148148148147, "normalized_numerical_error": 36103945.89720388, "status_accuracy": 0.4236111111111111, "format_validity": 0.9583333333333334, "pass_at_1": 0.20833333333333334, "pass_at_3": 0.22916666666666666, "pass_power_3": 0.16666666666666666, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "needs_clarification": { "needs_clarification": 18 }, "success": { "needs_clarification": 54, "success": 31, "invalid_input": 9, "terminal_or_invalid": 5, "out_of_domain": 9 }, "out_of_domain": { "invalid_input": 5, "out_of_domain": 12, "terminal_or_invalid": 1 } }, "cost_per_correct_task_usd": 0 } }, "nemotron-docs-only": { "flight-planning": { "n": 192, "task_accuracy": 0.22395833333333334, "macro_operation_accuracy": 0.22395833333333331, "field_accuracy": 0.006944444444444444, "normalized_numerical_error": 28914676.560869563, "status_accuracy": 0.6510416666666666, "format_validity": 0.90625, "pass_at_1": 0.21875, "pass_at_3": 0.265625, "pass_power_3": 0.171875, "terminal_failure_rate": 0.09375, "mean_runner_calls": 0, "mean_skill_reads": 2.0260416666666665, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 32, "terminal_or_invalid": 18, "success": 83, "out_of_domain": 6, "invalid_input": 5 }, "out_of_domain": { "out_of_domain": 19, "invalid_input": 5 }, "needs_clarification": { "needs_clarification": 23, "invalid_input": 1 } }, "cost_per_correct_task_usd": 0 }, "atmosphere": { "n": 48, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.9166666666666666, "format_validity": 1, "pass_at_1": 0.1875, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 1.3541666666666667, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "out_of_domain": { "invalid_input": 2, "out_of_domain": 4 }, "needs_clarification": { "needs_clarification": 6 }, "success": { "needs_clarification": 1, "success": 34, "out_of_domain": 1 } }, "cost_per_correct_task_usd": 0 }, "coordinate-frames": { "n": 72, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0, "normalized_numerical_error": 1167763.2056122825, "status_accuracy": 0.9305555555555556, "format_validity": 0.9722222222222222, "pass_at_1": 0.20833333333333334, "pass_at_3": 0.25, "pass_power_3": 0.16666666666666666, "terminal_failure_rate": 0.027777777777777776, "mean_runner_calls": 0, "mean_skill_reads": 1.1111111111111112, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 52, "terminal_or_invalid": 2 }, "out_of_domain": { "out_of_domain": 6, "invalid_input": 3 }, "needs_clarification": { "needs_clarification": 9 } }, "cost_per_correct_task_usd": 0 }, "aerodynamics": { "n": 96, "task_accuracy": 0.19791666666666666, "macro_operation_accuracy": 0.19791666666666669, "field_accuracy": 0, "normalized_numerical_error": 6499.123062535408, "status_accuracy": 0.8958333333333334, "format_validity": 0.96875, "pass_at_1": 0.1875, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0.03125, "mean_runner_calls": 0, "mean_skill_reads": 0.8333333333333334, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 67, "terminal_or_invalid": 2, "invalid_input": 1, "needs_clarification": 2 }, "needs_clarification": { "needs_clarification": 12 }, "out_of_domain": { "out_of_domain": 7, "invalid_input": 4, "terminal_or_invalid": 1 } }, "cost_per_correct_task_usd": 0 }, "propeller-systems": { "n": 72, "task_accuracy": 0.1527777777777778, "macro_operation_accuracy": 0.15277777777777776, "field_accuracy": 0.018518518518518517, "normalized_numerical_error": 12485.811166569905, "status_accuracy": 0.5138888888888888, "format_validity": 0.9861111111111112, "pass_at_1": 0.16666666666666666, "pass_at_3": 0.20833333333333334, "pass_power_3": 0.08333333333333333, "terminal_failure_rate": 0.013888888888888888, "mean_runner_calls": 0, "mean_skill_reads": 1.7777777777777777, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 25, "success": 26, "invalid_input": 2, "out_of_domain": 1 }, "out_of_domain": { "out_of_domain": 3, "invalid_input": 6 }, "needs_clarification": { "needs_clarification": 8, "terminal_or_invalid": 1 } }, "cost_per_correct_task_usd": 0 }, "rocket-analysis": { "n": 72, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0.037037037037037035, "normalized_numerical_error": 15000050.955355106, "status_accuracy": 0.3888888888888889, "format_validity": 0.9722222222222222, "pass_at_1": 0.16666666666666666, "pass_at_3": 0.2916666666666667, "pass_power_3": 0.16666666666666666, "terminal_failure_rate": 0.027777777777777776, "mean_runner_calls": 0, "mean_skill_reads": 1.5972222222222223, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "needs_clarification": { "needs_clarification": 9 }, "success": { "needs_clarification": 39, "success": 12, "out_of_domain": 1, "terminal_or_invalid": 2 }, "out_of_domain": { "out_of_domain": 7, "invalid_input": 2 } }, "cost_per_correct_task_usd": 0 }, "orbital-mechanics": { "n": 168, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.2083333333333333, "field_accuracy": 0.003968253968253968, "normalized_numerical_error": 72168.10664128046, "status_accuracy": 0.6488095238095238, "format_validity": 0.9642857142857143, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.16071428571428573, "terminal_failure_rate": 0.03571428571428571, "mean_runner_calls": 0, "mean_skill_reads": 1.0892857142857142, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 74, "terminal_or_invalid": 5, "needs_clarification": 42, "invalid_input": 3, "out_of_domain": 2 }, "needs_clarification": { "needs_clarification": 21 }, "out_of_domain": { "out_of_domain": 14, "terminal_or_invalid": 1, "invalid_input": 5, "needs_clarification": 1 } }, "cost_per_correct_task_usd": 0 }, "guidance-navigation-control": { "n": 48, "task_accuracy": 0.3333333333333333, "macro_operation_accuracy": 0.3333333333333333, "field_accuracy": 0.1388888888888889, "normalized_numerical_error": 10.830182268543574, "status_accuracy": 0.9166666666666666, "format_validity": 0.9375, "pass_at_1": 0.375, "pass_at_3": 0.5, "pass_power_3": 0.1875, "terminal_failure_rate": 0.0625, "mean_runner_calls": 0, "mean_skill_reads": 0.375, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "out_of_domain": { "out_of_domain": 5, "invalid_input": 1 }, "needs_clarification": { "needs_clarification": 6 }, "success": { "success": 33, "terminal_or_invalid": 3 } }, "cost_per_correct_task_usd": 0 }, "aircraft-performance": { "n": 168, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0.16666666666666666, "normalized_numerical_error": 4659.186015580772, "status_accuracy": 0.8571428571428571, "format_validity": 0.9404761904761905, "pass_at_1": 0.21428571428571427, "pass_at_3": 0.3392857142857143, "pass_power_3": 0.14285714285714285, "terminal_failure_rate": 0.05952380952380952, "mean_runner_calls": 0, "mean_skill_reads": 0.8571428571428571, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 112, "needs_clarification": 6, "terminal_or_invalid": 8 }, "needs_clarification": { "needs_clarification": 20, "terminal_or_invalid": 1 }, "out_of_domain": { "invalid_input": 8, "out_of_domain": 12, "terminal_or_invalid": 1 } }, "cost_per_correct_task_usd": 0 }, "trajectory-optimization": { "n": 144, "task_accuracy": 0.2222222222222222, "macro_operation_accuracy": 0.22222222222222224, "field_accuracy": 0.018518518518518517, "normalized_numerical_error": 23290996.143670637, "status_accuracy": 0.4652777777777778, "format_validity": 0.9583333333333334, "pass_at_1": 0.22916666666666666, "pass_at_3": 0.25, "pass_power_3": 0.16666666666666666, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0, "mean_skill_reads": 1.125, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "needs_clarification": { "needs_clarification": 17, "terminal_or_invalid": 1 }, "success": { "needs_clarification": 58, "success": 35, "invalid_input": 5, "out_of_domain": 5, "terminal_or_invalid": 5 }, "out_of_domain": { "out_of_domain": 15, "invalid_input": 3 } }, "cost_per_correct_task_usd": 0 } }, "nemotron-runner-only": { "flight-planning": { "n": 192, "task_accuracy": 0.5052083333333334, "macro_operation_accuracy": 0.5052083333333333, "field_accuracy": 0.3888888888888889, "normalized_numerical_error": 7025641.294871795, "status_accuracy": 0.8385416666666666, "format_validity": 0.890625, "pass_at_1": 0.46875, "pass_at_3": 0.671875, "pass_power_3": 0.328125, "terminal_failure_rate": 0.109375, "mean_runner_calls": 0.9322916666666666, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0.9777777777777777, "classification_matrix": { "success": { "success": 120, "terminal_or_invalid": 18, "needs_clarification": 4, "out_of_domain": 2 }, "out_of_domain": { "out_of_domain": 19, "invalid_input": 4, "terminal_or_invalid": 1 }, "needs_clarification": { "needs_clarification": 22, "terminal_or_invalid": 2 } }, "cost_per_correct_task_usd": 0 }, "atmosphere": { "n": 48, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "out_of_domain": { "out_of_domain": 6 }, "needs_clarification": { "needs_clarification": 6 }, "success": { "success": 36 } }, "cost_per_correct_task_usd": 0 }, "coordinate-frames": { "n": 72, "task_accuracy": 0.5277777777777778, "macro_operation_accuracy": 0.5277777777777778, "field_accuracy": 0.42592592592592593, "normalized_numerical_error": 0.00019476422533878812, "status_accuracy": 0.9444444444444444, "format_validity": 0.9722222222222222, "pass_at_1": 0.5416666666666666, "pass_at_3": 0.6666666666666666, "pass_power_3": 0.375, "terminal_failure_rate": 0.027777777777777776, "mean_runner_calls": 0.75, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0.9814814814814815, "classification_matrix": { "success": { "success": 53, "terminal_or_invalid": 1 }, "out_of_domain": { "invalid_input": 1, "out_of_domain": 8 }, "needs_clarification": { "needs_clarification": 7, "terminal_or_invalid": 1, "invalid_input": 1 } }, "cost_per_correct_task_usd": 0 }, "aerodynamics": { "n": 96, "task_accuracy": 0.6354166666666666, "macro_operation_accuracy": 0.6354166666666667, "field_accuracy": 0.5694444444444444, "normalized_numerical_error": 0.24340303059615281, "status_accuracy": 0.9583333333333334, "format_validity": 0.9791666666666666, "pass_at_1": 0.6875, "pass_at_3": 0.71875, "pass_power_3": 0.46875, "terminal_failure_rate": 0.020833333333333332, "mean_runner_calls": 0.78125, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0.9814814814814815, "classification_matrix": { "success": { "success": 71, "terminal_or_invalid": 1 }, "needs_clarification": { "needs_clarification": 11, "terminal_or_invalid": 1 }, "out_of_domain": { "out_of_domain": 10, "invalid_input": 2 } }, "cost_per_correct_task_usd": 0 }, "propeller-systems": { "n": 72, "task_accuracy": 0.6944444444444444, "macro_operation_accuracy": 0.6944444444444445, "field_accuracy": 0.6296296296296297, "normalized_numerical_error": 52.90889021571888, "status_accuracy": 0.9583333333333334, "format_validity": 0.9583333333333334, "pass_at_1": 0.625, "pass_at_3": 0.75, "pass_power_3": 0.5833333333333334, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0.8055555555555556, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 53, "terminal_or_invalid": 1 }, "out_of_domain": { "out_of_domain": 9 }, "needs_clarification": { "needs_clarification": 7, "terminal_or_invalid": 2 } }, "cost_per_correct_task_usd": 0 }, "rocket-analysis": { "n": 72, "task_accuracy": 0.6805555555555556, "macro_operation_accuracy": 0.6805555555555555, "field_accuracy": 0.7685185185185185, "normalized_numerical_error": 0.10404590019101218, "status_accuracy": 0.9583333333333334, "format_validity": 0.9722222222222222, "pass_at_1": 0.625, "pass_at_3": 0.75, "pass_power_3": 0.5416666666666666, "terminal_failure_rate": 0.027777777777777776, "mean_runner_calls": 0.7361111111111112, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0.9814814814814815, "classification_matrix": { "needs_clarification": { "terminal_or_invalid": 1, "needs_clarification": 8 }, "success": { "success": 53, "terminal_or_invalid": 1 }, "out_of_domain": { "out_of_domain": 8, "invalid_input": 1 } }, "cost_per_correct_task_usd": 0 }, "orbital-mechanics": { "n": 168, "task_accuracy": 0.6607142857142857, "macro_operation_accuracy": 0.6607142857142857, "field_accuracy": 0.5595238095238095, "normalized_numerical_error": 214.6084533666933, "status_accuracy": 0.9702380952380952, "format_validity": 0.9702380952380952, "pass_at_1": 0.6428571428571429, "pass_at_3": 0.875, "pass_power_3": 0.42857142857142855, "terminal_failure_rate": 0.02976190476190476, "mean_runner_calls": 0.8095238095238095, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0.9841269841269841, "classification_matrix": { "success": { "success": 122, "terminal_or_invalid": 4 }, "needs_clarification": { "needs_clarification": 20, "terminal_or_invalid": 1 }, "out_of_domain": { "out_of_domain": 21 } }, "cost_per_correct_task_usd": 0 }, "guidance-navigation-control": { "n": 48, "task_accuracy": 0.2708333333333333, "macro_operation_accuracy": 0.27083333333333337, "field_accuracy": 0.08333333333333333, "normalized_numerical_error": 0.42784286119436293, "status_accuracy": 0.9583333333333334, "format_validity": 0.9791666666666666, "pass_at_1": 0.25, "pass_at_3": 0.4375, "pass_power_3": 0.1875, "terminal_failure_rate": 0.020833333333333332, "mean_runner_calls": 0.7708333333333334, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "out_of_domain": { "out_of_domain": 4, "invalid_input": 1, "terminal_or_invalid": 1 }, "needs_clarification": { "needs_clarification": 6 }, "success": { "success": 36 } }, "cost_per_correct_task_usd": 0 }, "aircraft-performance": { "n": 168, "task_accuracy": 0.7023809523809523, "macro_operation_accuracy": 0.7023809523809523, "field_accuracy": 0.6785714285714286, "normalized_numerical_error": 0.8238636363639873, "status_accuracy": 0.9464285714285714, "format_validity": 0.9702380952380952, "pass_at_1": 0.6964285714285714, "pass_at_3": 0.9464285714285714, "pass_power_3": 0.42857142857142855, "terminal_failure_rate": 0.02976190476190476, "mean_runner_calls": 0.8452380952380952, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0.9841269841269841, "classification_matrix": { "success": { "success": 124, "terminal_or_invalid": 2 }, "needs_clarification": { "terminal_or_invalid": 3, "needs_clarification": 18 }, "out_of_domain": { "out_of_domain": 17, "invalid_input": 4 } }, "cost_per_correct_task_usd": 0 }, "trajectory-optimization": { "n": 144, "task_accuracy": 0.5763888888888888, "macro_operation_accuracy": 0.576388888888889, "field_accuracy": 0.5555555555555556, "normalized_numerical_error": 0.047700424219663334, "status_accuracy": 0.9166666666666666, "format_validity": 0.9652777777777778, "pass_at_1": 0.625, "pass_at_3": 0.7916666666666666, "pass_power_3": 0.3333333333333333, "terminal_failure_rate": 0.034722222222222224, "mean_runner_calls": 0.9722222222222222, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0.9907407407407407, "classification_matrix": { "needs_clarification": { "needs_clarification": 17, "terminal_or_invalid": 1 }, "success": { "success": 102, "out_of_domain": 1, "needs_clarification": 2, "terminal_or_invalid": 3 }, "out_of_domain": { "invalid_input": 4, "out_of_domain": 13, "terminal_or_invalid": 1 } }, "cost_per_correct_task_usd": 0 } }, "nemotron-full-skills": { "flight-planning": { "n": 192, "task_accuracy": 0.3958333333333333, "macro_operation_accuracy": 0.3958333333333333, "field_accuracy": 0.2951388888888889, "normalized_numerical_error": 15629939.327690786, "status_accuracy": 0.8177083333333334, "format_validity": 0.90625, "pass_at_1": 0.359375, "pass_at_3": 0.5625, "pass_power_3": 0.1875, "terminal_failure_rate": 0.09375, "mean_runner_calls": 1.0208333333333333, "mean_skill_reads": 1.4479166666666667, "runner_invocation_rate_on_required_tasks": 0.9666666666666667, "classification_matrix": { "success": { "success": 123, "terminal_or_invalid": 16, "needs_clarification": 3, "out_of_domain": 2 }, "out_of_domain": { "invalid_input": 12, "out_of_domain": 12 }, "needs_clarification": { "needs_clarification": 22, "terminal_or_invalid": 2 } }, "cost_per_correct_task_usd": 0 }, "atmosphere": { "n": 48, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.9583333333333334, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 1.1458333333333333, "mean_skill_reads": 1.8333333333333333, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "out_of_domain": { "out_of_domain": 4, "invalid_input": 2 }, "needs_clarification": { "needs_clarification": 6 }, "success": { "success": 36 } }, "cost_per_correct_task_usd": 0 }, "coordinate-frames": { "n": 72, "task_accuracy": 0.5833333333333334, "macro_operation_accuracy": 0.5833333333333334, "field_accuracy": 0.48148148148148145, "normalized_numerical_error": 0.0005353986052796245, "status_accuracy": 0.9722222222222222, "format_validity": 1, "pass_at_1": 0.5833333333333334, "pass_at_3": 0.75, "pass_power_3": 0.375, "terminal_failure_rate": 0, "mean_runner_calls": 0.7916666666666666, "mean_skill_reads": 0.9166666666666666, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 54 }, "out_of_domain": { "out_of_domain": 7, "invalid_input": 2 }, "needs_clarification": { "needs_clarification": 9 } }, "cost_per_correct_task_usd": 0 }, "aerodynamics": { "n": 96, "task_accuracy": 0.4895833333333333, "macro_operation_accuracy": 0.4895833333333333, "field_accuracy": 0.4375, "normalized_numerical_error": 0.25287866020347327, "status_accuracy": 0.90625, "format_validity": 0.96875, "pass_at_1": 0.46875, "pass_at_3": 0.65625, "pass_power_3": 0.3125, "terminal_failure_rate": 0.03125, "mean_runner_calls": 0.8541666666666666, "mean_skill_reads": 0.96875, "runner_invocation_rate_on_required_tasks": 0.9814814814814815, "classification_matrix": { "success": { "success": 70, "terminal_or_invalid": 2 }, "needs_clarification": { "needs_clarification": 12 }, "out_of_domain": { "invalid_input": 6, "out_of_domain": 5, "terminal_or_invalid": 1 } }, "cost_per_correct_task_usd": 0 }, "propeller-systems": { "n": 72, "task_accuracy": 0.5694444444444444, "macro_operation_accuracy": 0.5694444444444445, "field_accuracy": 0.5, "normalized_numerical_error": 61.42378549624259, "status_accuracy": 0.9444444444444444, "format_validity": 1, "pass_at_1": 0.5833333333333334, "pass_at_3": 0.7083333333333334, "pass_power_3": 0.4166666666666667, "terminal_failure_rate": 0, "mean_runner_calls": 0.9583333333333334, "mean_skill_reads": 1.5, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 54 }, "out_of_domain": { "invalid_input": 4, "out_of_domain": 5 }, "needs_clarification": { "needs_clarification": 9 } }, "cost_per_correct_task_usd": 0 }, "rocket-analysis": { "n": 72, "task_accuracy": 0.625, "macro_operation_accuracy": 0.625, "field_accuracy": 0.6944444444444444, "normalized_numerical_error": 0.12887859749762887, "status_accuracy": 0.9166666666666666, "format_validity": 0.9444444444444444, "pass_at_1": 0.5416666666666666, "pass_at_3": 0.7916666666666666, "pass_power_3": 0.4583333333333333, "terminal_failure_rate": 0.05555555555555555, "mean_runner_calls": 0.9305555555555556, "mean_skill_reads": 1.3888888888888888, "runner_invocation_rate_on_required_tasks": 0.9629629629629629, "classification_matrix": { "needs_clarification": { "needs_clarification": 8, "terminal_or_invalid": 1 }, "success": { "success": 51, "terminal_or_invalid": 3 }, "out_of_domain": { "out_of_domain": 7, "invalid_input": 2 } }, "cost_per_correct_task_usd": 0 }, "orbital-mechanics": { "n": 168, "task_accuracy": 0.5357142857142857, "macro_operation_accuracy": 0.5357142857142857, "field_accuracy": 0.4523809523809524, "normalized_numerical_error": 6.113070370386777, "status_accuracy": 0.9226190476190477, "format_validity": 0.9761904761904762, "pass_at_1": 0.5178571428571429, "pass_at_3": 0.7857142857142857, "pass_power_3": 0.2857142857142857, "terminal_failure_rate": 0.023809523809523808, "mean_runner_calls": 0.8988095238095238, "mean_skill_reads": 1.2916666666666667, "runner_invocation_rate_on_required_tasks": 0.9920634920634921, "classification_matrix": { "success": { "success": 120, "terminal_or_invalid": 3, "needs_clarification": 3 }, "needs_clarification": { "needs_clarification": 20, "terminal_or_invalid": 1 }, "out_of_domain": { "invalid_input": 6, "out_of_domain": 15 } }, "cost_per_correct_task_usd": 0 }, "guidance-navigation-control": { "n": 48, "task_accuracy": 0.375, "macro_operation_accuracy": 0.375, "field_accuracy": 0.19444444444444445, "normalized_numerical_error": 0.0031763371954795136, "status_accuracy": 0.9791666666666666, "format_validity": 1, "pass_at_1": 0.375, "pass_at_3": 0.5625, "pass_power_3": 0.1875, "terminal_failure_rate": 0, "mean_runner_calls": 1, "mean_skill_reads": 0.9375, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "out_of_domain": { "invalid_input": 1, "out_of_domain": 5 }, "needs_clarification": { "needs_clarification": 6 }, "success": { "success": 36 } }, "cost_per_correct_task_usd": 0 }, "aircraft-performance": { "n": 168, "task_accuracy": 0.6845238095238095, "macro_operation_accuracy": 0.6845238095238095, "field_accuracy": 0.7023809523809523, "normalized_numerical_error": 4.795698924730798, "status_accuracy": 0.9166666666666666, "format_validity": 0.9761904761904762, "pass_at_1": 0.8214285714285714, "pass_at_3": 0.9464285714285714, "pass_power_3": 0.39285714285714285, "terminal_failure_rate": 0.023809523809523808, "mean_runner_calls": 0.9285714285714286, "mean_skill_reads": 1.0773809523809523, "runner_invocation_rate_on_required_tasks": 0.9920634920634921, "classification_matrix": { "success": { "success": 124, "terminal_or_invalid": 2 }, "needs_clarification": { "needs_clarification": 21 }, "out_of_domain": { "out_of_domain": 9, "invalid_input": 10, "terminal_or_invalid": 2 } }, "cost_per_correct_task_usd": 0 }, "trajectory-optimization": { "n": 144, "task_accuracy": 0.5347222222222222, "macro_operation_accuracy": 0.5347222222222222, "field_accuracy": 0.46296296296296297, "normalized_numerical_error": 4692159.370816482, "status_accuracy": 0.8263888888888888, "format_validity": 0.9513888888888888, "pass_at_1": 0.5833333333333334, "pass_at_3": 0.7083333333333334, "pass_power_3": 0.2916666666666667, "terminal_failure_rate": 0.04861111111111111, "mean_runner_calls": 1.1180555555555556, "mean_skill_reads": 1.3680555555555556, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "needs_clarification": { "needs_clarification": 17, "terminal_or_invalid": 1 }, "success": { "success": 89, "terminal_or_invalid": 6, "needs_clarification": 9, "out_of_domain": 4 }, "out_of_domain": { "out_of_domain": 13, "invalid_input": 5 } }, "cost_per_correct_task_usd": 0 } }, "terra-base": { "flight-planning": { "n": 192, "task_accuracy": 0.34375, "macro_operation_accuracy": 0.34375, "field_accuracy": 0.14583333333333334, "normalized_numerical_error": 309.8076086956513, "status_accuracy": 0.46875, "format_validity": 1, "pass_at_1": 0.34375, "pass_at_3": 0.34375, "pass_power_3": 0.34375, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 102, "success": 42 }, "out_of_domain": { "out_of_domain": 24 }, "needs_clarification": { "needs_clarification": 24 } }, "cost_per_correct_task_usd": 0.020190227272727273 }, "atmosphere": { "n": 48, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.625, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "out_of_domain": { "out_of_domain": 6 }, "needs_clarification": { "needs_clarification": 6 }, "success": { "needs_clarification": 18, "success": 18 } }, "cost_per_correct_task_usd": 0.020189375 }, "coordinate-frames": { "n": 72, "task_accuracy": 0.4305555555555556, "macro_operation_accuracy": 0.4305555555555556, "field_accuracy": 0.24074074074074073, "normalized_numerical_error": 62165.35130546797, "status_accuracy": 0.75, "format_validity": 1, "pass_at_1": 0.4166666666666667, "pass_at_3": 0.5416666666666666, "pass_power_3": 0.3333333333333333, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 36, "needs_clarification": 18 }, "out_of_domain": { "out_of_domain": 9 }, "needs_clarification": { "needs_clarification": 9 } }, "cost_per_correct_task_usd": 0.05431185483870967 }, "aerodynamics": { "n": 96, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0.027777777777777776, "normalized_numerical_error": 797.1543482381709, "status_accuracy": 0.6041666666666666, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 38, "success": 34 }, "needs_clarification": { "needs_clarification": 12 }, "out_of_domain": { "out_of_domain": 12 } }, "cost_per_correct_task_usd": 0.026991249999999994 }, "propeller-systems": { "n": 72, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.25, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 54 }, "out_of_domain": { "out_of_domain": 9 }, "needs_clarification": { "needs_clarification": 9 } }, "cost_per_correct_task_usd": 0.014240416666666669 }, "rocket-analysis": { "n": 72, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": 12499.375, "status_accuracy": 0.25, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "needs_clarification": { "needs_clarification": 9 }, "success": { "needs_clarification": 54 }, "out_of_domain": { "out_of_domain": 9 } }, "cost_per_correct_task_usd": 0.017338750000000003 }, "orbital-mechanics": { "n": 168, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0.023809523809523808, "normalized_numerical_error": 0.00999999999995449, "status_accuracy": 0.34523809523809523, "format_validity": 0.9880952380952381, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0.011904761904761904, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 106, "success": 16, "terminal_or_invalid": 2, "invalid_input": 1, "out_of_domain": 1 }, "needs_clarification": { "needs_clarification": 21 }, "out_of_domain": { "out_of_domain": 21 } }, "cost_per_correct_task_usd": 0.04926517857142859 }, "guidance-navigation-control": { "n": 48, "task_accuracy": 0.5208333333333334, "macro_operation_accuracy": 0.5208333333333333, "field_accuracy": 0.3611111111111111, "normalized_numerical_error": 0.0003444981048770299, "status_accuracy": 0.7083333333333334, "format_validity": 1, "pass_at_1": 0.5, "pass_at_3": 0.625, "pass_power_3": 0.3125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "out_of_domain": { "out_of_domain": 6 }, "needs_clarification": { "needs_clarification": 6 }, "success": { "success": 22, "needs_clarification": 14 } }, "cost_per_correct_task_usd": 0.014528699999999997 }, "aircraft-performance": { "n": 168, "task_accuracy": 0.375, "macro_operation_accuracy": 0.37500000000000006, "field_accuracy": 0.23015873015873015, "normalized_numerical_error": 62.03339402741375, "status_accuracy": 0.5357142857142857, "format_validity": 1, "pass_at_1": 0.39285714285714285, "pass_at_3": 0.39285714285714285, "pass_power_3": 0.3392857142857143, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 71, "success": 50, "invalid_input": 5 }, "needs_clarification": { "needs_clarification": 21 }, "out_of_domain": { "out_of_domain": 19, "invalid_input": 2 } }, "cost_per_correct_task_usd": 0.015136071428571436 }, "trajectory-optimization": { "n": 144, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0.05555555555555555, "normalized_numerical_error": 0, "status_accuracy": 0.25, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "needs_clarification": { "needs_clarification": 18 }, "success": { "needs_clarification": 107, "invalid_input": 1 }, "out_of_domain": { "out_of_domain": 18 } }, "cost_per_correct_task_usd": 0.017337916666666665 } }, "terra-full-skills": { "flight-planning": { "n": 192, "task_accuracy": 0.953125, "macro_operation_accuracy": 0.953125, "field_accuracy": 0.9375, "normalized_numerical_error": 0.0625, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.953125, "pass_at_3": 0.953125, "pass_power_3": 0.953125, "terminal_failure_rate": 0, "mean_runner_calls": 0.7604166666666666, "mean_skill_reads": 1.6927083333333333, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 144 }, "out_of_domain": { "out_of_domain": 24 }, "needs_clarification": { "needs_clarification": 24 } }, "cost_per_correct_task_usd": 0.00697325 }, "atmosphere": { "n": 48, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.4583333333333333, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "out_of_domain": { "out_of_domain": 6 }, "needs_clarification": { "needs_clarification": 6 }, "success": { "success": 36 } }, "cost_per_correct_task_usd": 0.024725645833333334 }, "coordinate-frames": { "n": 72, "task_accuracy": 0.7222222222222222, "macro_operation_accuracy": 0.7222222222222222, "field_accuracy": 0.6296296296296297, "normalized_numerical_error": 1.4680314480381853e-7, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.75, "pass_at_3": 0.75, "pass_power_3": 0.6666666666666666, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.375, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 54 }, "out_of_domain": { "out_of_domain": 9 }, "needs_clarification": { "needs_clarification": 9 } }, "cost_per_correct_task_usd": 0.008799334134615385 }, "aerodynamics": { "n": 96, "task_accuracy": 0.53125, "macro_operation_accuracy": 0.53125, "field_accuracy": 0.375, "normalized_numerical_error": 0.1571903452572357, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.53125, "pass_at_3": 0.5625, "pass_power_3": 0.5, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.6875, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 72 }, "needs_clarification": { "needs_clarification": 12 }, "out_of_domain": { "out_of_domain": 12 } }, "cost_per_correct_task_usd": 0.01537581127450981 }, "propeller-systems": { "n": 72, "task_accuracy": 0.7222222222222222, "macro_operation_accuracy": 0.7222222222222222, "field_accuracy": 0.6296296296296297, "normalized_numerical_error": 5.257582965065042e-7, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.7083333333333334, "pass_at_3": 0.75, "pass_power_3": 0.7083333333333334, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.8472222222222223, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 54 }, "out_of_domain": { "out_of_domain": 9 }, "needs_clarification": { "needs_clarification": 9 } }, "cost_per_correct_task_usd": 0.012752564903846157 }, "rocket-analysis": { "n": 72, "task_accuracy": 0.75, "macro_operation_accuracy": 0.75, "field_accuracy": 0.8240740740740741, "normalized_numerical_error": 0.11557092833523842, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.75, "pass_at_3": 0.7916666666666666, "pass_power_3": 0.7083333333333334, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "needs_clarification": { "needs_clarification": 9 }, "success": { "success": 54 }, "out_of_domain": { "out_of_domain": 9 } }, "cost_per_correct_task_usd": 0.010923002314814816 }, "orbital-mechanics": { "n": 168, "task_accuracy": 0.9940476190476191, "macro_operation_accuracy": 0.9940476190476192, "field_accuracy": 0.996031746031746, "normalized_numerical_error": 0.00012152777770533006, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 0.9821428571428571, "terminal_failure_rate": 0, "mean_runner_calls": 0.7857142857142857, "mean_skill_reads": 1.4821428571428572, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 126 }, "needs_clarification": { "needs_clarification": 21 }, "out_of_domain": { "out_of_domain": 21 } }, "cost_per_correct_task_usd": 0.007601136976047906 }, "guidance-navigation-control": { "n": 48, "task_accuracy": 0.4583333333333333, "macro_operation_accuracy": 0.4583333333333333, "field_accuracy": 0.2777777777777778, "normalized_numerical_error": 0.0039243354500816, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.4375, "pass_at_3": 0.5, "pass_power_3": 0.4375, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "out_of_domain": { "out_of_domain": 6 }, "needs_clarification": { "needs_clarification": 6 }, "success": { "success": 36 } }, "cost_per_correct_task_usd": 0.01592822727272727 }, "aircraft-performance": { "n": 168, "task_accuracy": 0.9880952380952381, "macro_operation_accuracy": 0.988095238095238, "field_accuracy": 0.9841269841269841, "normalized_numerical_error": 0, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 0.9642857142857143, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.4821428571428572, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 126 }, "needs_clarification": { "needs_clarification": 21 }, "out_of_domain": { "out_of_domain": 21 } }, "cost_per_correct_task_usd": 0.007044554969879521 }, "trajectory-optimization": { "n": 144, "task_accuracy": 1, "macro_operation_accuracy": 1, "field_accuracy": 1, "normalized_numerical_error": 0.10341959715779805, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 1, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5972222222222223, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "needs_clarification": { "needs_clarification": 18 }, "success": { "success": 108 }, "out_of_domain": { "out_of_domain": 18 } }, "cost_per_correct_task_usd": 0.00870630555555555 } }, "sol-base": { "flight-planning": { "n": 192, "task_accuracy": 0.21354166666666666, "macro_operation_accuracy": 0.21354166666666666, "field_accuracy": 0.1076388888888889, "normalized_numerical_error": 4937651.806127361, "status_accuracy": 0.2864583333333333, "format_validity": 1, "pass_at_1": 0.203125, "pass_at_3": 0.234375, "pass_power_3": 0.1875, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 116, "success": 28 }, "out_of_domain": { "invalid_input": 21, "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 24 } }, "cost_per_correct_task_usd": 0.051518780487804866 }, "atmosphere": { "n": 48, "task_accuracy": 0.125, "macro_operation_accuracy": 0.125, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.5416666666666666, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.125, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "out_of_domain": { "invalid_input": 6 }, "needs_clarification": { "needs_clarification": 6 }, "success": { "needs_clarification": 16, "success": 20 } }, "cost_per_correct_task_usd": 0.09942249999999997 }, "coordinate-frames": { "n": 72, "task_accuracy": 0.1527777777777778, "macro_operation_accuracy": 0.1527777777777778, "field_accuracy": 0.018518518518518517, "normalized_numerical_error": 126.37050878707886, "status_accuracy": 0.6527777777777778, "format_validity": 1, "pass_at_1": 0.16666666666666666, "pass_at_3": 0.16666666666666666, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 36, "needs_clarification": 18 }, "out_of_domain": { "invalid_input": 7, "out_of_domain": 2 }, "needs_clarification": { "needs_clarification": 9 } }, "cost_per_correct_task_usd": 0.3610268181818181 }, "aerodynamics": { "n": 96, "task_accuracy": 0.125, "macro_operation_accuracy": 0.125, "field_accuracy": 0, "normalized_numerical_error": 230.2677727686655, "status_accuracy": 0.4895833333333333, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.125, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 37, "success": 35 }, "needs_clarification": { "needs_clarification": 12 }, "out_of_domain": { "invalid_input": 12 } }, "cost_per_correct_task_usd": 0.1013875 }, "propeller-systems": { "n": 72, "task_accuracy": 0.1388888888888889, "macro_operation_accuracy": 0.13888888888888887, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.1388888888888889, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.16666666666666666, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 54 }, "out_of_domain": { "invalid_input": 8, "out_of_domain": 1 }, "needs_clarification": { "needs_clarification": 9 } }, "cost_per_correct_task_usd": 0.06990149999999998 }, "rocket-analysis": { "n": 72, "task_accuracy": 0.1527777777777778, "macro_operation_accuracy": 0.15277777777777776, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.1527777777777778, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.20833333333333334, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "needs_clarification": { "needs_clarification": 9 }, "success": { "needs_clarification": 54 }, "out_of_domain": { "invalid_input": 7, "out_of_domain": 2 } }, "cost_per_correct_task_usd": 0.060500454545454545 }, "orbital-mechanics": { "n": 168, "task_accuracy": 0.22023809523809523, "macro_operation_accuracy": 0.2202380952380952, "field_accuracy": 0.07142857142857142, "normalized_numerical_error": 0.02500000000016378, "status_accuracy": 0.26785714285714285, "format_validity": 1, "pass_at_1": 0.21428571428571427, "pass_at_3": 0.25, "pass_power_3": 0.16071428571428573, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 110, "success": 16 }, "needs_clarification": { "needs_clarification": 21 }, "out_of_domain": { "out_of_domain": 8, "invalid_input": 13 } }, "cost_per_correct_task_usd": 0.11323256756756753 }, "guidance-navigation-control": { "n": 48, "task_accuracy": 0.2916666666666667, "macro_operation_accuracy": 0.2916666666666667, "field_accuracy": 0.19444444444444445, "normalized_numerical_error": 0.0003296239365428791, "status_accuracy": 0.5416666666666666, "format_validity": 1, "pass_at_1": 0.1875, "pass_at_3": 0.4375, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "out_of_domain": { "invalid_input": 5, "out_of_domain": 1 }, "needs_clarification": { "needs_clarification": 6 }, "success": { "success": 19, "needs_clarification": 17 } }, "cost_per_correct_task_usd": 0.06197250000000001 }, "aircraft-performance": { "n": 168, "task_accuracy": 0.2857142857142857, "macro_operation_accuracy": 0.2857142857142857, "field_accuracy": 0.24603174603174602, "normalized_numerical_error": 49.33309143000464, "status_accuracy": 0.44047619047619047, "format_validity": 1, "pass_at_1": 0.30357142857142855, "pass_at_3": 0.30357142857142855, "pass_power_3": 0.26785714285714285, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 68, "success": 52, "invalid_input": 6 }, "needs_clarification": { "needs_clarification": 21 }, "out_of_domain": { "invalid_input": 20, "out_of_domain": 1 } }, "cost_per_correct_task_usd": 0.049316562499999994 }, "trajectory-optimization": { "n": 144, "task_accuracy": 0.1527777777777778, "macro_operation_accuracy": 0.15277777777777776, "field_accuracy": 0.05092592592592592, "normalized_numerical_error": 0, "status_accuracy": 0.1527777777777778, "format_validity": 1, "pass_at_1": 0.14583333333333334, "pass_at_3": 0.16666666666666666, "pass_power_3": 0.14583333333333334, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "needs_clarification": { "needs_clarification": 18 }, "success": { "needs_clarification": 108 }, "out_of_domain": { "invalid_input": 14, "out_of_domain": 4 } }, "cost_per_correct_task_usd": 0.08373681818181816 } }, "sol-full-skills": { "flight-planning": { "n": 192, "task_accuracy": 0.6354166666666666, "macro_operation_accuracy": 0.6354166666666666, "field_accuracy": 0.6458333333333334, "normalized_numerical_error": 0.18421052631578946, "status_accuracy": 0.9010416666666666, "format_validity": 1, "pass_at_1": 0.6875, "pass_at_3": 0.765625, "pass_power_3": 0.484375, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.796875, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 144 }, "out_of_domain": { "invalid_input": 19, "out_of_domain": 5 }, "needs_clarification": { "needs_clarification": 24 } }, "cost_per_correct_task_usd": 0.02594746311475409 }, "atmosphere": { "n": 48, "task_accuracy": 0.16666666666666666, "macro_operation_accuracy": 0.16666666666666669, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.9166666666666666, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.1875, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.7916666666666667, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "out_of_domain": { "invalid_input": 4, "out_of_domain": 2 }, "needs_clarification": { "needs_clarification": 6 }, "success": { "success": 36 } }, "cost_per_correct_task_usd": 0.09597396875000003 }, "coordinate-frames": { "n": 72, "task_accuracy": 0.4583333333333333, "macro_operation_accuracy": 0.4583333333333333, "field_accuracy": 0.35185185185185186, "normalized_numerical_error": 2.7729482907387944e-7, "status_accuracy": 0.9444444444444444, "format_validity": 1, "pass_at_1": 0.4583333333333333, "pass_at_3": 0.5416666666666666, "pass_power_3": 0.4166666666666667, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5694444444444444, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 54 }, "out_of_domain": { "invalid_input": 4, "out_of_domain": 5 }, "needs_clarification": { "needs_clarification": 9 } }, "cost_per_correct_task_usd": 0.03511707575757576 }, "aerodynamics": { "n": 96, "task_accuracy": 0.40625, "macro_operation_accuracy": 0.40625000000000006, "field_accuracy": 0.375, "normalized_numerical_error": 0.12512732922820366, "status_accuracy": 0.8333333333333334, "format_validity": 1, "pass_at_1": 0.40625, "pass_at_3": 0.59375, "pass_power_3": 0.21875, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.9895833333333333, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 66, "needs_clarification": 6 }, "needs_clarification": { "needs_clarification": 12 }, "out_of_domain": { "invalid_input": 10, "out_of_domain": 2 } }, "cost_per_correct_task_usd": 0.05712149358974361 }, "propeller-systems": { "n": 72, "task_accuracy": 0.4861111111111111, "macro_operation_accuracy": 0.4861111111111111, "field_accuracy": 0.4074074074074074, "normalized_numerical_error": 4.1075880870365046e-7, "status_accuracy": 0.8888888888888888, "format_validity": 1, "pass_at_1": 0.5, "pass_at_3": 0.7083333333333334, "pass_power_3": 0.3333333333333333, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 2.2777777777777777, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 51, "needs_clarification": 3 }, "out_of_domain": { "invalid_input": 5, "out_of_domain": 4 }, "needs_clarification": { "needs_clarification": 9 } }, "cost_per_correct_task_usd": 0.05703525714285715 }, "rocket-analysis": { "n": 72, "task_accuracy": 0.5138888888888888, "macro_operation_accuracy": 0.5138888888888888, "field_accuracy": 0.5648148148148148, "normalized_numerical_error": 0.05281172686958533, "status_accuracy": 0.9305555555555556, "format_validity": 1, "pass_at_1": 0.5833333333333334, "pass_at_3": 0.625, "pass_power_3": 0.375, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 2.0694444444444446, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "needs_clarification": { "needs_clarification": 9 }, "success": { "success": 54 }, "out_of_domain": { "out_of_domain": 4, "invalid_input": 5 } }, "cost_per_correct_task_usd": 0.0486125945945946 }, "orbital-mechanics": { "n": 168, "task_accuracy": 0.9047619047619048, "macro_operation_accuracy": 0.9047619047619049, "field_accuracy": 0.9801587301587301, "normalized_numerical_error": 0.0001232394365462502, "status_accuracy": 0.9226190476190477, "format_validity": 1, "pass_at_1": 0.8928571428571429, "pass_at_3": 0.9642857142857143, "pass_power_3": 0.8571428571428571, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5833333333333333, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 125, "needs_clarification": 1 }, "needs_clarification": { "needs_clarification": 21 }, "out_of_domain": { "invalid_input": 12, "out_of_domain": 9 } }, "cost_per_correct_task_usd": 0.019516501644736832 }, "guidance-navigation-control": { "n": 48, "task_accuracy": 0.5416666666666666, "macro_operation_accuracy": 0.5416666666666666, "field_accuracy": 0.5277777777777778, "normalized_numerical_error": 0.003963666565432901, "status_accuracy": 0.8958333333333334, "format_validity": 1, "pass_at_1": 0.5625, "pass_at_3": 0.6875, "pass_power_3": 0.375, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "out_of_domain": { "out_of_domain": 1, "invalid_input": 5 }, "needs_clarification": { "needs_clarification": 6 }, "success": { "success": 36 } }, "cost_per_correct_task_usd": 0.02970133653846154 }, "aircraft-performance": { "n": 168, "task_accuracy": 0.6369047619047619, "macro_operation_accuracy": 0.636904761904762, "field_accuracy": 0.6587301587301587, "normalized_numerical_error": 0, "status_accuracy": 0.8928571428571429, "format_validity": 1, "pass_at_1": 0.6607142857142857, "pass_at_3": 0.7857142857142857, "pass_power_3": 0.4642857142857143, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.8333333333333333, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 126 }, "needs_clarification": { "needs_clarification": 21 }, "out_of_domain": { "invalid_input": 18, "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0.03002065654205607 }, "trajectory-optimization": { "n": 144, "task_accuracy": 0.6527777777777778, "macro_operation_accuracy": 0.6527777777777778, "field_accuracy": 0.6990740740740741, "normalized_numerical_error": 0.12967952193683777, "status_accuracy": 0.9097222222222222, "format_validity": 1, "pass_at_1": 0.625, "pass_at_3": 0.8333333333333334, "pass_power_3": 0.4375, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.7916666666666667, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "needs_clarification": { "needs_clarification": 18 }, "success": { "success": 108 }, "out_of_domain": { "invalid_input": 13, "out_of_domain": 5 } }, "cost_per_correct_task_usd": 0.029697933510638295 } } }, "by_operation": { "nemotron-base": { "search_airports": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": 1, "status_accuracy": 0.6666666666666666, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": null, "classification_matrix": { "success": { "out_of_domain": 3, "success": 10, "needs_clarification": 5 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0 }, "plan_flight": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0, "normalized_numerical_error": 3041.232352941175, "status_accuracy": 0.9166666666666666, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 17, "needs_clarification": 1 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 } }, "cost_per_correct_task_usd": 0 }, "plan_multi_leg_flight": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0, "normalized_numerical_error": 29616.087500000005, "status_accuracy": 0.3333333333333333, "format_validity": 0.9583333333333334, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "terminal_or_invalid": 1, "needs_clarification": 10, "invalid_input": 4, "success": 3 }, "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "calculate_distance": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": 346.911764705886, "status_accuracy": 0.9583333333333334, "format_validity": 0.9583333333333334, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "out_of_domain": { "out_of_domain": 3 }, "success": { "success": 17, "terminal_or_invalid": 1 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "get_aircraft_performance": { "n": 24, "task_accuracy": 0.125, "macro_operation_accuracy": 0.125, "field_accuracy": 0, "normalized_numerical_error": 577000000, "status_accuracy": 0.125, "format_validity": 0.9166666666666666, "pass_at_1": 0.125, "pass_at_3": 0.25, "pass_power_3": 0, "terminal_failure_rate": 0.08333333333333333, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 13, "terminal_or_invalid": 1, "invalid_input": 4 }, "out_of_domain": { "out_of_domain": 1, "invalid_input": 2 }, "needs_clarification": { "needs_clarification": 2, "terminal_or_invalid": 1 } }, "cost_per_correct_task_usd": 0 }, "get_aircraft_database": { "n": 24, "task_accuracy": 0.16666666666666666, "macro_operation_accuracy": 0.16666666666666666, "field_accuracy": 0, "normalized_numerical_error": 1, "status_accuracy": 0.3333333333333333, "format_validity": 0.9583333333333334, "pass_at_1": 0.125, "pass_at_3": 0.25, "pass_power_3": 0, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": null, "classification_matrix": { "success": { "out_of_domain": 5, "invalid_input": 7, "success": 4, "needs_clarification": 2 }, "needs_clarification": { "terminal_or_invalid": 1, "needs_clarification": 2 }, "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 } }, "cost_per_correct_task_usd": 0 }, "get_system_status": { "n": 24, "task_accuracy": 0.16666666666666666, "macro_operation_accuracy": 0.16666666666666666, "field_accuracy": 0, "normalized_numerical_error": 1, "status_accuracy": 0.4583333333333333, "format_validity": 0.9583333333333334, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": null, "classification_matrix": { "success": { "success": 7, "invalid_input": 7, "needs_clarification": 3, "terminal_or_invalid": 1 }, "out_of_domain": { "out_of_domain": 1, "invalid_input": 2 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "convert_units": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0 }, "get_atmosphere_profile": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.875, "format_validity": 0.9583333333333334, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "out_of_domain": { "out_of_domain": 3 }, "success": { "out_of_domain": 2, "terminal_or_invalid": 1, "success": 15 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "wind_model_simple": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0.05555555555555555, "normalized_numerical_error": 0, "status_accuracy": 0.5833333333333334, "format_validity": 0.8333333333333334, "pass_at_1": 0.125, "pass_at_3": 0.375, "pass_power_3": 0, "terminal_failure_rate": 0.16666666666666666, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "needs_clarification": { "needs_clarification": 2, "terminal_or_invalid": 1 }, "success": { "success": 10, "terminal_or_invalid": 3, "needs_clarification": 5 }, "out_of_domain": { "invalid_input": 1, "out_of_domain": 2 } }, "cost_per_correct_task_usd": 0 }, "transform_frames": { "n": 24, "task_accuracy": 0.16666666666666666, "macro_operation_accuracy": 0.16666666666666666, "field_accuracy": 0, "normalized_numerical_error": 132200568.89671366, "status_accuracy": 0.5833333333333334, "format_validity": 0.9583333333333334, "pass_at_1": 0.125, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "out_of_domain": { "invalid_input": 2, "out_of_domain": 1 }, "success": { "success": 10, "needs_clarification": 6, "terminal_or_invalid": 1, "invalid_input": 1 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "geodetic_to_ecef": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.9166666666666666, "format_validity": 0.9166666666666666, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0.08333333333333333, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "out_of_domain": { "out_of_domain": 3 }, "success": { "success": 17, "terminal_or_invalid": 1 }, "needs_clarification": { "needs_clarification": 2, "terminal_or_invalid": 1 } }, "cost_per_correct_task_usd": 0 }, "ecef_to_geodetic": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0, "normalized_numerical_error": 820833.1985603827, "status_accuracy": 0.75, "format_validity": 0.75, "pass_at_1": 0.125, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0.25, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 13, "terminal_or_invalid": 5 }, "needs_clarification": { "terminal_or_invalid": 1, "needs_clarification": 2 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0 }, "wing_vlm_analysis": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0.027777777777777776, "normalized_numerical_error": 5.411404529053118, "status_accuracy": 0.625, "format_validity": 0.9583333333333334, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 10, "needs_clarification": 8 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 2, "terminal_or_invalid": 1 } }, "cost_per_correct_task_usd": 0 }, "airfoil_polar_analysis": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.9583333333333334, "format_validity": 0.9583333333333334, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 2, "terminal_or_invalid": 1 } }, "cost_per_correct_task_usd": 0 }, "calculate_stability_derivatives": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": 145699.07482146096, "status_accuracy": 0.7083333333333334, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 11, "needs_clarification": 7 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0 }, "get_airfoil_database": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.9166666666666666, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": null, "classification_matrix": { "success": { "success": 16, "invalid_input": 1, "needs_clarification": 1 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0 }, "propeller_bemt_analysis": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0, "normalized_numerical_error": 26.22612638465418, "status_accuracy": 0.25, "format_validity": 0.875, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0.125, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 12, "terminal_or_invalid": 2, "invalid_input": 3, "success": 1 }, "needs_clarification": { "needs_clarification": 2, "terminal_or_invalid": 1 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0 }, "uav_energy_estimate": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0, "normalized_numerical_error": 10772.880280634106, "status_accuracy": 0.25, "format_validity": 0.9583333333333334, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 16, "out_of_domain": 1, "success": 1 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 2, "terminal_or_invalid": 1 } }, "cost_per_correct_task_usd": 0 }, "get_propeller_database": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.625, "format_validity": 0.9166666666666666, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0.08333333333333333, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": null, "classification_matrix": { "success": { "needs_clarification": 4, "invalid_input": 2, "success": 10, "out_of_domain": 1, "terminal_or_invalid": 1 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 2, "terminal_or_invalid": 1 } }, "cost_per_correct_task_usd": 0 }, "rocket_3dof_trajectory": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": 146.54632664347426, "status_accuracy": 0.375, "format_validity": 0.9583333333333334, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 14, "success": 3, "terminal_or_invalid": 1 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "estimate_rocket_sizing": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0, "normalized_numerical_error": 21585.06818181818, "status_accuracy": 0.20833333333333334, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 16, "invalid_input": 2 }, "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "optimize_launch_angle": { "n": 24, "task_accuracy": 0.16666666666666666, "macro_operation_accuracy": 0.16666666666666666, "field_accuracy": 0, "normalized_numerical_error": 54000050, "status_accuracy": 0.16666666666666666, "format_validity": 0.9583333333333334, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "needs_clarification": { "needs_clarification": 2, "invalid_input": 1 }, "success": { "needs_clarification": 16, "invalid_input": 1, "terminal_or_invalid": 1 }, "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 } }, "cost_per_correct_task_usd": 0 }, "elements_to_state_vector": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.875, "format_validity": 0.9166666666666666, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0.08333333333333333, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "needs_clarification": { "needs_clarification": 3 }, "success": { "success": 15, "terminal_or_invalid": 2, "needs_clarification": 1 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0 }, "state_vector_to_elements": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0, "normalized_numerical_error": 23218545.17617784, "status_accuracy": 0.5416666666666666, "format_validity": 0.875, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0.125, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 8, "terminal_or_invalid": 2, "invalid_input": 1, "needs_clarification": 7 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 2, "terminal_or_invalid": 1 } }, "cost_per_correct_task_usd": 0 }, "propagate_orbit_j2": { "n": 24, "task_accuracy": 0.16666666666666666, "macro_operation_accuracy": 0.16666666666666666, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.20833333333333334, "format_validity": 0.9166666666666666, "pass_at_1": 0.125, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0.08333333333333333, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 13, "success": 1, "terminal_or_invalid": 2, "out_of_domain": 1, "invalid_input": 1 }, "out_of_domain": { "invalid_input": 2, "out_of_domain": 1 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "calculate_ground_track": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0.027777777777777776, "normalized_numerical_error": 17696.164989583332, "status_accuracy": 0.3333333333333333, "format_validity": 0.9166666666666666, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0.08333333333333333, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 13, "success": 2, "terminal_or_invalid": 2, "invalid_input": 1 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0 }, "hohmann_transfer": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": 18436563.39097072, "status_accuracy": 0.6666666666666666, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 10, "needs_clarification": 8 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0 }, "orbital_rendezvous_planning": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0.027777777777777776, "normalized_numerical_error": 1893021.2, "status_accuracy": 0.625, "format_validity": 0.7916666666666666, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0.20833333333333334, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 9, "needs_clarification": 4, "terminal_or_invalid": 5 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0 }, "lambert_problem_solver": { "n": 24, "task_accuracy": 0.16666666666666666, "macro_operation_accuracy": 0.16666666666666666, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.20833333333333334, "format_validity": 0.875, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0.125, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "out_of_domain": { "out_of_domain": 1, "invalid_input": 2 }, "success": { "needs_clarification": 12, "terminal_or_invalid": 3, "invalid_input": 1, "out_of_domain": 1, "success": 1 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "kalman_filter_state_estimation": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 }, "success": { "success": 18 } }, "cost_per_correct_task_usd": 0 }, "lqr_controller_design": { "n": 24, "task_accuracy": 0.4166666666666667, "macro_operation_accuracy": 0.4166666666666667, "field_accuracy": 0.2777777777777778, "normalized_numerical_error": 2.2676127370386294, "status_accuracy": 0.9583333333333334, "format_validity": 1, "pass_at_1": 0.375, "pass_at_3": 0.75, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "invalid_input": 1, "out_of_domain": 2 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "density_altitude_calculator": { "n": 24, "task_accuracy": 0.125, "macro_operation_accuracy": 0.125, "field_accuracy": 0.027777777777777776, "normalized_numerical_error": 3716.6285046299317, "status_accuracy": 0.7083333333333334, "format_validity": 0.9583333333333334, "pass_at_1": 0.125, "pass_at_3": 0.125, "pass_power_3": 0.125, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "out_of_domain": { "invalid_input": 3 }, "success": { "success": 14, "needs_clarification": 3, "terminal_or_invalid": 1 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "true_airspeed_converter": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": 12588.641666666668, "status_accuracy": 0.25, "format_validity": 0.875, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0.125, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "needs_clarification": { "needs_clarification": 3 }, "success": { "needs_clarification": 14, "terminal_or_invalid": 3, "invalid_input": 1 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0 }, "stall_speed_calculator": { "n": 24, "task_accuracy": 0.4583333333333333, "macro_operation_accuracy": 0.4583333333333333, "field_accuracy": 0.3333333333333333, "normalized_numerical_error": 3017, "status_accuracy": 0.7916666666666666, "format_validity": 1, "pass_at_1": 0.375, "pass_at_3": 0.75, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 14, "needs_clarification": 4 }, "out_of_domain": { "invalid_input": 1, "out_of_domain": 2 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "weight_and_balance": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0.5, "normalized_numerical_error": 68.05555555554807, "status_accuracy": 0.9583333333333334, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "takeoff_performance": { "n": 24, "task_accuracy": 0.125, "macro_operation_accuracy": 0.125, "field_accuracy": 0, "normalized_numerical_error": 77.4375, "status_accuracy": 0.5416666666666666, "format_validity": 0.875, "pass_at_1": 0, "pass_at_3": 0.25, "pass_power_3": 0, "terminal_failure_rate": 0.125, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 5, "success": 10, "invalid_input": 1, "terminal_or_invalid": 2 }, "out_of_domain": { "invalid_input": 2, "out_of_domain": 1 }, "needs_clarification": { "needs_clarification": 2, "terminal_or_invalid": 1 } }, "cost_per_correct_task_usd": 0 }, "landing_performance": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": 120.25, "status_accuracy": 0.375, "format_validity": 0.9583333333333334, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 }, "success": { "needs_clarification": 14, "terminal_or_invalid": 1, "success": 3 } }, "cost_per_correct_task_usd": 0 }, "fuel_reserve_calculator": { "n": 24, "task_accuracy": 0.5833333333333334, "macro_operation_accuracy": 0.5833333333333334, "field_accuracy": 0.4444444444444444, "normalized_numerical_error": 43645.833333333336, "status_accuracy": 0.875, "format_validity": 0.9166666666666666, "pass_at_1": 0.75, "pass_at_3": 0.875, "pass_power_3": 0.375, "terminal_failure_rate": 0.08333333333333333, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 15, "terminal_or_invalid": 2, "needs_clarification": 1 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0 }, "optimize_thrust_profile": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0.1388888888888889, "normalized_numerical_error": 96430.52108755687, "status_accuracy": 0.6666666666666666, "format_validity": 0.9166666666666666, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0.08333333333333333, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 }, "success": { "success": 10, "terminal_or_invalid": 2, "needs_clarification": 6 } }, "cost_per_correct_task_usd": 0 }, "trajectory_sensitivity_analysis": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0, "normalized_numerical_error": 206.53134946562204, "status_accuracy": 0.375, "format_validity": 0.9583333333333334, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 11, "success": 4, "terminal_or_invalid": 1, "out_of_domain": 2 }, "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "genetic_algorithm_optimization": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0, "normalized_numerical_error": 5000000, "status_accuracy": 0.20833333333333334, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 15, "invalid_input": 3 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 } }, "cost_per_correct_task_usd": 0 }, "particle_swarm_optimization": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": 3828.6166666666672, "status_accuracy": 0.25, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "out_of_domain": { "out_of_domain": 3 }, "success": { "needs_clarification": 14, "invalid_input": 4 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "porkchop_plot_analysis": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0, "normalized_numerical_error": 41.66666666666668, "status_accuracy": 0.20833333333333334, "format_validity": 0.9166666666666666, "pass_at_1": 0.125, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0.08333333333333333, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "terminal_or_invalid": 1, "out_of_domain": 2 }, "success": { "invalid_input": 2, "needs_clarification": 8, "out_of_domain": 7, "terminal_or_invalid": 1 } }, "cost_per_correct_task_usd": 0 }, "monte_carlo_uncertainty_analysis": { "n": 24, "task_accuracy": 0.125, "macro_operation_accuracy": 0.125, "field_accuracy": 0, "normalized_numerical_error": 121139860.25876802, "status_accuracy": 0.8333333333333334, "format_validity": 0.9583333333333334, "pass_at_1": 0.125, "pass_at_3": 0.125, "pass_power_3": 0.125, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "invalid_input": 3 }, "success": { "success": 17, "terminal_or_invalid": 1 } }, "cost_per_correct_task_usd": 0 } }, "nemotron-docs-only": { "search_airports": { "n": 24, "task_accuracy": 0.16666666666666666, "macro_operation_accuracy": 0.16666666666666666, "field_accuracy": 0, "normalized_numerical_error": 1, "status_accuracy": 0.7916666666666666, "format_validity": 0.9583333333333334, "pass_at_1": 0.125, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0, "mean_skill_reads": 1.4583333333333333, "runner_invocation_rate_on_required_tasks": null, "classification_matrix": { "success": { "success": 15, "out_of_domain": 1, "terminal_or_invalid": 1, "needs_clarification": 1 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "invalid_input": 2, "out_of_domain": 1 } }, "cost_per_correct_task_usd": 0 }, "plan_flight": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0, "normalized_numerical_error": 8165.958333333331, "status_accuracy": 0.5833333333333334, "format_validity": 0.7083333333333334, "pass_at_1": 0.125, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0.2916666666666667, "mean_runner_calls": 0, "mean_skill_reads": 3.2916666666666665, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 9, "terminal_or_invalid": 7, "needs_clarification": 2 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "invalid_input": 1, "out_of_domain": 2 } }, "cost_per_correct_task_usd": 0 }, "plan_multi_leg_flight": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": 1921.1333333333334, "status_accuracy": 0.6666666666666666, "format_validity": 0.8333333333333334, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0.16666666666666666, "mean_runner_calls": 0, "mean_skill_reads": 3.2083333333333335, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 10, "needs_clarification": 4, "terminal_or_invalid": 4 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "calculate_distance": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0, "normalized_numerical_error": 810.3472222222279, "status_accuracy": 0.9583333333333334, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 1, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 }, "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "get_aircraft_performance": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": 665000000, "status_accuracy": 0.25, "format_validity": 0.9583333333333334, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0, "mean_skill_reads": 2.7916666666666665, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 14, "invalid_input": 3, "terminal_or_invalid": 1 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "get_aircraft_database": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0, "normalized_numerical_error": 1, "status_accuracy": 0.2916666666666667, "format_validity": 0.7916666666666666, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0.20833333333333334, "mean_runner_calls": 0, "mean_skill_reads": 2.4166666666666665, "runner_invocation_rate_on_required_tasks": null, "classification_matrix": { "success": { "needs_clarification": 9, "terminal_or_invalid": 5, "success": 2, "invalid_input": 2 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 } }, "cost_per_correct_task_usd": 0 }, "get_system_status": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0, "normalized_numerical_error": 1, "status_accuracy": 0.6666666666666666, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 1.4166666666666667, "runner_invocation_rate_on_required_tasks": null, "classification_matrix": { "success": { "out_of_domain": 5, "success": 11, "needs_clarification": 2 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 2, "invalid_input": 1 } }, "cost_per_correct_task_usd": 0 }, "convert_units": { "n": 24, "task_accuracy": 0.2916666666666667, "macro_operation_accuracy": 0.2916666666666667, "field_accuracy": 0.05555555555555555, "normalized_numerical_error": 0, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.375, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0.625, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0 }, "get_atmosphere_profile": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.9166666666666666, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 1.8333333333333333, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "out_of_domain": { "invalid_input": 1, "out_of_domain": 2 }, "success": { "success": 17, "out_of_domain": 1 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "wind_model_simple": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.9166666666666666, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0.875, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "needs_clarification": { "needs_clarification": 3 }, "success": { "needs_clarification": 1, "success": 17 }, "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 } }, "cost_per_correct_task_usd": 0 }, "transform_frames": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": 51637.55470905453, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 1.2083333333333333, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "out_of_domain": { "out_of_domain": 3 }, "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "geodetic_to_ecef": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.9583333333333334, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 1.125, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 }, "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "ecef_to_geodetic": { "n": 24, "task_accuracy": 0.16666666666666666, "macro_operation_accuracy": 0.16666666666666666, "field_accuracy": 0, "normalized_numerical_error": 1377036.765156638, "status_accuracy": 0.8333333333333334, "format_validity": 0.9166666666666666, "pass_at_1": 0.125, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0.08333333333333333, "mean_runner_calls": 0, "mean_skill_reads": 1, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 16, "terminal_or_invalid": 2 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "invalid_input": 2, "out_of_domain": 1 } }, "cost_per_correct_task_usd": 0 }, "wing_vlm_analysis": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0, "normalized_numerical_error": 4.998204902601936, "status_accuracy": 0.9166666666666666, "format_validity": 0.9583333333333334, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0, "mean_skill_reads": 1.25, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 17, "terminal_or_invalid": 1 }, "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "airfoil_polar_analysis": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0, "normalized_numerical_error": 78.05000000013496, "status_accuracy": 0.9583333333333334, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "invalid_input": 1, "out_of_domain": 2 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "calculate_stability_derivatives": { "n": 24, "task_accuracy": 0.16666666666666666, "macro_operation_accuracy": 0.16666666666666666, "field_accuracy": 0, "normalized_numerical_error": 16046.655831949573, "status_accuracy": 0.875, "format_validity": 0.9166666666666666, "pass_at_1": 0.125, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0.08333333333333333, "mean_runner_calls": 0, "mean_skill_reads": 0.5, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 17, "terminal_or_invalid": 1 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "invalid_input": 1, "terminal_or_invalid": 1, "out_of_domain": 1 } }, "cost_per_correct_task_usd": 0 }, "get_airfoil_database": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0, "normalized_numerical_error": 1, "status_accuracy": 0.8333333333333334, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 1.5833333333333333, "runner_invocation_rate_on_required_tasks": null, "classification_matrix": { "success": { "success": 15, "invalid_input": 1, "needs_clarification": 2 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 } }, "cost_per_correct_task_usd": 0 }, "propeller_bemt_analysis": { "n": 24, "task_accuracy": 0.16666666666666666, "macro_operation_accuracy": 0.16666666666666666, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.16666666666666666, "format_validity": 0.9583333333333334, "pass_at_1": 0.125, "pass_at_3": 0.25, "pass_power_3": 0, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0, "mean_skill_reads": 1.5416666666666667, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 15, "invalid_input": 2, "out_of_domain": 1 }, "needs_clarification": { "needs_clarification": 2, "terminal_or_invalid": 1 }, "out_of_domain": { "invalid_input": 1, "out_of_domain": 2 } }, "cost_per_correct_task_usd": 0 }, "uav_energy_estimate": { "n": 24, "task_accuracy": 0.16666666666666666, "macro_operation_accuracy": 0.16666666666666666, "field_accuracy": 0.05555555555555555, "normalized_numerical_error": 14566.61302766489, "status_accuracy": 0.7916666666666666, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 1.8333333333333333, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 3, "success": 15 }, "out_of_domain": { "out_of_domain": 1, "invalid_input": 2 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "get_propeller_database": { "n": 24, "task_accuracy": 0.125, "macro_operation_accuracy": 0.125, "field_accuracy": 0, "normalized_numerical_error": 1, "status_accuracy": 0.5833333333333334, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.125, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 1.9583333333333333, "runner_invocation_rate_on_required_tasks": null, "classification_matrix": { "success": { "needs_clarification": 7, "success": 11 }, "out_of_domain": { "invalid_input": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "rocket_3dof_trajectory": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": 32.98213062571992, "status_accuracy": 0.375, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 1.25, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 14, "success": 3, "out_of_domain": 1 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "estimate_rocket_sizing": { "n": 24, "task_accuracy": 0.2916666666666667, "macro_operation_accuracy": 0.2916666666666667, "field_accuracy": 0.1111111111111111, "normalized_numerical_error": 68.18749999999935, "status_accuracy": 0.5833333333333334, "format_validity": 0.9583333333333334, "pass_at_1": 0.125, "pass_at_3": 0.375, "pass_power_3": 0.125, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0, "mean_skill_reads": 1.7083333333333333, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 8, "success": 9, "terminal_or_invalid": 1 }, "out_of_domain": { "invalid_input": 1, "out_of_domain": 2 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "optimize_launch_angle": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0, "normalized_numerical_error": 90000000, "status_accuracy": 0.20833333333333334, "format_validity": 0.9583333333333334, "pass_at_1": 0.125, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0, "mean_skill_reads": 1.8333333333333333, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "needs_clarification": { "needs_clarification": 3 }, "success": { "needs_clarification": 17, "terminal_or_invalid": 1 }, "out_of_domain": { "invalid_input": 1, "out_of_domain": 2 } }, "cost_per_correct_task_usd": 0 }, "elements_to_state_vector": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 1.3333333333333333, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "needs_clarification": { "needs_clarification": 3 }, "success": { "success": 18 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0 }, "state_vector_to_elements": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0, "normalized_numerical_error": 130643.42513675982, "status_accuracy": 0.9166666666666666, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0.4583333333333333, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 17, "needs_clarification": 1 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 2, "needs_clarification": 1 } }, "cost_per_correct_task_usd": 0 }, "propagate_orbit_j2": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.25, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 1.4166666666666667, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 15, "invalid_input": 2, "success": 1 }, "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "calculate_ground_track": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.4166666666666667, "format_validity": 0.9166666666666666, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0.08333333333333333, "mean_runner_calls": 0, "mean_skill_reads": 1.75, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 4, "terminal_or_invalid": 2, "needs_clarification": 11, "invalid_input": 1 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0 }, "hohmann_transfer": { "n": 24, "task_accuracy": 0.16666666666666666, "macro_operation_accuracy": 0.16666666666666666, "field_accuracy": 0, "normalized_numerical_error": 31859.237342074368, "status_accuracy": 0.875, "format_validity": 0.9583333333333334, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0, "mean_skill_reads": 0.4166666666666667, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 17, "terminal_or_invalid": 1 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 1, "invalid_input": 2 } }, "cost_per_correct_task_usd": 0 }, "orbital_rendezvous_planning": { "n": 24, "task_accuracy": 0.16666666666666666, "macro_operation_accuracy": 0.16666666666666666, "field_accuracy": 0.027777777777777776, "normalized_numerical_error": 48411.98076923076, "status_accuracy": 0.875, "format_validity": 0.9583333333333334, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0, "mean_skill_reads": 0.6666666666666666, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 17, "terminal_or_invalid": 1 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 1, "invalid_input": 2 } }, "cost_per_correct_task_usd": 0 }, "lambert_problem_solver": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.20833333333333334, "format_validity": 0.9166666666666666, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0.08333333333333333, "mean_runner_calls": 0, "mean_skill_reads": 1.5833333333333333, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "out_of_domain": { "out_of_domain": 2, "terminal_or_invalid": 1 }, "success": { "needs_clarification": 15, "out_of_domain": 2, "terminal_or_invalid": 1 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "kalman_filter_state_estimation": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.875, "format_validity": 0.9166666666666666, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0.08333333333333333, "mean_runner_calls": 0, "mean_skill_reads": 0.4583333333333333, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 }, "needs_clarification": { "needs_clarification": 3 }, "success": { "success": 16, "terminal_or_invalid": 2 } }, "cost_per_correct_task_usd": 0 }, "lqr_controller_design": { "n": 24, "task_accuracy": 0.4583333333333333, "macro_operation_accuracy": 0.4583333333333333, "field_accuracy": 0.2777777777777778, "normalized_numerical_error": 10.830182268543574, "status_accuracy": 0.9583333333333334, "format_validity": 0.9583333333333334, "pass_at_1": 0.5, "pass_at_3": 0.75, "pass_power_3": 0.25, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0, "mean_skill_reads": 0.2916666666666667, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 17, "terminal_or_invalid": 1 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "density_altitude_calculator": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0, "normalized_numerical_error": 198.19871965594743, "status_accuracy": 0.9583333333333334, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 1.125, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "out_of_domain": { "invalid_input": 1, "out_of_domain": 2 }, "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "true_airspeed_converter": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0.1111111111111111, "normalized_numerical_error": 177.9307692307696, "status_accuracy": 0.75, "format_validity": 0.9166666666666666, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0.08333333333333333, "mean_runner_calls": 0, "mean_skill_reads": 1.3333333333333333, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "needs_clarification": { "needs_clarification": 3 }, "success": { "success": 13, "needs_clarification": 4, "terminal_or_invalid": 1 }, "out_of_domain": { "out_of_domain": 2, "terminal_or_invalid": 1 } }, "cost_per_correct_task_usd": 0 }, "stall_speed_calculator": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0.16666666666666666, "normalized_numerical_error": 10.437500000000432, "status_accuracy": 0.7916666666666666, "format_validity": 0.9166666666666666, "pass_at_1": 0.25, "pass_at_3": 0.375, "pass_power_3": 0.125, "terminal_failure_rate": 0.08333333333333333, "mean_runner_calls": 0, "mean_skill_reads": 0.16666666666666666, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 16, "terminal_or_invalid": 2 }, "out_of_domain": { "invalid_input": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "weight_and_balance": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0.5, "normalized_numerical_error": 112.13888888887847, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "takeoff_performance": { "n": 24, "task_accuracy": 0.16666666666666666, "macro_operation_accuracy": 0.16666666666666666, "field_accuracy": 0, "normalized_numerical_error": 41.33333333333333, "status_accuracy": 0.7083333333333334, "format_validity": 0.875, "pass_at_1": 0.125, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0.125, "mean_runner_calls": 0, "mean_skill_reads": 1.2916666666666667, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 13, "needs_clarification": 2, "terminal_or_invalid": 3 }, "out_of_domain": { "invalid_input": 2, "out_of_domain": 1 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "landing_performance": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0, "normalized_numerical_error": 25.333333333333332, "status_accuracy": 0.875, "format_validity": 0.9166666666666666, "pass_at_1": 0.125, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0.08333333333333333, "mean_runner_calls": 0, "mean_skill_reads": 2, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "out_of_domain": { "invalid_input": 1, "out_of_domain": 2 }, "needs_clarification": { "needs_clarification": 3 }, "success": { "success": 16, "terminal_or_invalid": 2 } }, "cost_per_correct_task_usd": 0 }, "fuel_reserve_calculator": { "n": 24, "task_accuracy": 0.4583333333333333, "macro_operation_accuracy": 0.4583333333333333, "field_accuracy": 0.3888888888888889, "normalized_numerical_error": 33145.833333333336, "status_accuracy": 0.9166666666666666, "format_validity": 0.9583333333333334, "pass_at_1": 0.375, "pass_at_3": 0.75, "pass_power_3": 0.125, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0, "mean_skill_reads": 0.08333333333333333, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 2, "terminal_or_invalid": 1 }, "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 } }, "cost_per_correct_task_usd": 0 }, "optimize_thrust_profile": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0.1111111111111111, "normalized_numerical_error": 45555.555555555555, "status_accuracy": 0.5416666666666666, "format_validity": 0.9583333333333334, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0, "mean_skill_reads": 1.625, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 }, "success": { "needs_clarification": 7, "success": 7, "invalid_input": 1, "out_of_domain": 2, "terminal_or_invalid": 1 } }, "cost_per_correct_task_usd": 0 }, "trajectory_sensitivity_analysis": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0, "normalized_numerical_error": 87.7118145613711, "status_accuracy": 0.3333333333333333, "format_validity": 0.9583333333333334, "pass_at_1": 0.125, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0, "mean_skill_reads": 1.6666666666666667, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 12, "success": 3, "out_of_domain": 2, "terminal_or_invalid": 1 }, "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "genetic_algorithm_optimization": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0, "normalized_numerical_error": 4575507.144444444, "status_accuracy": 0.4166666666666667, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 1, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 11, "success": 5, "out_of_domain": 1, "invalid_input": 1 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 } }, "cost_per_correct_task_usd": 0 }, "particle_swarm_optimization": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": 13565.430499999999, "status_accuracy": 0.5833333333333334, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0.9166666666666666, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "out_of_domain": { "out_of_domain": 3 }, "success": { "success": 8, "needs_clarification": 10 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "porkchop_plot_analysis": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": 41.66666666666667, "status_accuracy": 0.25, "format_validity": 0.875, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0.125, "mean_runner_calls": 0, "mean_skill_reads": 1.125, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 }, "success": { "invalid_input": 3, "needs_clarification": 12, "terminal_or_invalid": 3 } }, "cost_per_correct_task_usd": 0 }, "monte_carlo_uncertainty_analysis": { "n": 24, "task_accuracy": 0.16666666666666666, "macro_operation_accuracy": 0.16666666666666666, "field_accuracy": 0, "normalized_numerical_error": 79982283.20687845, "status_accuracy": 0.6666666666666666, "format_validity": 0.9583333333333334, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0, "mean_skill_reads": 0.4166666666666667, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "needs_clarification": { "needs_clarification": 2, "terminal_or_invalid": 1 }, "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 }, "success": { "success": 12, "needs_clarification": 6 } }, "cost_per_correct_task_usd": 0 } }, "nemotron-runner-only": { "search_airports": { "n": 24, "task_accuracy": 0.16666666666666666, "macro_operation_accuracy": 0.16666666666666666, "field_accuracy": 0, "normalized_numerical_error": 1, "status_accuracy": 0.9166666666666666, "format_validity": 0.9583333333333334, "pass_at_1": 0.125, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0.75, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": null, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "terminal_or_invalid": 1, "invalid_input": 1, "out_of_domain": 1 } }, "cost_per_correct_task_usd": 0 }, "plan_flight": { "n": 24, "task_accuracy": 0.7916666666666666, "macro_operation_accuracy": 0.7916666666666666, "field_accuracy": 0.7222222222222222, "normalized_numerical_error": 0, "status_accuracy": 0.9166666666666666, "format_validity": 0.9166666666666666, "pass_at_1": 0.625, "pass_at_3": 1, "pass_power_3": 0.5, "terminal_failure_rate": 0.08333333333333333, "mean_runner_calls": 0.875, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0.9444444444444444, "classification_matrix": { "success": { "success": 16, "terminal_or_invalid": 2 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0 }, "plan_multi_leg_flight": { "n": 24, "task_accuracy": 0.75, "macro_operation_accuracy": 0.75, "field_accuracy": 0.6666666666666666, "normalized_numerical_error": 0, "status_accuracy": 0.9583333333333334, "format_validity": 0.9583333333333334, "pass_at_1": 0.875, "pass_at_3": 0.875, "pass_power_3": 0.625, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0.75, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 17, "terminal_or_invalid": 1 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "calculate_distance": { "n": 24, "task_accuracy": 0.9583333333333334, "macro_operation_accuracy": 0.9583333333333334, "field_accuracy": 1, "normalized_numerical_error": 0, "status_accuracy": 0.9583333333333334, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 0.875, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 }, "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "get_aircraft_performance": { "n": 24, "task_accuracy": 0.5, "macro_operation_accuracy": 0.5, "field_accuracy": 0.3888888888888889, "normalized_numerical_error": 68500000, "status_accuracy": 0.5, "format_validity": 0.7083333333333334, "pass_at_1": 0.375, "pass_at_3": 0.875, "pass_power_3": 0.125, "terminal_failure_rate": 0.2916666666666667, "mean_runner_calls": 1.7083333333333333, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "needs_clarification": 3, "success": 7, "out_of_domain": 1, "terminal_or_invalid": 7 }, "out_of_domain": { "invalid_input": 1, "out_of_domain": 2 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "get_aircraft_database": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": 1, "status_accuracy": 0.6666666666666666, "format_validity": 0.7083333333333334, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0.2916666666666667, "mean_runner_calls": 1.2083333333333333, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": null, "classification_matrix": { "success": { "success": 10, "terminal_or_invalid": 7, "needs_clarification": 1 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0 }, "get_system_status": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0.05555555555555555, "normalized_numerical_error": 0.8571428571428571, "status_accuracy": 0.875, "format_validity": 0.9583333333333334, "pass_at_1": 0.125, "pass_at_3": 0.375, "pass_power_3": 0, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0.7083333333333334, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": null, "classification_matrix": { "success": { "success": 17, "out_of_domain": 1 }, "out_of_domain": { "invalid_input": 1, "out_of_domain": 2 }, "needs_clarification": { "terminal_or_invalid": 1, "needs_clarification": 2 } }, "cost_per_correct_task_usd": 0 }, "convert_units": { "n": 24, "task_accuracy": 0.4166666666666667, "macro_operation_accuracy": 0.4166666666666667, "field_accuracy": 0.2777777777777778, "normalized_numerical_error": 0, "status_accuracy": 0.9166666666666666, "format_validity": 0.9166666666666666, "pass_at_1": 0.375, "pass_at_3": 0.75, "pass_power_3": 0.125, "terminal_failure_rate": 0.08333333333333333, "mean_runner_calls": 0.7083333333333334, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0.9444444444444444, "classification_matrix": { "success": { "terminal_or_invalid": 1, "success": 17 }, "needs_clarification": { "needs_clarification": 2, "terminal_or_invalid": 1 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0 }, "get_atmosphere_profile": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "out_of_domain": { "out_of_domain": 3 }, "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "wind_model_simple": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "needs_clarification": { "needs_clarification": 3 }, "success": { "success": 18 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0 }, "transform_frames": { "n": 24, "task_accuracy": 0.375, "macro_operation_accuracy": 0.375, "field_accuracy": 0.2777777777777778, "normalized_numerical_error": 0.0008949171751737595, "status_accuracy": 0.875, "format_validity": 0.9166666666666666, "pass_at_1": 0.375, "pass_at_3": 0.75, "pass_power_3": 0, "terminal_failure_rate": 0.08333333333333333, "mean_runner_calls": 0.75, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0.9444444444444444, "classification_matrix": { "out_of_domain": { "invalid_input": 1, "out_of_domain": 2 }, "success": { "success": 17, "terminal_or_invalid": 1 }, "needs_clarification": { "needs_clarification": 2, "terminal_or_invalid": 1 } }, "cost_per_correct_task_usd": 0 }, "geodetic_to_ecef": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.9583333333333334, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "out_of_domain": { "out_of_domain": 3 }, "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 2, "invalid_input": 1 } }, "cost_per_correct_task_usd": 0 }, "ecef_to_geodetic": { "n": 24, "task_accuracy": 1, "macro_operation_accuracy": 1, "field_accuracy": 1, "normalized_numerical_error": 2.7729482907387944e-7, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 1, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0 }, "wing_vlm_analysis": { "n": 24, "task_accuracy": 0.8333333333333334, "macro_operation_accuracy": 0.8333333333333334, "field_accuracy": 0.8888888888888888, "normalized_numerical_error": 0.2453030311651553, "status_accuracy": 0.9166666666666666, "format_validity": 0.9583333333333334, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 0.5, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0.7083333333333334, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0.9444444444444444, "classification_matrix": { "success": { "success": 17, "terminal_or_invalid": 1 }, "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "airfoil_polar_analysis": { "n": 24, "task_accuracy": 0.4583333333333333, "macro_operation_accuracy": 0.4583333333333333, "field_accuracy": 0.3888888888888889, "normalized_numerical_error": 0, "status_accuracy": 0.9166666666666666, "format_validity": 0.9583333333333334, "pass_at_1": 0.5, "pass_at_3": 0.625, "pass_power_3": 0.125, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0.875, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 }, "needs_clarification": { "needs_clarification": 2, "terminal_or_invalid": 1 } }, "cost_per_correct_task_usd": 0 }, "calculate_stability_derivatives": { "n": 24, "task_accuracy": 1, "macro_operation_accuracy": 1, "field_accuracy": 1, "normalized_numerical_error": 0, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 1, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0 }, "get_airfoil_database": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": 1, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0.7916666666666666, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": null, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0 }, "propeller_bemt_analysis": { "n": 24, "task_accuracy": 0.9583333333333334, "macro_operation_accuracy": 0.9583333333333334, "field_accuracy": 1, "normalized_numerical_error": 9.930990045122858e-7, "status_accuracy": 0.9583333333333334, "format_validity": 0.9583333333333334, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 0.875, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0.7916666666666666, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 2, "terminal_or_invalid": 1 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0 }, "uav_energy_estimate": { "n": 24, "task_accuracy": 0.9166666666666666, "macro_operation_accuracy": 0.9166666666666666, "field_accuracy": 0.8888888888888888, "normalized_numerical_error": 108.93006703966932, "status_accuracy": 0.9583333333333334, "format_validity": 0.9583333333333334, "pass_at_1": 0.75, "pass_at_3": 1, "pass_power_3": 0.75, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0.875, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 17, "terminal_or_invalid": 1 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "get_propeller_database": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.9583333333333334, "format_validity": 0.9583333333333334, "pass_at_1": 0.125, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0.75, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": null, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "terminal_or_invalid": 1, "needs_clarification": 2 } }, "cost_per_correct_task_usd": 0 }, "rocket_3dof_trajectory": { "n": 24, "task_accuracy": 0.9166666666666666, "macro_operation_accuracy": 0.9166666666666666, "field_accuracy": 0.8888888888888888, "normalized_numerical_error": 0.32514343809691304, "status_accuracy": 0.9583333333333334, "format_validity": 0.9583333333333334, "pass_at_1": 0.875, "pass_at_3": 1, "pass_power_3": 0.75, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0.7083333333333334, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0.9444444444444444, "classification_matrix": { "success": { "success": 17, "terminal_or_invalid": 1 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "estimate_rocket_sizing": { "n": 24, "task_accuracy": 0.9166666666666666, "macro_operation_accuracy": 0.9166666666666666, "field_accuracy": 0.9444444444444444, "normalized_numerical_error": 0, "status_accuracy": 0.9583333333333334, "format_validity": 1, "pass_at_1": 0.875, "pass_at_3": 1, "pass_power_3": 0.75, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "optimize_launch_angle": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0.4722222222222222, "normalized_numerical_error": 0, "status_accuracy": 0.9583333333333334, "format_validity": 0.9583333333333334, "pass_at_1": 0.125, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0.75, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "needs_clarification": { "terminal_or_invalid": 1, "needs_clarification": 2 }, "success": { "success": 18 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0 }, "elements_to_state_vector": { "n": 24, "task_accuracy": 0.625, "macro_operation_accuracy": 0.625, "field_accuracy": 0.5, "normalized_numerical_error": null, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.5, "pass_at_3": 1, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "needs_clarification": { "needs_clarification": 3 }, "success": { "success": 18 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0 }, "state_vector_to_elements": { "n": 24, "task_accuracy": 0.9166666666666666, "macro_operation_accuracy": 0.9166666666666666, "field_accuracy": 0.9166666666666666, "normalized_numerical_error": 518.2368948687239, "status_accuracy": 0.9583333333333334, "format_validity": 0.9583333333333334, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 0.75, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0.8333333333333334, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 17, "terminal_or_invalid": 1 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0 }, "propagate_orbit_j2": { "n": 24, "task_accuracy": 0.625, "macro_operation_accuracy": 0.625, "field_accuracy": 0.5555555555555556, "normalized_numerical_error": null, "status_accuracy": 0.875, "format_validity": 0.875, "pass_at_1": 0.625, "pass_at_3": 1, "pass_power_3": 0.25, "terminal_failure_rate": 0.125, "mean_runner_calls": 0.7083333333333334, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0.9444444444444444, "classification_matrix": { "success": { "success": 16, "terminal_or_invalid": 2 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 2, "terminal_or_invalid": 1 } }, "cost_per_correct_task_usd": 0 }, "calculate_ground_track": { "n": 24, "task_accuracy": 0.5833333333333334, "macro_operation_accuracy": 0.5833333333333334, "field_accuracy": 0.4444444444444444, "normalized_numerical_error": 91.16359020833283, "status_accuracy": 0.9583333333333334, "format_validity": 0.9583333333333334, "pass_at_1": 0.75, "pass_at_3": 0.875, "pass_power_3": 0.375, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0.9166666666666666, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0.9444444444444444, "classification_matrix": { "success": { "success": 17, "terminal_or_invalid": 1 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0 }, "hohmann_transfer": { "n": 24, "task_accuracy": 0.875, "macro_operation_accuracy": 0.875, "field_accuracy": 0.8333333333333334, "normalized_numerical_error": 0, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.75, "pass_at_3": 1, "pass_power_3": 0.75, "terminal_failure_rate": 0, "mean_runner_calls": 0.9166666666666666, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0 }, "orbital_rendezvous_planning": { "n": 24, "task_accuracy": 0.75, "macro_operation_accuracy": 0.75, "field_accuracy": 0.6666666666666666, "normalized_numerical_error": 181.66428571427946, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.625, "pass_at_3": 1, "pass_power_3": 0.375, "terminal_failure_rate": 0, "mean_runner_calls": 0.7916666666666666, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0 }, "lambert_problem_solver": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "out_of_domain": { "out_of_domain": 3 }, "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "kalman_filter_state_estimation": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0.1111111111111111, "normalized_numerical_error": null, "status_accuracy": 0.9166666666666666, "format_validity": 0.9583333333333334, "pass_at_1": 0.25, "pass_at_3": 0.5, "pass_power_3": 0.125, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0.75, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "out_of_domain": { "out_of_domain": 1, "invalid_input": 1, "terminal_or_invalid": 1 }, "needs_clarification": { "needs_clarification": 3 }, "success": { "success": 18 } }, "cost_per_correct_task_usd": 0 }, "lqr_controller_design": { "n": 24, "task_accuracy": 0.2916666666666667, "macro_operation_accuracy": 0.2916666666666667, "field_accuracy": 0.05555555555555555, "normalized_numerical_error": 0.42784286119436293, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.375, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0.7916666666666666, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "density_altitude_calculator": { "n": 24, "task_accuracy": 0.9583333333333334, "macro_operation_accuracy": 0.9583333333333334, "field_accuracy": 0.9444444444444444, "normalized_numerical_error": 0, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 0.875, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "out_of_domain": { "out_of_domain": 3 }, "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "true_airspeed_converter": { "n": 24, "task_accuracy": 0.7916666666666666, "macro_operation_accuracy": 0.7916666666666666, "field_accuracy": 0.8611111111111112, "normalized_numerical_error": 0, "status_accuracy": 0.9166666666666666, "format_validity": 0.9166666666666666, "pass_at_1": 0.625, "pass_at_3": 1, "pass_power_3": 0.5, "terminal_failure_rate": 0.08333333333333333, "mean_runner_calls": 0.7916666666666666, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "needs_clarification": { "terminal_or_invalid": 2, "needs_clarification": 1 }, "success": { "success": 18 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0 }, "stall_speed_calculator": { "n": 24, "task_accuracy": 0.7916666666666666, "macro_operation_accuracy": 0.7916666666666666, "field_accuracy": 0.8333333333333334, "normalized_numerical_error": 0, "status_accuracy": 0.875, "format_validity": 0.9166666666666666, "pass_at_1": 0.875, "pass_at_3": 1, "pass_power_3": 0.5, "terminal_failure_rate": 0.08333333333333333, "mean_runner_calls": 0.7083333333333334, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0.9444444444444444, "classification_matrix": { "success": { "success": 17, "terminal_or_invalid": 1 }, "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 }, "needs_clarification": { "needs_clarification": 2, "terminal_or_invalid": 1 } }, "cost_per_correct_task_usd": 0 }, "weight_and_balance": { "n": 24, "task_accuracy": 0.8333333333333334, "macro_operation_accuracy": 0.8333333333333334, "field_accuracy": 0.8888888888888888, "normalized_numerical_error": 4.027777777779494, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.625, "pass_at_3": 1, "pass_power_3": 0.625, "terminal_failure_rate": 0, "mean_runner_calls": 1.4166666666666667, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "takeoff_performance": { "n": 24, "task_accuracy": 0.3333333333333333, "macro_operation_accuracy": 0.3333333333333333, "field_accuracy": 0.16666666666666666, "normalized_numerical_error": 0, "status_accuracy": 0.9583333333333334, "format_validity": 1, "pass_at_1": 0.5, "pass_at_3": 0.625, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "landing_performance": { "n": 24, "task_accuracy": 0.7083333333333334, "macro_operation_accuracy": 0.7083333333333334, "field_accuracy": 0.6111111111111112, "normalized_numerical_error": 0, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.5, "pass_at_3": 1, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 }, "success": { "success": 18 } }, "cost_per_correct_task_usd": 0 }, "fuel_reserve_calculator": { "n": 24, "task_accuracy": 0.5, "macro_operation_accuracy": 0.5, "field_accuracy": 0.4444444444444444, "normalized_numerical_error": 0, "status_accuracy": 0.875, "format_validity": 0.9583333333333334, "pass_at_1": 0.75, "pass_at_3": 1, "pass_power_3": 0.125, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0.75, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0.9444444444444444, "classification_matrix": { "success": { "terminal_or_invalid": 1, "success": 17 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 1, "invalid_input": 2 } }, "cost_per_correct_task_usd": 0 }, "optimize_thrust_profile": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0.3611111111111111, "normalized_numerical_error": 0, "status_accuracy": 0.9166666666666666, "format_validity": 0.9583333333333334, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0.7083333333333334, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0.9444444444444444, "classification_matrix": { "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 }, "needs_clarification": { "needs_clarification": 3 }, "success": { "success": 17, "terminal_or_invalid": 1 } }, "cost_per_correct_task_usd": 0 }, "trajectory_sensitivity_analysis": { "n": 24, "task_accuracy": 0.4166666666666667, "macro_operation_accuracy": 0.4166666666666667, "field_accuracy": 0.3055555555555556, "normalized_numerical_error": 0.5326547371195739, "status_accuracy": 0.9583333333333334, "format_validity": 0.9583333333333334, "pass_at_1": 0.375, "pass_at_3": 0.5, "pass_power_3": 0.25, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 1.1666666666666667, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "terminal_or_invalid": 1, "out_of_domain": 2 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "genetic_algorithm_optimization": { "n": 24, "task_accuracy": 0.8333333333333334, "macro_operation_accuracy": 0.8333333333333334, "field_accuracy": 0.7777777777777778, "normalized_numerical_error": 0, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.875, "pass_at_3": 1, "pass_power_3": 0.625, "terminal_failure_rate": 0, "mean_runner_calls": 0.875, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0 }, "particle_swarm_optimization": { "n": 24, "task_accuracy": 0.8333333333333334, "macro_operation_accuracy": 0.8333333333333334, "field_accuracy": 0.7777777777777778, "normalized_numerical_error": 0, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.875, "pass_at_3": 1, "pass_power_3": 0.625, "terminal_failure_rate": 0, "mean_runner_calls": 0.9583333333333334, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "out_of_domain": { "out_of_domain": 3 }, "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "porkchop_plot_analysis": { "n": 24, "task_accuracy": 0.5833333333333334, "macro_operation_accuracy": 0.5833333333333334, "field_accuracy": 0.5555555555555556, "normalized_numerical_error": 0, "status_accuracy": 0.7083333333333334, "format_validity": 0.875, "pass_at_1": 0.75, "pass_at_3": 1, "pass_power_3": 0.125, "terminal_failure_rate": 0.125, "mean_runner_calls": 1.2916666666666667, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "needs_clarification": { "needs_clarification": 2, "terminal_or_invalid": 1 }, "out_of_domain": { "invalid_input": 1, "out_of_domain": 2 }, "success": { "success": 13, "out_of_domain": 1, "needs_clarification": 2, "terminal_or_invalid": 2 } }, "cost_per_correct_task_usd": 0 }, "monte_carlo_uncertainty_analysis": { "n": 24, "task_accuracy": 0.5833333333333334, "macro_operation_accuracy": 0.5833333333333334, "field_accuracy": 0.5555555555555556, "normalized_numerical_error": 0, "status_accuracy": 0.9166666666666666, "format_validity": 1, "pass_at_1": 0.625, "pass_at_3": 1, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0.8333333333333334, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "invalid_input": 2, "out_of_domain": 1 }, "success": { "success": 18 } }, "cost_per_correct_task_usd": 0 } }, "nemotron-full-skills": { "search_airports": { "n": 24, "task_accuracy": 0.16666666666666666, "macro_operation_accuracy": 0.16666666666666666, "field_accuracy": 0, "normalized_numerical_error": 1, "status_accuracy": 0.9166666666666666, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0.8333333333333334, "mean_skill_reads": 1.0416666666666667, "runner_invocation_rate_on_required_tasks": null, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "invalid_input": 2, "out_of_domain": 1 } }, "cost_per_correct_task_usd": 0 }, "plan_flight": { "n": 24, "task_accuracy": 0.4166666666666667, "macro_operation_accuracy": 0.4166666666666667, "field_accuracy": 0.3888888888888889, "normalized_numerical_error": 0, "status_accuracy": 0.4583333333333333, "format_validity": 0.5416666666666666, "pass_at_1": 0.375, "pass_at_3": 0.875, "pass_power_3": 0, "terminal_failure_rate": 0.4583333333333333, "mean_runner_calls": 1.9166666666666667, "mean_skill_reads": 1.7083333333333333, "runner_invocation_rate_on_required_tasks": 0.9444444444444444, "classification_matrix": { "success": { "success": 8, "terminal_or_invalid": 10 }, "needs_clarification": { "terminal_or_invalid": 1, "needs_clarification": 2 }, "out_of_domain": { "out_of_domain": 1, "invalid_input": 2 } }, "cost_per_correct_task_usd": 0 }, "plan_multi_leg_flight": { "n": 24, "task_accuracy": 0.625, "macro_operation_accuracy": 0.625, "field_accuracy": 0.6666666666666666, "normalized_numerical_error": 0, "status_accuracy": 0.7083333333333334, "format_validity": 0.8333333333333334, "pass_at_1": 0.75, "pass_at_3": 0.875, "pass_power_3": 0.125, "terminal_failure_rate": 0.16666666666666666, "mean_runner_calls": 1.2083333333333333, "mean_skill_reads": 1.4166666666666667, "runner_invocation_rate_on_required_tasks": 0.9444444444444444, "classification_matrix": { "success": { "success": 14, "terminal_or_invalid": 4 }, "out_of_domain": { "invalid_input": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "calculate_distance": { "n": 24, "task_accuracy": 0.875, "macro_operation_accuracy": 0.875, "field_accuracy": 0.9444444444444444, "normalized_numerical_error": 0, "status_accuracy": 0.875, "format_validity": 0.9166666666666666, "pass_at_1": 0.75, "pass_at_3": 1, "pass_power_3": 0.625, "terminal_failure_rate": 0.08333333333333333, "mean_runner_calls": 0.75, "mean_skill_reads": 1.2916666666666667, "runner_invocation_rate_on_required_tasks": 0.9444444444444444, "classification_matrix": { "out_of_domain": { "invalid_input": 1, "out_of_domain": 2 }, "success": { "success": 17, "terminal_or_invalid": 1 }, "needs_clarification": { "needs_clarification": 2, "terminal_or_invalid": 1 } }, "cost_per_correct_task_usd": 0 }, "get_aircraft_performance": { "n": 24, "task_accuracy": 0.4583333333333333, "macro_operation_accuracy": 0.4583333333333333, "field_accuracy": 0.3611111111111111, "normalized_numerical_error": 93779635.46614471, "status_accuracy": 0.7916666666666666, "format_validity": 0.9583333333333334, "pass_at_1": 0.25, "pass_at_3": 0.75, "pass_power_3": 0.125, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 1.25, "mean_skill_reads": 2.75, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 14, "terminal_or_invalid": 1, "needs_clarification": 3 }, "out_of_domain": { "invalid_input": 1, "out_of_domain": 2 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "get_aircraft_database": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.9583333333333334, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5, "runner_invocation_rate_on_required_tasks": null, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 } }, "cost_per_correct_task_usd": 0 }, "get_system_status": { "n": 24, "task_accuracy": 0.16666666666666666, "macro_operation_accuracy": 0.16666666666666666, "field_accuracy": 0, "normalized_numerical_error": 1, "status_accuracy": 0.8333333333333334, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0.6666666666666666, "mean_skill_reads": 0.9166666666666666, "runner_invocation_rate_on_required_tasks": null, "classification_matrix": { "success": { "success": 16, "out_of_domain": 2 }, "out_of_domain": { "out_of_domain": 1, "invalid_input": 2 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "convert_units": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0.7916666666666666, "mean_skill_reads": 0.9583333333333334, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0 }, "get_atmosphere_profile": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.9583333333333334, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 1.25, "mean_skill_reads": 1.9583333333333333, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 }, "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "wind_model_simple": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.9583333333333334, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 1.0416666666666667, "mean_skill_reads": 1.7083333333333333, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "needs_clarification": { "needs_clarification": 3 }, "success": { "success": 18 }, "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 } }, "cost_per_correct_task_usd": 0 }, "transform_frames": { "n": 24, "task_accuracy": 0.5416666666666666, "macro_operation_accuracy": 0.5416666666666666, "field_accuracy": 0.4444444444444444, "normalized_numerical_error": 0.0016724967281334102, "status_accuracy": 0.9583333333333334, "format_validity": 1, "pass_at_1": 0.5, "pass_at_3": 0.875, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0.8333333333333334, "mean_skill_reads": 1.25, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 }, "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "geodetic_to_ecef": { "n": 24, "task_accuracy": 0.2916666666666667, "macro_operation_accuracy": 0.2916666666666667, "field_accuracy": 0.05555555555555555, "normalized_numerical_error": null, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.375, "pass_at_3": 0.375, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 0.6666666666666666, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "out_of_domain": { "out_of_domain": 3 }, "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "ecef_to_geodetic": { "n": 24, "task_accuracy": 0.9166666666666666, "macro_operation_accuracy": 0.9166666666666666, "field_accuracy": 0.9444444444444444, "normalized_numerical_error": 2.9360628960763706e-7, "status_accuracy": 0.9583333333333334, "format_validity": 1, "pass_at_1": 0.875, "pass_at_3": 1, "pass_power_3": 0.75, "terminal_failure_rate": 0, "mean_runner_calls": 0.7916666666666666, "mean_skill_reads": 0.8333333333333334, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "invalid_input": 1, "out_of_domain": 2 } }, "cost_per_correct_task_usd": 0 }, "wing_vlm_analysis": { "n": 24, "task_accuracy": 0.7083333333333334, "macro_operation_accuracy": 0.7083333333333334, "field_accuracy": 0.8055555555555556, "normalized_numerical_error": 0.2880868179832491, "status_accuracy": 0.9166666666666666, "format_validity": 1, "pass_at_1": 0.625, "pass_at_3": 0.875, "pass_power_3": 0.5, "terminal_failure_rate": 0, "mean_runner_calls": 1.125, "mean_skill_reads": 1.3333333333333333, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "invalid_input": 2, "out_of_domain": 1 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "airfoil_polar_analysis": { "n": 24, "task_accuracy": 0.3333333333333333, "macro_operation_accuracy": 0.3333333333333333, "field_accuracy": 0.2222222222222222, "normalized_numerical_error": 0, "status_accuracy": 0.875, "format_validity": 0.9583333333333334, "pass_at_1": 0.25, "pass_at_3": 0.625, "pass_power_3": 0.125, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0.7083333333333334, "mean_skill_reads": 0.4583333333333333, "runner_invocation_rate_on_required_tasks": 0.9444444444444444, "classification_matrix": { "success": { "success": 17, "terminal_or_invalid": 1 }, "out_of_domain": { "invalid_input": 2, "out_of_domain": 1 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "calculate_stability_derivatives": { "n": 24, "task_accuracy": 0.6666666666666666, "macro_operation_accuracy": 0.6666666666666666, "field_accuracy": 0.7222222222222222, "normalized_numerical_error": 0, "status_accuracy": 0.875, "format_validity": 0.9583333333333334, "pass_at_1": 0.75, "pass_at_3": 0.875, "pass_power_3": 0.375, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0.875, "mean_skill_reads": 1.375, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "invalid_input": 2, "terminal_or_invalid": 1 } }, "cost_per_correct_task_usd": 0 }, "get_airfoil_database": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": 1, "status_accuracy": 0.9583333333333334, "format_validity": 0.9583333333333334, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0.7083333333333334, "mean_skill_reads": 0.7083333333333334, "runner_invocation_rate_on_required_tasks": null, "classification_matrix": { "success": { "success": 17, "terminal_or_invalid": 1 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0 }, "propeller_bemt_analysis": { "n": 24, "task_accuracy": 0.9583333333333334, "macro_operation_accuracy": 0.9583333333333334, "field_accuracy": 1, "normalized_numerical_error": 0.009221883043975615, "status_accuracy": 0.9583333333333334, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 0.875, "terminal_failure_rate": 0, "mean_runner_calls": 1.2083333333333333, "mean_skill_reads": 1.2083333333333333, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 } }, "cost_per_correct_task_usd": 0 }, "uav_energy_estimate": { "n": 24, "task_accuracy": 0.5, "macro_operation_accuracy": 0.5, "field_accuracy": 0.5, "normalized_numerical_error": 171.97000000000006, "status_accuracy": 0.875, "format_validity": 1, "pass_at_1": 0.5, "pass_at_3": 0.875, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 1.1666666666666667, "mean_skill_reads": 1.75, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "invalid_input": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "get_propeller_database": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0.5, "mean_skill_reads": 1.5416666666666667, "runner_invocation_rate_on_required_tasks": null, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "rocket_3dof_trajectory": { "n": 24, "task_accuracy": 0.9583333333333334, "macro_operation_accuracy": 0.9583333333333334, "field_accuracy": 0.9444444444444444, "normalized_numerical_error": 0.3251399712963241, "status_accuracy": 0.9583333333333334, "format_validity": 0.9583333333333334, "pass_at_1": 0.875, "pass_at_3": 1, "pass_power_3": 0.875, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 1.2083333333333333, "mean_skill_reads": 1.5833333333333333, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 17, "terminal_or_invalid": 1 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "estimate_rocket_sizing": { "n": 24, "task_accuracy": 0.625, "macro_operation_accuracy": 0.625, "field_accuracy": 0.6944444444444444, "normalized_numerical_error": 0, "status_accuracy": 0.8333333333333334, "format_validity": 0.9166666666666666, "pass_at_1": 0.5, "pass_at_3": 1, "pass_power_3": 0.25, "terminal_failure_rate": 0.08333333333333333, "mean_runner_calls": 0.7083333333333334, "mean_skill_reads": 1.2916666666666667, "runner_invocation_rate_on_required_tasks": 0.9444444444444444, "classification_matrix": { "success": { "success": 17, "terminal_or_invalid": 1 }, "out_of_domain": { "out_of_domain": 1, "invalid_input": 2 }, "needs_clarification": { "terminal_or_invalid": 1, "needs_clarification": 2 } }, "cost_per_correct_task_usd": 0 }, "optimize_launch_angle": { "n": 24, "task_accuracy": 0.2916666666666667, "macro_operation_accuracy": 0.2916666666666667, "field_accuracy": 0.4444444444444444, "normalized_numerical_error": 0.018143825023719296, "status_accuracy": 0.9583333333333334, "format_validity": 0.9583333333333334, "pass_at_1": 0.25, "pass_at_3": 0.375, "pass_power_3": 0.25, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0.875, "mean_skill_reads": 1.2916666666666667, "runner_invocation_rate_on_required_tasks": 0.9444444444444444, "classification_matrix": { "needs_clarification": { "needs_clarification": 3 }, "success": { "success": 17, "terminal_or_invalid": 1 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0 }, "elements_to_state_vector": { "n": 24, "task_accuracy": 0.6666666666666666, "macro_operation_accuracy": 0.6666666666666666, "field_accuracy": 0.5555555555555556, "normalized_numerical_error": null, "status_accuracy": 0.9583333333333334, "format_validity": 0.9583333333333334, "pass_at_1": 0.5, "pass_at_3": 0.875, "pass_power_3": 0.5, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 1.0833333333333333, "mean_skill_reads": 1.3333333333333333, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "needs_clarification": { "needs_clarification": 3 }, "success": { "success": 17, "terminal_or_invalid": 1 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0 }, "state_vector_to_elements": { "n": 24, "task_accuracy": 0.9166666666666666, "macro_operation_accuracy": 0.9166666666666666, "field_accuracy": 0.9722222222222222, "normalized_numerical_error": 0, "status_accuracy": 0.9583333333333334, "format_validity": 1, "pass_at_1": 0.875, "pass_at_3": 1, "pass_power_3": 0.75, "terminal_failure_rate": 0, "mean_runner_calls": 0.9166666666666666, "mean_skill_reads": 0.9583333333333334, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 } }, "cost_per_correct_task_usd": 0 }, "propagate_orbit_j2": { "n": 24, "task_accuracy": 0.5, "macro_operation_accuracy": 0.5, "field_accuracy": 0.3888888888888889, "normalized_numerical_error": null, "status_accuracy": 0.9583333333333334, "format_validity": 1, "pass_at_1": 0.375, "pass_at_3": 0.875, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0.7916666666666666, "mean_skill_reads": 1.4583333333333333, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "invalid_input": 1, "out_of_domain": 2 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "calculate_ground_track": { "n": 24, "task_accuracy": 0.4166666666666667, "macro_operation_accuracy": 0.4166666666666667, "field_accuracy": 0.2777777777777778, "normalized_numerical_error": 8.91469444444384, "status_accuracy": 0.8333333333333334, "format_validity": 1, "pass_at_1": 0.625, "pass_at_3": 0.875, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0.9583333333333334, "mean_skill_reads": 1.8333333333333333, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 15, "needs_clarification": 3 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 } }, "cost_per_correct_task_usd": 0 }, "hohmann_transfer": { "n": 24, "task_accuracy": 0.7083333333333334, "macro_operation_accuracy": 0.7083333333333334, "field_accuracy": 0.7777777777777778, "normalized_numerical_error": 0, "status_accuracy": 0.9166666666666666, "format_validity": 0.9583333333333334, "pass_at_1": 0.75, "pass_at_3": 1, "pass_power_3": 0.125, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0.75, "mean_skill_reads": 1.125, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 2, "terminal_or_invalid": 1 }, "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 } }, "cost_per_correct_task_usd": 0 }, "orbital_rendezvous_planning": { "n": 24, "task_accuracy": 0.375, "macro_operation_accuracy": 0.375, "field_accuracy": 0.19444444444444445, "normalized_numerical_error": 36.93333333345698, "status_accuracy": 0.9166666666666666, "format_validity": 0.9166666666666666, "pass_at_1": 0.375, "pass_at_3": 0.625, "pass_power_3": 0.25, "terminal_failure_rate": 0.08333333333333333, "mean_runner_calls": 0.9166666666666666, "mean_skill_reads": 1.4166666666666667, "runner_invocation_rate_on_required_tasks": 0.9444444444444444, "classification_matrix": { "success": { "success": 16, "terminal_or_invalid": 2 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0 }, "lambert_problem_solver": { "n": 24, "task_accuracy": 0.16666666666666666, "macro_operation_accuracy": 0.16666666666666666, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.9166666666666666, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0.875, "mean_skill_reads": 0.9166666666666666, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "out_of_domain": { "invalid_input": 2, "out_of_domain": 1 }, "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "kalman_filter_state_estimation": { "n": 24, "task_accuracy": 0.2916666666666667, "macro_operation_accuracy": 0.2916666666666667, "field_accuracy": 0.1111111111111111, "normalized_numerical_error": null, "status_accuracy": 0.9583333333333334, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.5, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0.9583333333333334, "mean_skill_reads": 0.9583333333333334, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "out_of_domain": { "invalid_input": 1, "out_of_domain": 2 }, "needs_clarification": { "needs_clarification": 3 }, "success": { "success": 18 } }, "cost_per_correct_task_usd": 0 }, "lqr_controller_design": { "n": 24, "task_accuracy": 0.4583333333333333, "macro_operation_accuracy": 0.4583333333333333, "field_accuracy": 0.2777777777777778, "normalized_numerical_error": 0.0031763371954795136, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.625, "pass_at_3": 0.625, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 1.0416666666666667, "mean_skill_reads": 0.9166666666666666, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "density_altitude_calculator": { "n": 24, "task_accuracy": 0.6666666666666666, "macro_operation_accuracy": 0.6666666666666666, "field_accuracy": 0.6666666666666666, "normalized_numerical_error": 0, "status_accuracy": 0.9166666666666666, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 0.375, "terminal_failure_rate": 0, "mean_runner_calls": 0.875, "mean_skill_reads": 1.5, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "out_of_domain": { "out_of_domain": 1, "invalid_input": 2 }, "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "true_airspeed_converter": { "n": 24, "task_accuracy": 0.9166666666666666, "macro_operation_accuracy": 0.9166666666666666, "field_accuracy": 0.9444444444444444, "normalized_numerical_error": 0, "status_accuracy": 0.9166666666666666, "format_validity": 0.9583333333333334, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 0.75, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0.9166666666666666, "mean_skill_reads": 1.2916666666666667, "runner_invocation_rate_on_required_tasks": 0.9444444444444444, "classification_matrix": { "needs_clarification": { "needs_clarification": 3 }, "success": { "success": 17, "terminal_or_invalid": 1 }, "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 } }, "cost_per_correct_task_usd": 0 }, "stall_speed_calculator": { "n": 24, "task_accuracy": 0.7916666666666666, "macro_operation_accuracy": 0.7916666666666666, "field_accuracy": 0.8333333333333334, "normalized_numerical_error": 0.6875000000000631, "status_accuracy": 0.9166666666666666, "format_validity": 1, "pass_at_1": 0.75, "pass_at_3": 1, "pass_power_3": 0.5, "terminal_failure_rate": 0, "mean_runner_calls": 0.9583333333333334, "mean_skill_reads": 1, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "invalid_input": 2, "out_of_domain": 1 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "weight_and_balance": { "n": 24, "task_accuracy": 0.5416666666666666, "macro_operation_accuracy": 0.5416666666666666, "field_accuracy": 0.75, "normalized_numerical_error": 25.588235294115485, "status_accuracy": 0.8333333333333334, "format_validity": 0.9166666666666666, "pass_at_1": 0.625, "pass_at_3": 0.75, "pass_power_3": 0.375, "terminal_failure_rate": 0.08333333333333333, "mean_runner_calls": 0.9583333333333334, "mean_skill_reads": 0.4583333333333333, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 17, "terminal_or_invalid": 1 }, "out_of_domain": { "terminal_or_invalid": 1, "invalid_input": 2 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "takeoff_performance": { "n": 24, "task_accuracy": 0.5416666666666666, "macro_operation_accuracy": 0.5416666666666666, "field_accuracy": 0.5, "normalized_numerical_error": 0, "status_accuracy": 0.9166666666666666, "format_validity": 1, "pass_at_1": 0.625, "pass_at_3": 0.875, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0.9583333333333334, "mean_skill_reads": 1.25, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "out_of_domain": 1, "invalid_input": 2 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "landing_performance": { "n": 24, "task_accuracy": 0.6666666666666666, "macro_operation_accuracy": 0.6666666666666666, "field_accuracy": 0.6111111111111112, "normalized_numerical_error": 0, "status_accuracy": 0.9583333333333334, "format_validity": 1, "pass_at_1": 0.75, "pass_at_3": 1, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 1.0416666666666667, "mean_skill_reads": 1.4583333333333333, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 }, "needs_clarification": { "needs_clarification": 3 }, "success": { "success": 18 } }, "cost_per_correct_task_usd": 0 }, "fuel_reserve_calculator": { "n": 24, "task_accuracy": 0.6666666666666666, "macro_operation_accuracy": 0.6666666666666666, "field_accuracy": 0.6111111111111112, "normalized_numerical_error": 0, "status_accuracy": 0.9583333333333334, "format_validity": 0.9583333333333334, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 0.25, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0.7916666666666666, "mean_skill_reads": 0.5833333333333334, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 2, "terminal_or_invalid": 1 } }, "cost_per_correct_task_usd": 0 }, "optimize_thrust_profile": { "n": 24, "task_accuracy": 0.16666666666666666, "macro_operation_accuracy": 0.16666666666666666, "field_accuracy": 0.027777777777777776, "normalized_numerical_error": 131333.33333333334, "status_accuracy": 0.875, "format_validity": 0.9583333333333334, "pass_at_1": 0.125, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 1.2083333333333333, "mean_skill_reads": 1.4583333333333333, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "out_of_domain": { "invalid_input": 2, "out_of_domain": 1 }, "needs_clarification": { "needs_clarification": 3 }, "success": { "success": 17, "terminal_or_invalid": 1 } }, "cost_per_correct_task_usd": 0 }, "trajectory_sensitivity_analysis": { "n": 24, "task_accuracy": 0.4583333333333333, "macro_operation_accuracy": 0.4583333333333333, "field_accuracy": 0.2777777777777778, "normalized_numerical_error": 0.5476971625478387, "status_accuracy": 0.8333333333333334, "format_validity": 0.8333333333333334, "pass_at_1": 0.5, "pass_at_3": 0.625, "pass_power_3": 0.25, "terminal_failure_rate": 0.16666666666666666, "mean_runner_calls": 1.875, "mean_skill_reads": 1.3333333333333333, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 14, "terminal_or_invalid": 4 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "genetic_algorithm_optimization": { "n": 24, "task_accuracy": 0.875, "macro_operation_accuracy": 0.875, "field_accuracy": 0.8333333333333334, "normalized_numerical_error": 0, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 0.625, "terminal_failure_rate": 0, "mean_runner_calls": 0.9166666666666666, "mean_skill_reads": 1.0416666666666667, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0 }, "particle_swarm_optimization": { "n": 24, "task_accuracy": 0.8333333333333334, "macro_operation_accuracy": 0.8333333333333334, "field_accuracy": 0.8333333333333334, "normalized_numerical_error": 0, "status_accuracy": 0.9166666666666666, "format_validity": 0.9583333333333334, "pass_at_1": 0.75, "pass_at_3": 1, "pass_power_3": 0.625, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 0.8333333333333334, "mean_skill_reads": 0.8333333333333334, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "out_of_domain": { "invalid_input": 1, "out_of_domain": 2 }, "success": { "success": 17, "terminal_or_invalid": 1 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0 }, "porkchop_plot_analysis": { "n": 24, "task_accuracy": 0.2916666666666667, "macro_operation_accuracy": 0.2916666666666667, "field_accuracy": 0.16666666666666666, "normalized_numerical_error": 10.416666666666668, "status_accuracy": 0.375, "format_validity": 0.9583333333333334, "pass_at_1": 0.375, "pass_at_3": 0.5, "pass_power_3": 0, "terminal_failure_rate": 0.041666666666666664, "mean_runner_calls": 1.0416666666666667, "mean_skill_reads": 2.3333333333333335, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "needs_clarification": { "needs_clarification": 2, "terminal_or_invalid": 1 }, "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 }, "success": { "success": 5, "needs_clarification": 9, "out_of_domain": 4 } }, "cost_per_correct_task_usd": 0 }, "monte_carlo_uncertainty_analysis": { "n": 24, "task_accuracy": 0.5833333333333334, "macro_operation_accuracy": 0.5833333333333334, "field_accuracy": 0.6388888888888888, "normalized_numerical_error": 21452596.812078986, "status_accuracy": 0.9583333333333334, "format_validity": 1, "pass_at_1": 0.75, "pass_at_3": 0.875, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0.8333333333333334, "mean_skill_reads": 1.2083333333333333, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 }, "success": { "success": 18 } }, "cost_per_correct_task_usd": 0 } }, "terra-base": { "search_airports": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": 1, "status_accuracy": 0.375, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": null, "classification_matrix": { "success": { "needs_clarification": 15, "success": 3 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0.011553750000000002 }, "plan_flight": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": 13.862500000002953, "status_accuracy": 0.5833333333333334, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 8, "needs_clarification": 10 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0.03891375 }, "plan_multi_leg_flight": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.25, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 18 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.015288750000000002 }, "calculate_distance": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0.16666666666666666, "normalized_numerical_error": 831.602941176467, "status_accuracy": 0.7916666666666666, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "out_of_domain": { "out_of_domain": 3 }, "success": { "needs_clarification": 5, "success": 13 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.11497875000000002 }, "get_aircraft_performance": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.25, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 18 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.01087 }, "get_aircraft_database": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.25, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": null, "classification_matrix": { "success": { "needs_clarification": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0.010276250000000002 }, "get_system_status": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.25, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": null, "classification_matrix": { "success": { "needs_clarification": 18 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.012063750000000003 }, "convert_units": { "n": 24, "task_accuracy": 1, "macro_operation_accuracy": 1, "field_accuracy": 1, "normalized_numerical_error": 0, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 1, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0.002036875 }, "get_atmosphere_profile": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "out_of_domain": { "out_of_domain": 3 }, "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.02422625 }, "wind_model_simple": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.25, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "needs_clarification": { "needs_clarification": 3 }, "success": { "needs_clarification": 18 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0.0161525 }, "transform_frames": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.25, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "out_of_domain": { "out_of_domain": 3 }, "success": { "needs_clarification": 18 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.013936249999999997 }, "geodetic_to_ecef": { "n": 24, "task_accuracy": 0.75, "macro_operation_accuracy": 0.75, "field_accuracy": 0.6666666666666666, "normalized_numerical_error": null, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.625, "pass_at_3": 1, "pass_power_3": 0.5, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "out_of_domain": { "out_of_domain": 3 }, "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.02703375 }, "ecef_to_geodetic": { "n": 24, "task_accuracy": 0.2916666666666667, "macro_operation_accuracy": 0.2916666666666667, "field_accuracy": 0.05555555555555555, "normalized_numerical_error": 62165.35130546797, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.375, "pass_at_3": 0.375, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0.1590632142857143 }, "wing_vlm_analysis": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.25, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 18 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.02011125 }, "airfoil_polar_analysis": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.020301250000000003 }, "calculate_stability_derivatives": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0.1111111111111111, "normalized_numerical_error": 797.1543482381709, "status_accuracy": 0.9166666666666666, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 16, "needs_clarification": 2 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0.05602125 }, "get_airfoil_database": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.25, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": null, "classification_matrix": { "success": { "needs_clarification": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0.01153125 }, "propeller_bemt_analysis": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.25, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0.016916250000000004 }, "uav_energy_estimate": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.25, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 18 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.014691249999999998 }, "get_propeller_database": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.25, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": null, "classification_matrix": { "success": { "needs_clarification": 18 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.011113749999999999 }, "rocket_3dof_trajectory": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.25, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 18 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.014538749999999998 }, "estimate_rocket_sizing": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": 12499.375, "status_accuracy": 0.25, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 18 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.020811249999999996 }, "optimize_launch_angle": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.25, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "needs_clarification": { "needs_clarification": 3 }, "success": { "needs_clarification": 18 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0.01666625 }, "elements_to_state_vector": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0.05555555555555555, "normalized_numerical_error": null, "status_accuracy": 0.25, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "needs_clarification": { "needs_clarification": 3 }, "success": { "needs_clarification": 18 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0.029768749999999997 }, "state_vector_to_elements": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.25, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0.01022125 }, "propagate_orbit_j2": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.25, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 18 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.012503750000000003 }, "calculate_ground_track": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.25, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0.02714875000000001 }, "hohmann_transfer": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.25, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0.010971249999999997 }, "orbital_rendezvous_planning": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0.1111111111111111, "normalized_numerical_error": 0.00999999999995449, "status_accuracy": 0.25, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 16, "invalid_input": 1, "out_of_domain": 1 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0.03010624999999999 }, "lambert_problem_solver": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.9166666666666666, "format_validity": 0.9166666666666666, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0.08333333333333333, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "out_of_domain": { "out_of_domain": 3 }, "success": { "success": 16, "terminal_or_invalid": 2 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.22413625000000004 }, "kalman_filter_state_estimation": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.4166666666666667, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 }, "success": { "needs_clarification": 14, "success": 4 } }, "cost_per_correct_task_usd": 0.04211500000000001 }, "lqr_controller_design": { "n": 24, "task_accuracy": 0.7916666666666666, "macro_operation_accuracy": 0.7916666666666666, "field_accuracy": 0.7222222222222222, "normalized_numerical_error": 0.0003444981048770299, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.75, "pass_at_3": 1, "pass_power_3": 0.375, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.005817236842105262 }, "density_altitude_calculator": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": 115.87335972083646, "status_accuracy": 0.7916666666666666, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "out_of_domain": { "out_of_domain": 3 }, "success": { "success": 13, "invalid_input": 5 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.06608875 }, "true_airspeed_converter": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.25, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "needs_clarification": { "needs_clarification": 3 }, "success": { "needs_clarification": 18 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0.013443749999999997 }, "stall_speed_calculator": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.20833333333333334, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 18 }, "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.0167295 }, "weight_and_balance": { "n": 24, "task_accuracy": 0.4583333333333333, "macro_operation_accuracy": 0.4583333333333333, "field_accuracy": 0.6666666666666666, "normalized_numerical_error": 1.906445833323013, "status_accuracy": 0.9583333333333334, "format_validity": 1, "pass_at_1": 0.5, "pass_at_3": 0.5, "pass_power_3": 0.375, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.008249318181818182 }, "takeoff_performance": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": 60.99999999999999, "status_accuracy": 0.2916666666666667, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 17, "success": 1 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.01972375 }, "landing_performance": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.25, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 }, "success": { "needs_clarification": 18 } }, "cost_per_correct_task_usd": 0.015216249999999999 }, "fuel_reserve_calculator": { "n": 24, "task_accuracy": 0.9583333333333334, "macro_operation_accuracy": 0.9583333333333334, "field_accuracy": 0.9444444444444444, "normalized_numerical_error": 83.33333333333333, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 0.875, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0.004015108695652173 }, "optimize_thrust_profile": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0.3333333333333333, "normalized_numerical_error": 0, "status_accuracy": 0.25, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 }, "success": { "needs_clarification": 17, "invalid_input": 1 } }, "cost_per_correct_task_usd": 0.025278749999999992 }, "trajectory_sensitivity_analysis": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.25, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 18 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.014473749999999999 }, "genetic_algorithm_optimization": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.25, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0.014216250000000001 }, "particle_swarm_optimization": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.25, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "out_of_domain": { "out_of_domain": 3 }, "success": { "needs_clarification": 18 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.015356250000000002 }, "porkchop_plot_analysis": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.25, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 }, "success": { "needs_clarification": 18 } }, "cost_per_correct_task_usd": 0.018616250000000004 }, "monte_carlo_uncertainty_analysis": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.25, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 }, "success": { "needs_clarification": 18 } }, "cost_per_correct_task_usd": 0.01608625 } }, "terra-full-skills": { "search_airports": { "n": 24, "task_accuracy": 0.625, "macro_operation_accuracy": 0.625, "field_accuracy": 0.5, "normalized_numerical_error": 0.5, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.625, "pass_at_3": 0.625, "pass_power_3": 0.625, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 2.25, "runner_invocation_rate_on_required_tasks": null, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0.011541725 }, "plan_flight": { "n": 24, "task_accuracy": 1, "macro_operation_accuracy": 1, "field_accuracy": 1, "normalized_numerical_error": 0, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 1, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0.006787552083333333 }, "plan_multi_leg_flight": { "n": 24, "task_accuracy": 1, "macro_operation_accuracy": 1, "field_accuracy": 1, "normalized_numerical_error": 0, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 1, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.0063882760416666675 }, "calculate_distance": { "n": 24, "task_accuracy": 1, "macro_operation_accuracy": 1, "field_accuracy": 1, "normalized_numerical_error": 0, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 1, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.4166666666666667, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "out_of_domain": { "out_of_domain": 3 }, "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.006450864583333331 }, "get_aircraft_performance": { "n": 24, "task_accuracy": 1, "macro_operation_accuracy": 1, "field_accuracy": 1, "normalized_numerical_error": 0, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 1, "terminal_failure_rate": 0, "mean_runner_calls": 0.8333333333333334, "mean_skill_reads": 1.4583333333333333, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.006759151041666667 }, "get_aircraft_database": { "n": 24, "task_accuracy": 1, "macro_operation_accuracy": 1, "field_accuracy": 1, "normalized_numerical_error": 0, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 1, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 2.25, "runner_invocation_rate_on_required_tasks": null, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0.007673421875 }, "get_system_status": { "n": 24, "task_accuracy": 1, "macro_operation_accuracy": 1, "field_accuracy": 1, "normalized_numerical_error": 0, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 1, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 2.125, "runner_invocation_rate_on_required_tasks": null, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.006820505208333334 }, "convert_units": { "n": 24, "task_accuracy": 1, "macro_operation_accuracy": 1, "field_accuracy": 1, "normalized_numerical_error": 0, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 1, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.0416666666666667, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0.0050776822916666665 }, "get_atmosphere_profile": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.4166666666666667, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "out_of_domain": { "out_of_domain": 3 }, "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.023872875 }, "wind_model_simple": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "needs_clarification": { "needs_clarification": 3 }, "success": { "success": 18 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0.02557841666666667 }, "transform_frames": { "n": 24, "task_accuracy": 0.9166666666666666, "macro_operation_accuracy": 0.9166666666666666, "field_accuracy": 0.8888888888888888, "normalized_numerical_error": 0, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 0.75, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "out_of_domain": { "out_of_domain": 3 }, "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.007248749999999999 }, "geodetic_to_ecef": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.2916666666666667, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "out_of_domain": { "out_of_domain": 3 }, "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.024419229166666664 }, "ecef_to_geodetic": { "n": 24, "task_accuracy": 1, "macro_operation_accuracy": 1, "field_accuracy": 1, "normalized_numerical_error": 2.7729482907387944e-7, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 1, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.3333333333333333, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0.0063157291666666665 }, "wing_vlm_analysis": { "n": 24, "task_accuracy": 1, "macro_operation_accuracy": 1, "field_accuracy": 1, "normalized_numerical_error": 0.23578551788585353, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 1, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.009093979166666665 }, "airfoil_polar_analysis": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.0325398125 }, "calculate_stability_derivatives": { "n": 24, "task_accuracy": 0.5, "macro_operation_accuracy": 0.5, "field_accuracy": 0.3333333333333333, "normalized_numerical_error": 0, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.5, "pass_at_3": 0.625, "pass_power_3": 0.375, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0.014631572916666667 }, "get_airfoil_database": { "n": 24, "task_accuracy": 0.375, "macro_operation_accuracy": 0.375, "field_accuracy": 0.16666666666666666, "normalized_numerical_error": 0, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.375, "pass_at_3": 0.375, "pass_power_3": 0.375, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 2.25, "runner_invocation_rate_on_required_tasks": null, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0.021677013888888888 }, "propeller_bemt_analysis": { "n": 24, "task_accuracy": 1, "macro_operation_accuracy": 1, "field_accuracy": 1, "normalized_numerical_error": 9.930990045122858e-7, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 1, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0.010980229166666666 }, "uav_energy_estimate": { "n": 24, "task_accuracy": 0.9166666666666666, "macro_operation_accuracy": 0.9166666666666666, "field_accuracy": 0.8888888888888888, "normalized_numerical_error": 0, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.875, "pass_at_3": 1, "pass_power_3": 0.875, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.7916666666666667, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.01066740909090909 }, "get_propeller_database": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 2.25, "runner_invocation_rate_on_required_tasks": null, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.02748747916666666 }, "rocket_3dof_trajectory": { "n": 24, "task_accuracy": 1, "macro_operation_accuracy": 1, "field_accuracy": 1, "normalized_numerical_error": 0.32516290954696114, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 1, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.009355619791666664 }, "estimate_rocket_sizing": { "n": 24, "task_accuracy": 0.9583333333333334, "macro_operation_accuracy": 0.9583333333333334, "field_accuracy": 0.9444444444444444, "normalized_numerical_error": 0, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 0.875, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.006965635869565218 }, "optimize_launch_angle": { "n": 24, "task_accuracy": 0.2916666666666667, "macro_operation_accuracy": 0.2916666666666667, "field_accuracy": 0.5277777777777778, "normalized_numerical_error": 0.015129268329018638, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.375, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "needs_clarification": { "needs_clarification": 3 }, "success": { "success": 18 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0.02929966071428572 }, "elements_to_state_vector": { "n": 24, "task_accuracy": 1, "macro_operation_accuracy": 1, "field_accuracy": 1, "normalized_numerical_error": null, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 1, "terminal_failure_rate": 0, "mean_runner_calls": 0.9583333333333334, "mean_skill_reads": 1.5833333333333333, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "needs_clarification": { "needs_clarification": 3 }, "success": { "success": 18 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0.008444395833333335 }, "state_vector_to_elements": { "n": 24, "task_accuracy": 0.9583333333333334, "macro_operation_accuracy": 0.9583333333333334, "field_accuracy": 0.9722222222222222, "normalized_numerical_error": 0, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 0.875, "terminal_failure_rate": 0, "mean_runner_calls": 0.7916666666666666, "mean_skill_reads": 1.4583333333333333, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0.00745911956521739 }, "propagate_orbit_j2": { "n": 24, "task_accuracy": 1, "macro_operation_accuracy": 1, "field_accuracy": 1, "normalized_numerical_error": null, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 1, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.00797713541666667 }, "calculate_ground_track": { "n": 24, "task_accuracy": 1, "macro_operation_accuracy": 1, "field_accuracy": 1, "normalized_numerical_error": 0.00048611111082132023, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 1, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0.007372593750000001 }, "hohmann_transfer": { "n": 24, "task_accuracy": 1, "macro_operation_accuracy": 1, "field_accuracy": 1, "normalized_numerical_error": 0, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 1, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0.006808838541666665 }, "orbital_rendezvous_planning": { "n": 24, "task_accuracy": 1, "macro_operation_accuracy": 1, "field_accuracy": 1, "normalized_numerical_error": 0, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 1, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0.008183546875000001 }, "lambert_problem_solver": { "n": 24, "task_accuracy": 1, "macro_operation_accuracy": 1, "field_accuracy": 1, "normalized_numerical_error": null, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 1, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.3333333333333333, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "out_of_domain": { "out_of_domain": 3 }, "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.006956411458333335 }, "kalman_filter_state_estimation": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 }, "success": { "success": 18 } }, "cost_per_correct_task_usd": 0.0321331875 }, "lqr_controller_design": { "n": 24, "task_accuracy": 0.6666666666666666, "macro_operation_accuracy": 0.6666666666666666, "field_accuracy": 0.5555555555555556, "normalized_numerical_error": 0.0039243354500816, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.625, "pass_at_3": 0.75, "pass_power_3": 0.625, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.009851367187500001 }, "density_altitude_calculator": { "n": 24, "task_accuracy": 1, "macro_operation_accuracy": 1, "field_accuracy": 1, "normalized_numerical_error": 0, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 1, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.4166666666666667, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "out_of_domain": { "out_of_domain": 3 }, "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.006638968749999999 }, "true_airspeed_converter": { "n": 24, "task_accuracy": 1, "macro_operation_accuracy": 1, "field_accuracy": 1, "normalized_numerical_error": 0, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 1, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "needs_clarification": { "needs_clarification": 3 }, "success": { "success": 18 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0.006516338541666667 }, "stall_speed_calculator": { "n": 24, "task_accuracy": 1, "macro_operation_accuracy": 1, "field_accuracy": 1, "normalized_numerical_error": 0, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 1, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.006647557291666668 }, "weight_and_balance": { "n": 24, "task_accuracy": 1, "macro_operation_accuracy": 1, "field_accuracy": 1, "normalized_numerical_error": 0, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 1, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.007189515624999999 }, "takeoff_performance": { "n": 24, "task_accuracy": 1, "macro_operation_accuracy": 1, "field_accuracy": 1, "normalized_numerical_error": 0, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 1, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.007035755208333334 }, "landing_performance": { "n": 24, "task_accuracy": 1, "macro_operation_accuracy": 1, "field_accuracy": 1, "normalized_numerical_error": 0, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 1, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 }, "success": { "success": 18 } }, "cost_per_correct_task_usd": 0.007174151041666668 }, "fuel_reserve_calculator": { "n": 24, "task_accuracy": 0.9166666666666666, "macro_operation_accuracy": 0.9166666666666666, "field_accuracy": 0.8888888888888888, "normalized_numerical_error": 0, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 0.75, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.4583333333333333, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0.008206420454545454 }, "optimize_thrust_profile": { "n": 24, "task_accuracy": 1, "macro_operation_accuracy": 1, "field_accuracy": 1, "normalized_numerical_error": 0.092951123526404, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 1, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 2.0833333333333335, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 }, "success": { "success": 18 } }, "cost_per_correct_task_usd": 0.01220703125 }, "trajectory_sensitivity_analysis": { "n": 24, "task_accuracy": 1, "macro_operation_accuracy": 1, "field_accuracy": 1, "normalized_numerical_error": 0.5275664594203842, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 1, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.009157906249999999 }, "genetic_algorithm_optimization": { "n": 24, "task_accuracy": 1, "macro_operation_accuracy": 1, "field_accuracy": 1, "normalized_numerical_error": 0, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 1, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0.007198322916666665 }, "particle_swarm_optimization": { "n": 24, "task_accuracy": 1, "macro_operation_accuracy": 1, "field_accuracy": 1, "normalized_numerical_error": 0, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 1, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "out_of_domain": { "out_of_domain": 3 }, "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.007082536458333333 }, "porkchop_plot_analysis": { "n": 24, "task_accuracy": 1, "macro_operation_accuracy": 1, "field_accuracy": 1, "normalized_numerical_error": 0, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 1, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 }, "success": { "success": 18 } }, "cost_per_correct_task_usd": 0.009520677083333333 }, "monte_carlo_uncertainty_analysis": { "n": 24, "task_accuracy": 1, "macro_operation_accuracy": 1, "field_accuracy": 1, "normalized_numerical_error": 0, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 1, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 }, "success": { "success": 18 } }, "cost_per_correct_task_usd": 0.007071359374999998 } }, "sol-base": { "search_airports": { "n": 24, "task_accuracy": 0.125, "macro_operation_accuracy": 0.125, "field_accuracy": 0, "normalized_numerical_error": 1, "status_accuracy": 0.2916666666666667, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.125, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": null, "classification_matrix": { "success": { "success": 4, "needs_clarification": 14 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "invalid_input": 3 } }, "cost_per_correct_task_usd": 0.060885000000000015 }, "plan_flight": { "n": 24, "task_accuracy": 0.125, "macro_operation_accuracy": 0.125, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.125, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.125, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "invalid_input": 3 } }, "cost_per_correct_task_usd": 0.06918500000000001 }, "plan_multi_leg_flight": { "n": 24, "task_accuracy": 0.16666666666666666, "macro_operation_accuracy": 0.16666666666666666, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.16666666666666666, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 18 }, "out_of_domain": { "invalid_input": 2, "out_of_domain": 1 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.036438750000000006 }, "calculate_distance": { "n": 24, "task_accuracy": 0.125, "macro_operation_accuracy": 0.125, "field_accuracy": 0.08333333333333333, "normalized_numerical_error": 3.227272727272957, "status_accuracy": 0.16666666666666666, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.125, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "out_of_domain": { "invalid_input": 3 }, "success": { "needs_clarification": 17, "success": 1 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.313705 }, "get_aircraft_performance": { "n": 24, "task_accuracy": 0.125, "macro_operation_accuracy": 0.125, "field_accuracy": 0, "normalized_numerical_error": 33576024.38166605, "status_accuracy": 0.3333333333333333, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.125, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 13, "success": 5 }, "out_of_domain": { "invalid_input": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.07988 }, "get_aircraft_database": { "n": 24, "task_accuracy": 0.125, "macro_operation_accuracy": 0.125, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.125, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.125, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": null, "classification_matrix": { "success": { "needs_clarification": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "invalid_input": 3 } }, "cost_per_correct_task_usd": 0.04397499999999999 }, "get_system_status": { "n": 24, "task_accuracy": 0.125, "macro_operation_accuracy": 0.125, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.125, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.125, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": null, "classification_matrix": { "success": { "needs_clarification": 18 }, "out_of_domain": { "invalid_input": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.056775 }, "convert_units": { "n": 24, "task_accuracy": 0.7916666666666666, "macro_operation_accuracy": 0.7916666666666666, "field_accuracy": 0.7777777777777778, "normalized_numerical_error": 0, "status_accuracy": 0.9583333333333334, "format_validity": 1, "pass_at_1": 0.75, "pass_at_3": 0.875, "pass_power_3": 0.625, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 } }, "cost_per_correct_task_usd": 0.004910526315789474 }, "get_atmosphere_profile": { "n": 24, "task_accuracy": 0.125, "macro_operation_accuracy": 0.125, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.875, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.125, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "out_of_domain": { "invalid_input": 3 }, "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.12774500000000003 }, "wind_model_simple": { "n": 24, "task_accuracy": 0.125, "macro_operation_accuracy": 0.125, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.20833333333333334, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.125, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "needs_clarification": { "needs_clarification": 3 }, "success": { "needs_clarification": 16, "success": 2 }, "out_of_domain": { "invalid_input": 3 } }, "cost_per_correct_task_usd": 0.07110000000000001 }, "transform_frames": { "n": 24, "task_accuracy": 0.125, "macro_operation_accuracy": 0.125, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.125, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.125, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "out_of_domain": { "invalid_input": 3 }, "success": { "needs_clarification": 18 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.06195499999999999 }, "geodetic_to_ecef": { "n": 24, "task_accuracy": 0.125, "macro_operation_accuracy": 0.125, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.875, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.125, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "out_of_domain": { "invalid_input": 3 }, "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.400995 }, "ecef_to_geodetic": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0.05555555555555555, "normalized_numerical_error": 126.37050878707886, "status_accuracy": 0.9583333333333334, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 } }, "cost_per_correct_task_usd": 0.516489 }, "wing_vlm_analysis": { "n": 24, "task_accuracy": 0.125, "macro_operation_accuracy": 0.125, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.125, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.125, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 18 }, "out_of_domain": { "invalid_input": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.090455 }, "airfoil_polar_analysis": { "n": 24, "task_accuracy": 0.125, "macro_operation_accuracy": 0.125, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.875, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.125, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "invalid_input": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.08993499999999999 }, "calculate_stability_derivatives": { "n": 24, "task_accuracy": 0.125, "macro_operation_accuracy": 0.125, "field_accuracy": 0, "normalized_numerical_error": 230.2677727686655, "status_accuracy": 0.8333333333333334, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.125, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 17, "needs_clarification": 1 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "invalid_input": 3 } }, "cost_per_correct_task_usd": 0.17282500000000003 }, "get_airfoil_database": { "n": 24, "task_accuracy": 0.125, "macro_operation_accuracy": 0.125, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.125, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.125, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": null, "classification_matrix": { "success": { "needs_clarification": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "invalid_input": 3 } }, "cost_per_correct_task_usd": 0.05233499999999999 }, "propeller_bemt_analysis": { "n": 24, "task_accuracy": 0.125, "macro_operation_accuracy": 0.125, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.125, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.125, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "invalid_input": 3 } }, "cost_per_correct_task_usd": 0.09180500000000001 }, "uav_energy_estimate": { "n": 24, "task_accuracy": 0.16666666666666666, "macro_operation_accuracy": 0.16666666666666666, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.16666666666666666, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 18 }, "out_of_domain": { "invalid_input": 2, "out_of_domain": 1 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.06309375 }, "get_propeller_database": { "n": 24, "task_accuracy": 0.125, "macro_operation_accuracy": 0.125, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.125, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.125, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": null, "classification_matrix": { "success": { "needs_clarification": 18 }, "out_of_domain": { "invalid_input": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.057074999999999994 }, "rocket_3dof_trajectory": { "n": 24, "task_accuracy": 0.16666666666666666, "macro_operation_accuracy": 0.16666666666666666, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.16666666666666666, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 18 }, "out_of_domain": { "invalid_input": 2, "out_of_domain": 1 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.05938875000000001 }, "estimate_rocket_sizing": { "n": 24, "task_accuracy": 0.125, "macro_operation_accuracy": 0.125, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.125, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.125, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 18 }, "out_of_domain": { "invalid_input": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.065365 }, "optimize_launch_angle": { "n": 24, "task_accuracy": 0.16666666666666666, "macro_operation_accuracy": 0.16666666666666666, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.16666666666666666, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "needs_clarification": { "needs_clarification": 3 }, "success": { "needs_clarification": 18 }, "out_of_domain": { "invalid_input": 2, "out_of_domain": 1 } }, "cost_per_correct_task_usd": 0.057963749999999994 }, "elements_to_state_vector": { "n": 24, "task_accuracy": 0.125, "macro_operation_accuracy": 0.125, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.125, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.125, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "needs_clarification": { "needs_clarification": 3 }, "success": { "needs_clarification": 18 }, "out_of_domain": { "invalid_input": 3 } }, "cost_per_correct_task_usd": 0.11765500000000002 }, "state_vector_to_elements": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.20833333333333334, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "invalid_input": 1, "out_of_domain": 2 } }, "cost_per_correct_task_usd": 0.028221000000000003 }, "propagate_orbit_j2": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.20833333333333334, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 18 }, "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.04301099999999999 }, "calculate_ground_track": { "n": 24, "task_accuracy": 0.125, "macro_operation_accuracy": 0.125, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.125, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.125, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "invalid_input": 3 } }, "cost_per_correct_task_usd": 0.079525 }, "hohmann_transfer": { "n": 24, "task_accuracy": 0.125, "macro_operation_accuracy": 0.125, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.125, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.125, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "invalid_input": 3 } }, "cost_per_correct_task_usd": 0.049105 }, "orbital_rendezvous_planning": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0.05555555555555555, "normalized_numerical_error": 0.02500000000016378, "status_accuracy": 0.20833333333333334, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 } }, "cost_per_correct_task_usd": 0.07639500000000002 }, "lambert_problem_solver": { "n": 24, "task_accuracy": 0.5416666666666666, "macro_operation_accuracy": 0.5416666666666666, "field_accuracy": 0.4444444444444444, "normalized_numerical_error": null, "status_accuracy": 0.875, "format_validity": 1, "pass_at_1": 0.5, "pass_at_3": 0.625, "pass_power_3": 0.375, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 }, "success": { "success": 16, "needs_clarification": 2 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.2086626923076923 }, "kalman_filter_state_estimation": { "n": 24, "task_accuracy": 0.16666666666666666, "macro_operation_accuracy": 0.16666666666666666, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.20833333333333334, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "out_of_domain": { "invalid_input": 2, "out_of_domain": 1 }, "needs_clarification": { "needs_clarification": 3 }, "success": { "needs_clarification": 17, "success": 1 } }, "cost_per_correct_task_usd": 0.151275 }, "lqr_controller_design": { "n": 24, "task_accuracy": 0.4166666666666667, "macro_operation_accuracy": 0.4166666666666667, "field_accuracy": 0.3888888888888889, "normalized_numerical_error": 0.0003296239365428791, "status_accuracy": 0.875, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.625, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "invalid_input": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.0262515 }, "density_altitude_calculator": { "n": 24, "task_accuracy": 0.125, "macro_operation_accuracy": 0.125, "field_accuracy": 0, "normalized_numerical_error": 204.7779795300263, "status_accuracy": 0.625, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.125, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "out_of_domain": { "invalid_input": 3 }, "success": { "success": 12, "invalid_input": 6 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.26635499999999995 }, "true_airspeed_converter": { "n": 24, "task_accuracy": 0.125, "macro_operation_accuracy": 0.125, "field_accuracy": 0, "normalized_numerical_error": 4.450000000001463, "status_accuracy": 0.20833333333333334, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.125, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "needs_clarification": { "needs_clarification": 3 }, "success": { "needs_clarification": 16, "success": 2 }, "out_of_domain": { "invalid_input": 3 } }, "cost_per_correct_task_usd": 0.09601500000000002 }, "stall_speed_calculator": { "n": 24, "task_accuracy": 0.125, "macro_operation_accuracy": 0.125, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.125, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.125, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 18 }, "out_of_domain": { "invalid_input": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.07289499999999999 }, "weight_and_balance": { "n": 24, "task_accuracy": 0.5, "macro_operation_accuracy": 0.5, "field_accuracy": 0.7222222222222222, "normalized_numerical_error": 1.2824999999957083, "status_accuracy": 0.9166666666666666, "format_validity": 1, "pass_at_1": 0.625, "pass_at_3": 0.625, "pass_power_3": 0.375, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "out_of_domain": 1, "invalid_input": 2 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.01857125 }, "takeoff_performance": { "n": 24, "task_accuracy": 0.125, "macro_operation_accuracy": 0.125, "field_accuracy": 0, "normalized_numerical_error": 38, "status_accuracy": 0.20833333333333334, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.125, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 16, "success": 2 }, "out_of_domain": { "invalid_input": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.11442500000000001 }, "landing_performance": { "n": 24, "task_accuracy": 0.125, "macro_operation_accuracy": 0.125, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.125, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.125, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "out_of_domain": { "invalid_input": 3 }, "needs_clarification": { "needs_clarification": 3 }, "success": { "needs_clarification": 18 } }, "cost_per_correct_task_usd": 0.079515 }, "fuel_reserve_calculator": { "n": 24, "task_accuracy": 0.875, "macro_operation_accuracy": 0.875, "field_accuracy": 1, "normalized_numerical_error": 0, "status_accuracy": 0.875, "format_validity": 1, "pass_at_1": 0.875, "pass_at_3": 0.875, "pass_power_3": 0.875, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "invalid_input": 3 } }, "cost_per_correct_task_usd": 0.012225 }, "optimize_thrust_profile": { "n": 24, "task_accuracy": 0.25, "macro_operation_accuracy": 0.25, "field_accuracy": 0.3055555555555556, "normalized_numerical_error": 0, "status_accuracy": 0.25, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 }, "success": { "needs_clarification": 18 } }, "cost_per_correct_task_usd": 0.05464750000000001 }, "trajectory_sensitivity_analysis": { "n": 24, "task_accuracy": 0.125, "macro_operation_accuracy": 0.125, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.125, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.125, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 18 }, "out_of_domain": { "invalid_input": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.07469500000000001 }, "genetic_algorithm_optimization": { "n": 24, "task_accuracy": 0.125, "macro_operation_accuracy": 0.125, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.125, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.125, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "success": { "needs_clarification": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "invalid_input": 3 } }, "cost_per_correct_task_usd": 0.073785 }, "particle_swarm_optimization": { "n": 24, "task_accuracy": 0.125, "macro_operation_accuracy": 0.125, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.125, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.125, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "out_of_domain": { "invalid_input": 3 }, "success": { "needs_clarification": 18 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.069675 }, "porkchop_plot_analysis": { "n": 24, "task_accuracy": 0.125, "macro_operation_accuracy": 0.125, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.125, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.125, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "invalid_input": 3 }, "success": { "needs_clarification": 18 } }, "cost_per_correct_task_usd": 0.20455500000000001 }, "monte_carlo_uncertainty_analysis": { "n": 24, "task_accuracy": 0.16666666666666666, "macro_operation_accuracy": 0.16666666666666666, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.16666666666666666, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0, "mean_skill_reads": 0, "runner_invocation_rate_on_required_tasks": 0, "classification_matrix": { "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "invalid_input": 2, "out_of_domain": 1 }, "success": { "needs_clarification": 18 } }, "cost_per_correct_task_usd": 0.061548750000000006 } }, "sol-full-skills": { "search_airports": { "n": 24, "task_accuracy": 0.16666666666666666, "macro_operation_accuracy": 0.16666666666666666, "field_accuracy": 0.05555555555555555, "normalized_numerical_error": 0.75, "status_accuracy": 0.875, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 2.25, "runner_invocation_rate_on_required_tasks": null, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "invalid_input": 3 } }, "cost_per_correct_task_usd": 0.11370012499999999 }, "plan_flight": { "n": 24, "task_accuracy": 0.8333333333333334, "macro_operation_accuracy": 0.8333333333333334, "field_accuracy": 0.8888888888888888, "normalized_numerical_error": 0, "status_accuracy": 0.9166666666666666, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 0.625, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 1, "invalid_input": 2 } }, "cost_per_correct_task_usd": 0.018358625 }, "plan_multi_leg_flight": { "n": 24, "task_accuracy": 0.8333333333333334, "macro_operation_accuracy": 0.8333333333333334, "field_accuracy": 0.8888888888888888, "normalized_numerical_error": 0, "status_accuracy": 0.9166666666666666, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 0.625, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "out_of_domain": 1, "invalid_input": 2 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.015976862499999998 }, "calculate_distance": { "n": 24, "task_accuracy": 0.875, "macro_operation_accuracy": 0.875, "field_accuracy": 1, "normalized_numerical_error": 0, "status_accuracy": 0.875, "format_validity": 1, "pass_at_1": 0.875, "pass_at_3": 0.875, "pass_power_3": 0.875, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5833333333333333, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "out_of_domain": { "invalid_input": 3 }, "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.01964997619047619 }, "get_aircraft_performance": { "n": 24, "task_accuracy": 0.75, "macro_operation_accuracy": 0.75, "field_accuracy": 0.8333333333333334, "normalized_numerical_error": 0, "status_accuracy": 0.875, "format_validity": 1, "pass_at_1": 0.875, "pass_at_3": 0.875, "pass_power_3": 0.5, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "invalid_input": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.01824933333333333 }, "get_aircraft_database": { "n": 24, "task_accuracy": 0.16666666666666666, "macro_operation_accuracy": 0.16666666666666666, "field_accuracy": 0, "normalized_numerical_error": 1, "status_accuracy": 0.9166666666666666, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 2.25, "runner_invocation_rate_on_required_tasks": null, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "invalid_input": 2, "out_of_domain": 1 } }, "cost_per_correct_task_usd": 0.1043508125 }, "get_system_status": { "n": 24, "task_accuracy": 0.5833333333333334, "macro_operation_accuracy": 0.5833333333333334, "field_accuracy": 0.5, "normalized_numerical_error": 0.4375, "status_accuracy": 0.9583333333333334, "format_validity": 1, "pass_at_1": 0.625, "pass_at_3": 1, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 2.375, "runner_invocation_rate_on_required_tasks": null, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "invalid_input": 1, "out_of_domain": 2 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.042169839285714286 }, "convert_units": { "n": 24, "task_accuracy": 0.875, "macro_operation_accuracy": 0.875, "field_accuracy": 1, "normalized_numerical_error": 0, "status_accuracy": 0.875, "format_validity": 1, "pass_at_1": 0.875, "pass_at_3": 0.875, "pass_power_3": 0.875, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.4166666666666667, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "invalid_input": 3 } }, "cost_per_correct_task_usd": 0.013102940476190478 }, "get_atmosphere_profile": { "n": 24, "task_accuracy": 0.20833333333333334, "macro_operation_accuracy": 0.20833333333333334, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.9583333333333334, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "out_of_domain": { "invalid_input": 1, "out_of_domain": 2 }, "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.06766399999999999 }, "wind_model_simple": { "n": 24, "task_accuracy": 0.125, "macro_operation_accuracy": 0.125, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.875, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.125, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 2.0833333333333335, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "needs_clarification": { "needs_clarification": 3 }, "success": { "success": 18 }, "out_of_domain": { "invalid_input": 3 } }, "cost_per_correct_task_usd": 0.14315724999999999 }, "transform_frames": { "n": 24, "task_accuracy": 0.16666666666666666, "macro_operation_accuracy": 0.16666666666666666, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.9166666666666666, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5833333333333333, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "out_of_domain": { "invalid_input": 2, "out_of_domain": 1 }, "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.09504481250000003 }, "geodetic_to_ecef": { "n": 24, "task_accuracy": 0.2916666666666667, "macro_operation_accuracy": 0.2916666666666667, "field_accuracy": 0.05555555555555555, "normalized_numerical_error": null, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.375, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "out_of_domain": { "out_of_domain": 3 }, "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.05742717857142856 }, "ecef_to_geodetic": { "n": 24, "task_accuracy": 0.9166666666666666, "macro_operation_accuracy": 0.9166666666666666, "field_accuracy": 1, "normalized_numerical_error": 2.7729482907387944e-7, "status_accuracy": 0.9166666666666666, "format_validity": 1, "pass_at_1": 0.875, "pass_at_3": 1, "pass_power_3": 0.875, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.625, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "invalid_input": 2, "out_of_domain": 1 } }, "cost_per_correct_task_usd": 0.01712245454545455 }, "wing_vlm_analysis": { "n": 24, "task_accuracy": 0.2916666666666667, "macro_operation_accuracy": 0.2916666666666667, "field_accuracy": 0.3333333333333333, "normalized_numerical_error": 0.23486840075928916, "status_accuracy": 0.625, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.5, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.9583333333333333, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 12, "needs_clarification": 6 }, "out_of_domain": { "invalid_input": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.11892499999999996 }, "airfoil_polar_analysis": { "n": 24, "task_accuracy": 0.16666666666666666, "macro_operation_accuracy": 0.16666666666666666, "field_accuracy": 0.05555555555555555, "normalized_numerical_error": 0, "status_accuracy": 0.875, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.25, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.875, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "invalid_input": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.11661074999999999 }, "calculate_stability_derivatives": { "n": 24, "task_accuracy": 0.75, "macro_operation_accuracy": 0.75, "field_accuracy": 0.8333333333333334, "normalized_numerical_error": 0, "status_accuracy": 0.875, "format_validity": 1, "pass_at_1": 0.625, "pass_at_3": 0.875, "pass_power_3": 0.5, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.875, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "invalid_input": 3 } }, "cost_per_correct_task_usd": 0.02259213888888889 }, "get_airfoil_database": { "n": 24, "task_accuracy": 0.4166666666666667, "macro_operation_accuracy": 0.4166666666666667, "field_accuracy": 0.2777777777777778, "normalized_numerical_error": 0.2857142857142857, "status_accuracy": 0.9583333333333334, "format_validity": 1, "pass_at_1": 0.5, "pass_at_3": 0.75, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 2.25, "runner_invocation_rate_on_required_tasks": null, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 } }, "cost_per_correct_task_usd": 0.052216175 }, "propeller_bemt_analysis": { "n": 24, "task_accuracy": 0.7916666666666666, "macro_operation_accuracy": 0.7916666666666666, "field_accuracy": 0.8333333333333334, "normalized_numerical_error": 6.02446252765354e-7, "status_accuracy": 0.7916666666666666, "format_validity": 1, "pass_at_1": 0.75, "pass_at_3": 1, "pass_power_3": 0.625, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 2, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 15, "needs_clarification": 3 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "invalid_input": 2, "out_of_domain": 1 } }, "cost_per_correct_task_usd": 0.04575111842105263 }, "uav_energy_estimate": { "n": 24, "task_accuracy": 0.125, "macro_operation_accuracy": 0.125, "field_accuracy": 0, "normalized_numerical_error": null, "status_accuracy": 0.875, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.125, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 2.5416666666666665, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "invalid_input": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.21848516666666665 }, "get_propeller_database": { "n": 24, "task_accuracy": 0.5416666666666666, "macro_operation_accuracy": 0.5416666666666666, "field_accuracy": 0.3888888888888889, "normalized_numerical_error": 0, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 0.625, "pass_at_3": 1, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 2.2916666666666665, "runner_invocation_rate_on_required_tasks": null, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "out_of_domain": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.03626978846153847 }, "rocket_3dof_trajectory": { "n": 24, "task_accuracy": 0.4583333333333333, "macro_operation_accuracy": 0.4583333333333333, "field_accuracy": 0.3333333333333333, "normalized_numerical_error": 0.3256723156957762, "status_accuracy": 0.9583333333333334, "format_validity": 1, "pass_at_1": 0.625, "pass_at_3": 0.75, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 2.1666666666666665, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.06886038636363637 }, "estimate_rocket_sizing": { "n": 24, "task_accuracy": 0.9583333333333334, "macro_operation_accuracy": 0.9583333333333334, "field_accuracy": 1, "normalized_numerical_error": 0, "status_accuracy": 0.9583333333333334, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 0.875, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.875, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.01668508695652174 }, "optimize_launch_angle": { "n": 24, "task_accuracy": 0.125, "macro_operation_accuracy": 0.125, "field_accuracy": 0.3611111111111111, "normalized_numerical_error": 0, "status_accuracy": 0.875, "format_validity": 1, "pass_at_1": 0.125, "pass_at_3": 0.125, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 2.1666666666666665, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "needs_clarification": { "needs_clarification": 3 }, "success": { "success": 18 }, "out_of_domain": { "invalid_input": 3 } }, "cost_per_correct_task_usd": 0.21914824999999996 }, "elements_to_state_vector": { "n": 24, "task_accuracy": 0.875, "macro_operation_accuracy": 0.875, "field_accuracy": 1, "normalized_numerical_error": null, "status_accuracy": 0.875, "format_validity": 1, "pass_at_1": 0.875, "pass_at_3": 0.875, "pass_power_3": 0.875, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.625, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "needs_clarification": { "needs_clarification": 3 }, "success": { "success": 18 }, "out_of_domain": { "invalid_input": 3 } }, "cost_per_correct_task_usd": 0.020404666666666665 }, "state_vector_to_elements": { "n": 24, "task_accuracy": 0.75, "macro_operation_accuracy": 0.75, "field_accuracy": 0.8611111111111112, "normalized_numerical_error": 0, "status_accuracy": 0.875, "format_validity": 1, "pass_at_1": 0.75, "pass_at_3": 0.875, "pass_power_3": 0.625, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.6666666666666667, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 17, "needs_clarification": 1 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "invalid_input": 2, "out_of_domain": 1 } }, "cost_per_correct_task_usd": 0.029511569444444448 }, "propagate_orbit_j2": { "n": 24, "task_accuracy": 0.9166666666666666, "macro_operation_accuracy": 0.9166666666666666, "field_accuracy": 1, "normalized_numerical_error": null, "status_accuracy": 0.9166666666666666, "format_validity": 1, "pass_at_1": 0.875, "pass_at_3": 1, "pass_power_3": 0.875, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "invalid_input": 2, "out_of_domain": 1 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.018976931818181816 }, "calculate_ground_track": { "n": 24, "task_accuracy": 0.9583333333333334, "macro_operation_accuracy": 0.9583333333333334, "field_accuracy": 1, "normalized_numerical_error": 0.00048611111082132023, "status_accuracy": 0.9583333333333334, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 0.875, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.75, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 2, "invalid_input": 1 } }, "cost_per_correct_task_usd": 0.01790294565217391 }, "hohmann_transfer": { "n": 24, "task_accuracy": 1, "macro_operation_accuracy": 1, "field_accuracy": 1, "normalized_numerical_error": 0, "status_accuracy": 1, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 1, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5416666666666667, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 3 } }, "cost_per_correct_task_usd": 0.014541031249999998 }, "orbital_rendezvous_planning": { "n": 24, "task_accuracy": 0.9166666666666666, "macro_operation_accuracy": 0.9166666666666666, "field_accuracy": 1, "normalized_numerical_error": 0, "status_accuracy": 0.9166666666666666, "format_validity": 1, "pass_at_1": 0.875, "pass_at_3": 1, "pass_power_3": 0.875, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "invalid_input": 2, "out_of_domain": 1 } }, "cost_per_correct_task_usd": 0.020017738636363632 }, "lambert_problem_solver": { "n": 24, "task_accuracy": 0.9166666666666666, "macro_operation_accuracy": 0.9166666666666666, "field_accuracy": 1, "normalized_numerical_error": null, "status_accuracy": 0.9166666666666666, "format_validity": 1, "pass_at_1": 0.875, "pass_at_3": 1, "pass_power_3": 0.875, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "out_of_domain": { "invalid_input": 2, "out_of_domain": 1 }, "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.017643943181818184 }, "kalman_filter_state_estimation": { "n": 24, "task_accuracy": 0.2916666666666667, "macro_operation_accuracy": 0.2916666666666667, "field_accuracy": 0.16666666666666666, "normalized_numerical_error": null, "status_accuracy": 0.9166666666666666, "format_validity": 1, "pass_at_1": 0.25, "pass_at_3": 0.5, "pass_power_3": 0.125, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "out_of_domain": { "out_of_domain": 1, "invalid_input": 2 }, "needs_clarification": { "needs_clarification": 3 }, "success": { "success": 18 } }, "cost_per_correct_task_usd": 0.061438964285714305 }, "lqr_controller_design": { "n": 24, "task_accuracy": 0.7916666666666666, "macro_operation_accuracy": 0.7916666666666666, "field_accuracy": 0.8888888888888888, "normalized_numerical_error": 0.003963666565432901, "status_accuracy": 0.875, "format_validity": 1, "pass_at_1": 0.875, "pass_at_3": 0.875, "pass_power_3": 0.625, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "invalid_input": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.01800852631578947 }, "density_altitude_calculator": { "n": 24, "task_accuracy": 0.75, "macro_operation_accuracy": 0.75, "field_accuracy": 0.8333333333333334, "normalized_numerical_error": 0, "status_accuracy": 0.875, "format_validity": 1, "pass_at_1": 0.875, "pass_at_3": 0.875, "pass_power_3": 0.625, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "out_of_domain": { "invalid_input": 3 }, "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.02393541666666667 }, "true_airspeed_converter": { "n": 24, "task_accuracy": 0.5416666666666666, "macro_operation_accuracy": 0.5416666666666666, "field_accuracy": 0.5, "normalized_numerical_error": 0, "status_accuracy": 0.9166666666666666, "format_validity": 1, "pass_at_1": 0.5, "pass_at_3": 0.75, "pass_power_3": 0.375, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 2.5, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "needs_clarification": { "needs_clarification": 3 }, "success": { "success": 18 }, "out_of_domain": { "invalid_input": 2, "out_of_domain": 1 } }, "cost_per_correct_task_usd": 0.04373684615384616 }, "stall_speed_calculator": { "n": 24, "task_accuracy": 0.5416666666666666, "macro_operation_accuracy": 0.5416666666666666, "field_accuracy": 0.5555555555555556, "normalized_numerical_error": 0, "status_accuracy": 0.875, "format_validity": 1, "pass_at_1": 0.5, "pass_at_3": 0.625, "pass_power_3": 0.375, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 2.0416666666666665, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "invalid_input": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.04088067307692307 }, "weight_and_balance": { "n": 24, "task_accuracy": 0.5416666666666666, "macro_operation_accuracy": 0.5416666666666666, "field_accuracy": 0.5, "normalized_numerical_error": 0, "status_accuracy": 0.9166666666666666, "format_validity": 1, "pass_at_1": 0.625, "pass_at_3": 0.75, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.875, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "invalid_input": 2, "out_of_domain": 1 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.03997875 }, "takeoff_performance": { "n": 24, "task_accuracy": 0.7083333333333334, "macro_operation_accuracy": 0.7083333333333334, "field_accuracy": 0.7777777777777778, "normalized_numerical_error": 0, "status_accuracy": 0.875, "format_validity": 1, "pass_at_1": 0.75, "pass_at_3": 0.875, "pass_power_3": 0.375, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.625, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "invalid_input": 3 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.02260983823529412 }, "landing_performance": { "n": 24, "task_accuracy": 0.4583333333333333, "macro_operation_accuracy": 0.4583333333333333, "field_accuracy": 0.4444444444444444, "normalized_numerical_error": 0, "status_accuracy": 0.875, "format_validity": 1, "pass_at_1": 0.375, "pass_at_3": 0.625, "pass_power_3": 0.375, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.7916666666666667, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "out_of_domain": { "invalid_input": 3 }, "needs_clarification": { "needs_clarification": 3 }, "success": { "success": 18 } }, "cost_per_correct_task_usd": 0.038381477272727275 }, "fuel_reserve_calculator": { "n": 24, "task_accuracy": 0.9166666666666666, "macro_operation_accuracy": 0.9166666666666666, "field_accuracy": 1, "normalized_numerical_error": 0, "status_accuracy": 0.9166666666666666, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 0.875, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 1, "invalid_input": 2 } }, "cost_per_correct_task_usd": 0.016138988636363635 }, "optimize_thrust_profile": { "n": 24, "task_accuracy": 0.625, "macro_operation_accuracy": 0.625, "field_accuracy": 0.5833333333333334, "normalized_numerical_error": 0.07983322594364586, "status_accuracy": 0.9583333333333334, "format_validity": 1, "pass_at_1": 0.5, "pass_at_3": 0.875, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 2.625, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "out_of_domain": { "invalid_input": 1, "out_of_domain": 2 }, "needs_clarification": { "needs_clarification": 3 }, "success": { "success": 18 } }, "cost_per_correct_task_usd": 0.04753403333333333 }, "trajectory_sensitivity_analysis": { "n": 24, "task_accuracy": 0.9166666666666666, "macro_operation_accuracy": 0.9166666666666666, "field_accuracy": 1, "normalized_numerical_error": 0.5275664594203842, "status_accuracy": 0.9166666666666666, "format_validity": 1, "pass_at_1": 0.875, "pass_at_3": 1, "pass_power_3": 0.875, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.875, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "out_of_domain": { "invalid_input": 2, "out_of_domain": 1 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.02281404545454546 }, "genetic_algorithm_optimization": { "n": 24, "task_accuracy": 0.4166666666666667, "macro_operation_accuracy": 0.4166666666666667, "field_accuracy": 0.3333333333333333, "normalized_numerical_error": 0, "status_accuracy": 0.9166666666666666, "format_validity": 1, "pass_at_1": 0.375, "pass_at_3": 0.625, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.75, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "out_of_domain": 1, "invalid_input": 2 } }, "cost_per_correct_task_usd": 0.038195675 }, "particle_swarm_optimization": { "n": 24, "task_accuracy": 0.875, "macro_operation_accuracy": 0.875, "field_accuracy": 0.9444444444444444, "normalized_numerical_error": 0, "status_accuracy": 0.9166666666666666, "format_validity": 1, "pass_at_1": 1, "pass_at_3": 1, "pass_power_3": 0.75, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "out_of_domain": { "out_of_domain": 1, "invalid_input": 2 }, "success": { "success": 18 }, "needs_clarification": { "needs_clarification": 3 } }, "cost_per_correct_task_usd": 0.016468654761904764 }, "porkchop_plot_analysis": { "n": 24, "task_accuracy": 0.5416666666666666, "macro_operation_accuracy": 0.5416666666666666, "field_accuracy": 0.5555555555555556, "normalized_numerical_error": 0, "status_accuracy": 0.875, "format_validity": 1, "pass_at_1": 0.375, "pass_at_3": 0.75, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "invalid_input": 3 }, "success": { "success": 18 } }, "cost_per_correct_task_usd": 0.03806096153846155 }, "monte_carlo_uncertainty_analysis": { "n": 24, "task_accuracy": 0.5416666666666666, "macro_operation_accuracy": 0.5416666666666666, "field_accuracy": 0.7777777777777778, "normalized_numerical_error": 0, "status_accuracy": 0.875, "format_validity": 1, "pass_at_1": 0.625, "pass_at_3": 0.75, "pass_power_3": 0.25, "terminal_failure_rate": 0, "mean_runner_calls": 0.75, "mean_skill_reads": 1.5, "runner_invocation_rate_on_required_tasks": 1, "classification_matrix": { "needs_clarification": { "needs_clarification": 3 }, "out_of_domain": { "invalid_input": 3 }, "success": { "success": 18 } }, "cost_per_correct_task_usd": 0.02723809615384615 } } }, "power_simulation": { "power": 0.8846, "tasks": 360, "repeats": 3, "simulations": 5000, "baseline": 0.5, "uplift": 0.05, "intraclass_correlation_assumption": 0.2, "test": "two-sided normal approximation on paired task means", "seed": 5000005 } } }, "economics": { "status": "complete", "scope": "sealed_evaluation", "accounted_trials": 8640, "expected_trials": 8640, "metrics": [ "cost_per_completed_trial_usd", "cost_per_correct_task_usd" ], "arms": { "nemotron-base": { "completed_trials": 1080, "key_answer_accuracy": 0.22870370370370371, "cost_per_completed_trial_usd": 0, "cost_per_correct_task_usd": 0 }, "nemotron-docs-only": { "completed_trials": 1080, "key_answer_accuracy": 0.22314814814814815, "cost_per_completed_trial_usd": 0, "cost_per_correct_task_usd": 0 }, "nemotron-runner-only": { "completed_trials": 1080, "key_answer_accuracy": 0.5851851851851851, "cost_per_completed_trial_usd": 0, "cost_per_correct_task_usd": 0 }, "nemotron-full-skills": { "completed_trials": 1080, "key_answer_accuracy": 0.5194444444444445, "cost_per_completed_trial_usd": 0, "cost_per_correct_task_usd": 0 }, "terra-base": { "completed_trials": 1080, "key_answer_accuracy": 0.3101851851851852, "cost_per_completed_trial_usd": 0.007856298611111114, "cost_per_correct_task_usd": 0.025327768656716425 }, "terra-full-skills": { "completed_trials": 1080, "key_answer_accuracy": 0.8361111111111111, "cost_per_completed_trial_usd": 0.00750966180555555, "cost_per_correct_task_usd": 0.008981655315614612 }, "sol-base": { "completed_trials": 1080, "key_answer_accuracy": 0.1962962962962963, "cost_per_completed_trial_usd": 0.017155458333333307, "cost_per_correct_task_usd": 0.08739573113207534 }, "sol-full-skills": { "completed_trials": 1080, "key_answer_accuracy": 0.6046296296296296, "cost_per_completed_trial_usd": 0.01931244722222221, "cost_per_correct_task_usd": 0.03194095405819294 } }, "basis": "provider-recorded usage where available; cache-aware direct-provider token rates otherwise" }, "blocker_summary": { "independent_reference_implementations_remaining": 0, "required_external_reviewers": 0, "sealed_tasks_remaining": 0, "model_inference_released": true }, "blockers": [], "v1": { "status": "exploratory_runner_consistency_study_superseded", "report_path": "/benchmark/v1" } }