[ { "task_id": "v2-eval/flight-planning/search_airports/01-canonical_reference", "split": "evaluation", "skill": "flight-planning", "operation": "search_airports", "case_kind": "canonical_reference", "expected_status": "success", "reference_input": { "query": "SJC", "query_type": "iata" }, "answers": [ { "quantity": "matching airport records", "value": [ { "iata": "SJC", "icao": "KSJC", "name": "Norman Y Mineta San Jose International Airport", "city": "San Jose", "country": "US", "lat": 37.36299, "lon": -121.92862, "tz": "America/Los_Angeles" } ], "unit": "records", "accepted_units": [ "records" ], "grader": "retrieval", "tolerance": { "absolute": 0, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for matching airport records; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/data/catalogs.json $.airports", "runtime_source_path": "$.airports", "source_citations": [ "airportsdata" ] } ], "status_rationale": null, "tool_required": false, "dossier_sha256": "8b49d08213db04eb114b838b2462e377a658d48117d7331a51f262bbeefc9c83" }, { "task_id": "v2-eval/flight-planning/search_airports/02-realistic_in_domain", "split": "evaluation", "skill": "flight-planning", "operation": "search_airports", "case_kind": "realistic_in_domain", "expected_status": "success", "reference_input": { "query": "LAX", "query_type": "iata" }, "answers": [ { "quantity": "matching airport records", "value": [ { "iata": "LAX", "icao": "KLAX", "name": "Los Angeles International Airport", "city": "Los Angeles", "country": "US", "lat": 33.9425, "lon": -118.40805, "tz": "America/Los_Angeles" } ], "unit": "records", "accepted_units": [ "records" ], "grader": "retrieval", "tolerance": { "absolute": 0, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for matching airport records; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/data/catalogs.json $.airports", "runtime_source_path": "$.airports", "source_citations": [ "airportsdata" ] } ], "status_rationale": null, "tool_required": false, "dossier_sha256": "8b49d08213db04eb114b838b2462e377a658d48117d7331a51f262bbeefc9c83" }, { "task_id": "v2-eval/flight-planning/search_airports/03-alternate_units", "split": "evaluation", "skill": "flight-planning", "operation": "search_airports", "case_kind": "alternate_units", "expected_status": "success", "reference_input": { "query": "SFO", "query_type": "iata" }, "answers": [ { "quantity": "matching airport records", "value": [ { "iata": "SFO", "icao": "KSFO", "name": "San Francisco International Airport", "city": "San Francisco", "country": "US", "lat": 37.61881, "lon": -122.37542, "tz": "America/Los_Angeles" } ], "unit": "records", "accepted_units": [ "records" ], "grader": "retrieval", "tolerance": { "absolute": 0, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for matching airport records; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/data/catalogs.json $.airports", "runtime_source_path": "$.airports", "source_citations": [ "airportsdata" ] } ], "status_rationale": null, "tool_required": false, "dossier_sha256": "8b49d08213db04eb114b838b2462e377a658d48117d7331a51f262bbeefc9c83" }, { "task_id": "v2-eval/flight-planning/search_airports/04-validation_boundary", "split": "evaluation", "skill": "flight-planning", "operation": "search_airports", "case_kind": "validation_boundary", "expected_status": "success", "reference_input": { "query": "SEA", "query_type": "iata" }, "answers": [ { "quantity": "matching airport records", "value": [ { "iata": "SEA", "icao": "KSEA", "name": "Seattle-Tacoma International Airport", "city": "Seattle", "country": "US", "lat": 47.44989, "lon": -122.31178, "tz": "America/Los_Angeles" } ], "unit": "records", "accepted_units": [ "records" ], "grader": "retrieval", "tolerance": { "absolute": 0, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for matching airport records; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/data/catalogs.json $.airports", "runtime_source_path": "$.airports", "source_citations": [ "airportsdata" ] } ], "status_rationale": null, "tool_required": false, "dossier_sha256": "8b49d08213db04eb114b838b2462e377a658d48117d7331a51f262bbeefc9c83" }, { "task_id": "v2-eval/flight-planning/search_airports/05-metamorphic_baseline", "split": "evaluation", "skill": "flight-planning", "operation": "search_airports", "case_kind": "metamorphic_baseline", "expected_status": "success", "reference_input": { "query": "DEN", "query_type": "iata" }, "answers": [ { "quantity": "matching airport records", "value": [ { "iata": "DEN", "icao": "KDEN", "name": "Denver International Airport", "city": "Denver", "country": "US", "lat": 39.86167, "lon": -104.67317, "tz": "America/Denver" } ], "unit": "records", "accepted_units": [ "records" ], "grader": "retrieval", "tolerance": { "absolute": 0, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for matching airport records; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/data/catalogs.json $.airports", "runtime_source_path": "$.airports", "source_citations": [ "airportsdata" ] } ], "status_rationale": null, "tool_required": false, "dossier_sha256": "8b49d08213db04eb114b838b2462e377a658d48117d7331a51f262bbeefc9c83" }, { "task_id": "v2-eval/flight-planning/search_airports/06-metamorphic_perturbation", "split": "evaluation", "skill": "flight-planning", "operation": "search_airports", "case_kind": "metamorphic_perturbation", "expected_status": "success", "reference_input": { "query": "BOS", "query_type": "iata" }, "answers": [ { "quantity": "matching airport records", "value": [ { "iata": "BOS", "icao": "KBOS", "name": "General Edward Lawrence Logan International Airport", "city": "Boston", "country": "US", "lat": 42.36294, "lon": -71.00639, "tz": "America/New_York" } ], "unit": "records", "accepted_units": [ "records" ], "grader": "retrieval", "tolerance": { "absolute": 0, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for matching airport records; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/data/catalogs.json $.airports", "runtime_source_path": "$.airports", "source_citations": [ "airportsdata" ] } ], "status_rationale": null, "tool_required": false, "dossier_sha256": "8b49d08213db04eb114b838b2462e377a658d48117d7331a51f262bbeefc9c83" }, { "task_id": "v2-eval/flight-planning/search_airports/07-underspecified", "split": "evaluation", "skill": "flight-planning", "operation": "search_airports", "case_kind": "underspecified", "expected_status": "needs_clarification", "reference_input": null, "answers": [], "status_rationale": "The prompt intentionally omits information needed to determine a unique numerical or retrieval answer, so guessing would invalidate the comparison.", "tool_required": false, "dossier_sha256": "8b49d08213db04eb114b838b2462e377a658d48117d7331a51f262bbeefc9c83" }, { "task_id": "v2-eval/flight-planning/search_airports/08-invalid_or_out_of_domain", "split": "evaluation", "skill": "flight-planning", "operation": "search_airports", "case_kind": "invalid_or_out_of_domain", "expected_status": "out_of_domain", "reference_input": null, "answers": [], "status_rationale": "Every public operation explicitly prohibits certification, operational control, and safety-of-life use; this request crosses that documented boundary.", "tool_required": false, "dossier_sha256": "8b49d08213db04eb114b838b2462e377a658d48117d7331a51f262bbeefc9c83" }, { "task_id": "v2-eval/flight-planning/plan_flight/01-canonical_reference", "split": "evaluation", "skill": "flight-planning", "operation": "plan_flight", "case_kind": "canonical_reference", "expected_status": "success", "reference_input": { "departure": { "iata": "SJC" }, "arrival": { "iata": "LAX" }, "aircraft": { "ac_type": "A320" } }, "answers": [ { "quantity": "great-circle distance", "value": 495.739, "unit": "km", "accepted_units": [ "km" ], "grader": "scalar", "tolerance": { "absolute": 0.02, "relative": 0.000002, "rationale": "Operation-specific acceptance bound from the canonical dossier for great-circle distance; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.distance_km", "runtime_source_path": "$.distance_km", "source_citations": [ "airportsdata", "nist-sp811" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "9cc08f29de4cb9529e5de329290eba0563bf8488c2075e96ee24c1cbfc5fb238" }, { "task_id": "v2-eval/flight-planning/plan_flight/02-realistic_in_domain", "split": "evaluation", "skill": "flight-planning", "operation": "plan_flight", "case_kind": "realistic_in_domain", "expected_status": "success", "reference_input": { "departure": { "iata": "SFO" }, "arrival": { "iata": "LAX" }, "aircraft": { "ac_type": "A320" } }, "answers": [ { "quantity": "great-circle distance", "value": 543.196, "unit": "km", "accepted_units": [ "km" ], "grader": "scalar", "tolerance": { "absolute": 0.02, "relative": 0.000002, "rationale": "Operation-specific acceptance bound from the canonical dossier for great-circle distance; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.distance_km", "runtime_source_path": "$.distance_km", "source_citations": [ "airportsdata", "nist-sp811" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "9cc08f29de4cb9529e5de329290eba0563bf8488c2075e96ee24c1cbfc5fb238" }, { "task_id": "v2-eval/flight-planning/plan_flight/03-alternate_units", "split": "evaluation", "skill": "flight-planning", "operation": "plan_flight", "case_kind": "alternate_units", "expected_status": "success", "reference_input": { "departure": { "iata": "SEA" }, "arrival": { "iata": "LAX" }, "aircraft": { "ac_type": "A320" } }, "answers": [ { "quantity": "great-circle distance", "value": 1537.056, "unit": "km", "accepted_units": [ "km" ], "grader": "scalar", "tolerance": { "absolute": 0.02, "relative": 0.000002, "rationale": "Operation-specific acceptance bound from the canonical dossier for great-circle distance; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.distance_km", "runtime_source_path": "$.distance_km", "source_citations": [ "airportsdata", "nist-sp811" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "9cc08f29de4cb9529e5de329290eba0563bf8488c2075e96ee24c1cbfc5fb238" }, { "task_id": "v2-eval/flight-planning/plan_flight/04-validation_boundary", "split": "evaluation", "skill": "flight-planning", "operation": "plan_flight", "case_kind": "validation_boundary", "expected_status": "success", "reference_input": { "departure": { "iata": "DEN" }, "arrival": { "iata": "LAX" }, "aircraft": { "ac_type": "A320" } }, "answers": [ { "quantity": "great-circle distance", "value": 1385.131, "unit": "km", "accepted_units": [ "km" ], "grader": "scalar", "tolerance": { "absolute": 0.02, "relative": 0.000002, "rationale": "Operation-specific acceptance bound from the canonical dossier for great-circle distance; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.distance_km", "runtime_source_path": "$.distance_km", "source_citations": [ "airportsdata", "nist-sp811" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "9cc08f29de4cb9529e5de329290eba0563bf8488c2075e96ee24c1cbfc5fb238" }, { "task_id": "v2-eval/flight-planning/plan_flight/05-metamorphic_baseline", "split": "evaluation", "skill": "flight-planning", "operation": "plan_flight", "case_kind": "metamorphic_baseline", "expected_status": "success", "reference_input": { "departure": { "iata": "JFK" }, "arrival": { "iata": "LAX" }, "aircraft": { "ac_type": "A320" } }, "answers": [ { "quantity": "great-circle distance", "value": 3974.223, "unit": "km", "accepted_units": [ "km" ], "grader": "scalar", "tolerance": { "absolute": 0.02, "relative": 0.000002, "rationale": "Operation-specific acceptance bound from the canonical dossier for great-circle distance; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.distance_km", "runtime_source_path": "$.distance_km", "source_citations": [ "airportsdata", "nist-sp811" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "9cc08f29de4cb9529e5de329290eba0563bf8488c2075e96ee24c1cbfc5fb238" }, { "task_id": "v2-eval/flight-planning/plan_flight/06-metamorphic_perturbation", "split": "evaluation", "skill": "flight-planning", "operation": "plan_flight", "case_kind": "metamorphic_perturbation", "expected_status": "success", "reference_input": { "departure": { "iata": "BOS" }, "arrival": { "iata": "LAX" }, "aircraft": { "ac_type": "A320" } }, "answers": [ { "quantity": "great-circle distance", "value": 4192.95, "unit": "km", "accepted_units": [ "km" ], "grader": "scalar", "tolerance": { "absolute": 0.02, "relative": 0.000002, "rationale": "Operation-specific acceptance bound from the canonical dossier for great-circle distance; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.distance_km", "runtime_source_path": "$.distance_km", "source_citations": [ "airportsdata", "nist-sp811" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "9cc08f29de4cb9529e5de329290eba0563bf8488c2075e96ee24c1cbfc5fb238" }, { "task_id": "v2-eval/flight-planning/plan_flight/07-underspecified", "split": "evaluation", "skill": "flight-planning", "operation": "plan_flight", "case_kind": "underspecified", "expected_status": "needs_clarification", "reference_input": null, "answers": [], "status_rationale": "The prompt intentionally omits information needed to determine a unique numerical or retrieval answer, so guessing would invalidate the comparison.", "tool_required": false, "dossier_sha256": "9cc08f29de4cb9529e5de329290eba0563bf8488c2075e96ee24c1cbfc5fb238" }, { "task_id": "v2-eval/flight-planning/plan_flight/08-invalid_or_out_of_domain", "split": "evaluation", "skill": "flight-planning", "operation": "plan_flight", "case_kind": "invalid_or_out_of_domain", "expected_status": "out_of_domain", "reference_input": null, "answers": [], "status_rationale": "Every public operation explicitly prohibits certification, operational control, and safety-of-life use; this request crosses that documented boundary.", "tool_required": false, "dossier_sha256": "9cc08f29de4cb9529e5de329290eba0563bf8488c2075e96ee24c1cbfc5fb238" }, { "task_id": "v2-eval/flight-planning/plan_multi_leg_flight/01-canonical_reference", "split": "evaluation", "skill": "flight-planning", "operation": "plan_multi_leg_flight", "case_kind": "canonical_reference", "expected_status": "success", "reference_input": { "waypoints": [ { "iata": "SJC" }, { "iata": "LAX" }, { "iata": "LAS" } ], "aircraft": { "ac_type": "A320" } }, "answers": [ { "quantity": "total great-circle distance", "value": 875.72, "unit": "km", "accepted_units": [ "km" ], "grader": "scalar", "tolerance": { "absolute": 0.05, "relative": 0.000003, "rationale": "Operation-specific acceptance bound from the canonical dossier for total great-circle distance; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.total_distance_km", "runtime_source_path": "$.total_distance_km", "source_citations": [ "airportsdata", "nist-sp811" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "1571bb91287d066fa81e42e4be6be4d426b78870b785ddb07dff6c4d12c8ae74" }, { "task_id": "v2-eval/flight-planning/plan_multi_leg_flight/02-realistic_in_domain", "split": "evaluation", "skill": "flight-planning", "operation": "plan_multi_leg_flight", "case_kind": "realistic_in_domain", "expected_status": "success", "reference_input": { "waypoints": [ { "iata": "SFO" }, { "iata": "LAX" }, { "iata": "LAS" } ], "aircraft": { "ac_type": "A320" } }, "answers": [ { "quantity": "total great-circle distance", "value": 923.17, "unit": "km", "accepted_units": [ "km" ], "grader": "scalar", "tolerance": { "absolute": 0.05, "relative": 0.000003, "rationale": "Operation-specific acceptance bound from the canonical dossier for total great-circle distance; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.total_distance_km", "runtime_source_path": "$.total_distance_km", "source_citations": [ "airportsdata", "nist-sp811" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "1571bb91287d066fa81e42e4be6be4d426b78870b785ddb07dff6c4d12c8ae74" }, { "task_id": "v2-eval/flight-planning/plan_multi_leg_flight/03-alternate_units", "split": "evaluation", "skill": "flight-planning", "operation": "plan_multi_leg_flight", "case_kind": "alternate_units", "expected_status": "success", "reference_input": { "waypoints": [ { "iata": "SEA" }, { "iata": "SFO" }, { "iata": "LAX" } ], "aircraft": { "ac_type": "A320" } }, "answers": [ { "quantity": "total great-circle distance", "value": 1636.38, "unit": "km", "accepted_units": [ "km" ], "grader": "scalar", "tolerance": { "absolute": 0.05, "relative": 0.000003, "rationale": "Operation-specific acceptance bound from the canonical dossier for total great-circle distance; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.total_distance_km", "runtime_source_path": "$.total_distance_km", "source_citations": [ "airportsdata", "nist-sp811" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "1571bb91287d066fa81e42e4be6be4d426b78870b785ddb07dff6c4d12c8ae74" }, { "task_id": "v2-eval/flight-planning/plan_multi_leg_flight/04-validation_boundary", "split": "evaluation", "skill": "flight-planning", "operation": "plan_multi_leg_flight", "case_kind": "validation_boundary", "expected_status": "success", "reference_input": { "waypoints": [ { "iata": "DEN" }, { "iata": "LAS" }, { "iata": "LAX" } ], "aircraft": { "ac_type": "A320" } }, "answers": [ { "quantity": "total great-circle distance", "value": 1389.35, "unit": "km", "accepted_units": [ "km" ], "grader": "scalar", "tolerance": { "absolute": 0.05, "relative": 0.000003, "rationale": "Operation-specific acceptance bound from the canonical dossier for total great-circle distance; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.total_distance_km", "runtime_source_path": "$.total_distance_km", "source_citations": [ "airportsdata", "nist-sp811" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "1571bb91287d066fa81e42e4be6be4d426b78870b785ddb07dff6c4d12c8ae74" }, { "task_id": "v2-eval/flight-planning/plan_multi_leg_flight/05-metamorphic_baseline", "split": "evaluation", "skill": "flight-planning", "operation": "plan_multi_leg_flight", "case_kind": "metamorphic_baseline", "expected_status": "success", "reference_input": { "waypoints": [ { "iata": "BOS" }, { "iata": "JFK" }, { "iata": "DEN" } ], "aircraft": { "ac_type": "A320" } }, "answers": [ { "quantity": "total great-circle distance", "value": 2909.8, "unit": "km", "accepted_units": [ "km" ], "grader": "scalar", "tolerance": { "absolute": 0.05, "relative": 0.000003, "rationale": "Operation-specific acceptance bound from the canonical dossier for total great-circle distance; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.total_distance_km", "runtime_source_path": "$.total_distance_km", "source_citations": [ "airportsdata", "nist-sp811" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "1571bb91287d066fa81e42e4be6be4d426b78870b785ddb07dff6c4d12c8ae74" }, { "task_id": "v2-eval/flight-planning/plan_multi_leg_flight/06-metamorphic_perturbation", "split": "evaluation", "skill": "flight-planning", "operation": "plan_multi_leg_flight", "case_kind": "metamorphic_perturbation", "expected_status": "success", "reference_input": { "waypoints": [ { "iata": "SJC" }, { "iata": "SEA" }, { "iata": "DEN" } ], "aircraft": { "ac_type": "A320" } }, "answers": [ { "quantity": "total great-circle distance", "value": 2767.34, "unit": "km", "accepted_units": [ "km" ], "grader": "scalar", "tolerance": { "absolute": 0.05, "relative": 0.000003, "rationale": "Operation-specific acceptance bound from the canonical dossier for total great-circle distance; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.total_distance_km", "runtime_source_path": "$.total_distance_km", "source_citations": [ "airportsdata", "nist-sp811" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "1571bb91287d066fa81e42e4be6be4d426b78870b785ddb07dff6c4d12c8ae74" }, { "task_id": "v2-eval/flight-planning/plan_multi_leg_flight/07-underspecified", "split": "evaluation", "skill": "flight-planning", "operation": "plan_multi_leg_flight", "case_kind": "underspecified", "expected_status": "needs_clarification", "reference_input": null, "answers": [], "status_rationale": "The prompt intentionally omits information needed to determine a unique numerical or retrieval answer, so guessing would invalidate the comparison.", "tool_required": false, "dossier_sha256": "1571bb91287d066fa81e42e4be6be4d426b78870b785ddb07dff6c4d12c8ae74" }, { "task_id": "v2-eval/flight-planning/plan_multi_leg_flight/08-invalid_or_out_of_domain", "split": "evaluation", "skill": "flight-planning", "operation": "plan_multi_leg_flight", "case_kind": "invalid_or_out_of_domain", "expected_status": "out_of_domain", "reference_input": null, "answers": [], "status_rationale": "Every public operation explicitly prohibits certification, operational control, and safety-of-life use; this request crosses that documented boundary.", "tool_required": false, "dossier_sha256": "1571bb91287d066fa81e42e4be6be4d426b78870b785ddb07dff6c4d12c8ae74" }, { "task_id": "v2-eval/flight-planning/calculate_distance/01-canonical_reference", "split": "evaluation", "skill": "flight-planning", "operation": "calculate_distance", "case_kind": "canonical_reference", "expected_status": "success", "reference_input": { "lat1": 37.6213, "lon1": -122.379, "lat2": 34.0522, "lon2": -118.2437 }, "answers": [ { "quantity": "great-circle distance", "value": 544.36, "unit": "km", "accepted_units": [ "km" ], "grader": "scalar", "tolerance": { "absolute": 0.002, "relative": 5e-7, "rationale": "Operation-specific acceptance bound from the canonical dossier for great-circle distance; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.distance_km", "runtime_source_path": "$.distance_km", "source_citations": [ "airportsdata", "nist-sp811" ] }, { "quantity": "initial bearing", "value": 135.564, "unit": "deg", "accepted_units": [ "deg" ], "grader": "scalar", "tolerance": { "absolute": 0.001, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for initial bearing; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.initial_bearing_deg", "runtime_source_path": "$.initial_bearing_deg", "source_citations": [ "airportsdata", "nist-sp811" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "9bb9240e8beeac4e16d42b9b315ba2767c58903d7d1ae32d0fee0a883ef557c5" }, { "task_id": "v2-eval/flight-planning/calculate_distance/02-realistic_in_domain", "split": "evaluation", "skill": "flight-planning", "operation": "calculate_distance", "case_kind": "realistic_in_domain", "expected_status": "success", "reference_input": { "lat1": 37.6213, "lon1": -122.379, "lat2": 33.9425, "lon2": -118.2437 }, "answers": [ { "quantity": "great-circle distance", "value": 553.483, "unit": "km", "accepted_units": [ "km" ], "grader": "scalar", "tolerance": { "absolute": 0.002, "relative": 5e-7, "rationale": "Operation-specific acceptance bound from the canonical dossier for great-circle distance; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.distance_km", "runtime_source_path": "$.distance_km", "source_citations": [ "airportsdata", "nist-sp811" ] }, { "quantity": "initial bearing", "value": 136.411, "unit": "deg", "accepted_units": [ "deg" ], "grader": "scalar", "tolerance": { "absolute": 0.001, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for initial bearing; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.initial_bearing_deg", "runtime_source_path": "$.initial_bearing_deg", "source_citations": [ "airportsdata", "nist-sp811" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "9bb9240e8beeac4e16d42b9b315ba2767c58903d7d1ae32d0fee0a883ef557c5" }, { "task_id": "v2-eval/flight-planning/calculate_distance/03-alternate_units", "split": "evaluation", "skill": "flight-planning", "operation": "calculate_distance", "case_kind": "alternate_units", "expected_status": "success", "reference_input": { "lat1": 37.6213, "lon1": -122.379, "lat2": 36.08004, "lon2": -118.2437 }, "answers": [ { "quantity": "great-circle distance", "value": 405.854, "unit": "km", "accepted_units": [ "km" ], "grader": "scalar", "tolerance": { "absolute": 0.002, "relative": 5e-7, "rationale": "Operation-specific acceptance bound from the canonical dossier for great-circle distance; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.distance_km", "runtime_source_path": "$.distance_km", "source_citations": [ "airportsdata", "nist-sp811" ] }, { "quantity": "initial bearing", "value": 113.724, "unit": "deg", "accepted_units": [ "deg" ], "grader": "scalar", "tolerance": { "absolute": 0.001, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for initial bearing; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.initial_bearing_deg", "runtime_source_path": "$.initial_bearing_deg", "source_citations": [ "airportsdata", "nist-sp811" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "9bb9240e8beeac4e16d42b9b315ba2767c58903d7d1ae32d0fee0a883ef557c5" }, { "task_id": "v2-eval/flight-planning/calculate_distance/04-validation_boundary", "split": "evaluation", "skill": "flight-planning", "operation": "calculate_distance", "case_kind": "validation_boundary", "expected_status": "success", "reference_input": { "lat1": 37.6213, "lon1": -122.379, "lat2": 39.86167, "lon2": -118.2437 }, "answers": [ { "quantity": "great-circle distance", "value": 436.603, "unit": "km", "accepted_units": [ "km" ], "grader": "scalar", "tolerance": { "absolute": 0.002, "relative": 5e-7, "rationale": "Operation-specific acceptance bound from the canonical dossier for great-circle distance; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.distance_km", "runtime_source_path": "$.distance_km", "source_citations": [ "airportsdata", "nist-sp811" ] }, { "quantity": "initial bearing", "value": 53.935, "unit": "deg", "accepted_units": [ "deg" ], "grader": "scalar", "tolerance": { "absolute": 0.001, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for initial bearing; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.initial_bearing_deg", "runtime_source_path": "$.initial_bearing_deg", "source_citations": [ "airportsdata", "nist-sp811" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "9bb9240e8beeac4e16d42b9b315ba2767c58903d7d1ae32d0fee0a883ef557c5" }, { "task_id": "v2-eval/flight-planning/calculate_distance/05-metamorphic_baseline", "split": "evaluation", "skill": "flight-planning", "operation": "calculate_distance", "case_kind": "metamorphic_baseline", "expected_status": "success", "reference_input": { "lat1": 37.6213, "lon1": -122.379, "lat2": 42.36294, "lon2": -118.2437 }, "answers": [ { "quantity": "great-circle distance", "value": 633.919, "unit": "km", "accepted_units": [ "km" ], "grader": "scalar", "tolerance": { "absolute": 0.002, "relative": 5e-7, "rationale": "Operation-specific acceptance bound from the canonical dossier for great-circle distance; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.distance_km", "runtime_source_path": "$.distance_km", "source_citations": [ "airportsdata", "nist-sp811" ] }, { "quantity": "initial bearing", "value": 32.438, "unit": "deg", "accepted_units": [ "deg" ], "grader": "scalar", "tolerance": { "absolute": 0.001, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for initial bearing; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.initial_bearing_deg", "runtime_source_path": "$.initial_bearing_deg", "source_citations": [ "airportsdata", "nist-sp811" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "9bb9240e8beeac4e16d42b9b315ba2767c58903d7d1ae32d0fee0a883ef557c5" }, { "task_id": "v2-eval/flight-planning/calculate_distance/06-metamorphic_perturbation", "split": "evaluation", "skill": "flight-planning", "operation": "calculate_distance", "case_kind": "metamorphic_perturbation", "expected_status": "success", "reference_input": { "lat1": 37.6213, "lon1": -122.379, "lat2": 47.44989, "lon2": -118.2437 }, "answers": [ { "quantity": "great-circle distance", "value": 1143.764, "unit": "km", "accepted_units": [ "km" ], "grader": "scalar", "tolerance": { "absolute": 0.002, "relative": 5e-7, "rationale": "Operation-specific acceptance bound from the canonical dossier for great-circle distance; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.distance_km", "runtime_source_path": "$.distance_km", "source_citations": [ "airportsdata", "nist-sp811" ] }, { "quantity": "initial bearing", "value": 15.848, "unit": "deg", "accepted_units": [ "deg" ], "grader": "scalar", "tolerance": { "absolute": 0.001, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for initial bearing; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.initial_bearing_deg", "runtime_source_path": "$.initial_bearing_deg", "source_citations": [ "airportsdata", "nist-sp811" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "9bb9240e8beeac4e16d42b9b315ba2767c58903d7d1ae32d0fee0a883ef557c5" }, { "task_id": "v2-eval/flight-planning/calculate_distance/07-underspecified", "split": "evaluation", "skill": "flight-planning", "operation": "calculate_distance", "case_kind": "underspecified", "expected_status": "needs_clarification", "reference_input": null, "answers": [], "status_rationale": "The prompt intentionally omits information needed to determine a unique numerical or retrieval answer, so guessing would invalidate the comparison.", "tool_required": false, "dossier_sha256": "9bb9240e8beeac4e16d42b9b315ba2767c58903d7d1ae32d0fee0a883ef557c5" }, { "task_id": "v2-eval/flight-planning/calculate_distance/08-invalid_or_out_of_domain", "split": "evaluation", "skill": "flight-planning", "operation": "calculate_distance", "case_kind": "invalid_or_out_of_domain", "expected_status": "out_of_domain", "reference_input": null, "answers": [], "status_rationale": "Every public operation explicitly prohibits certification, operational control, and safety-of-life use; this request crosses that documented boundary.", "tool_required": false, "dossier_sha256": "9bb9240e8beeac4e16d42b9b315ba2767c58903d7d1ae32d0fee0a883ef557c5" }, { "task_id": "v2-eval/flight-planning/get_aircraft_performance/01-canonical_reference", "split": "evaluation", "skill": "flight-planning", "operation": "get_aircraft_performance", "case_kind": "canonical_reference", "expected_status": "success", "reference_input": { "aircraft_type": "A320", "distance_km": 500 }, "answers": [ { "quantity": "representative block time", "value": 78, "unit": "min", "accepted_units": [ "min" ], "grader": "scalar", "tolerance": { "absolute": 0.000001, "relative": 1e-9, "rationale": "Operation-specific acceptance bound from the canonical dossier for representative block time; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.block_time_min", "runtime_source_path": "$.block_time_min", "source_citations": [ "aerospace-skills-contract" ] }, { "quantity": "representative fuel estimate", "value": 2374.9, "unit": "kg", "accepted_units": [ "kg" ], "grader": "scalar", "tolerance": { "absolute": 0.000001, "relative": 1e-9, "rationale": "Operation-specific acceptance bound from the canonical dossier for representative fuel estimate; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.estimated_fuel_kg", "runtime_source_path": "$.estimated_fuel_kg", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "6597b52bceeaec3345a9eaa0277c4c30997a91463affa65dfbc363705afb01ba" }, { "task_id": "v2-eval/flight-planning/get_aircraft_performance/02-realistic_in_domain", "split": "evaluation", "skill": "flight-planning", "operation": "get_aircraft_performance", "case_kind": "realistic_in_domain", "expected_status": "success", "reference_input": { "aircraft_type": "A320", "distance_km": 250 }, "answers": [ { "quantity": "representative block time", "value": 60, "unit": "min", "accepted_units": [ "min" ], "grader": "scalar", "tolerance": { "absolute": 0.000001, "relative": 1e-9, "rationale": "Operation-specific acceptance bound from the canonical dossier for representative block time; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.block_time_min", "runtime_source_path": "$.block_time_min", "source_citations": [ "aerospace-skills-contract" ] }, { "quantity": "representative fuel estimate", "value": 1624.9, "unit": "kg", "accepted_units": [ "kg" ], "grader": "scalar", "tolerance": { "absolute": 0.000001, "relative": 1e-9, "rationale": "Operation-specific acceptance bound from the canonical dossier for representative fuel estimate; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.estimated_fuel_kg", "runtime_source_path": "$.estimated_fuel_kg", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "6597b52bceeaec3345a9eaa0277c4c30997a91463affa65dfbc363705afb01ba" }, { "task_id": "v2-eval/flight-planning/get_aircraft_performance/03-alternate_units", "split": "evaluation", "skill": "flight-planning", "operation": "get_aircraft_performance", "case_kind": "alternate_units", "expected_status": "success", "reference_input": { "aircraft_type": "A320", "distance_km": 750 }, "answers": [ { "quantity": "representative block time", "value": 96, "unit": "min", "accepted_units": [ "min" ], "grader": "scalar", "tolerance": { "absolute": 0.000001, "relative": 1e-9, "rationale": "Operation-specific acceptance bound from the canonical dossier for representative block time; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.block_time_min", "runtime_source_path": "$.block_time_min", "source_citations": [ "aerospace-skills-contract" ] }, { "quantity": "representative fuel estimate", "value": 3124.8, "unit": "kg", "accepted_units": [ "kg" ], "grader": "scalar", "tolerance": { "absolute": 0.000001, "relative": 1e-9, "rationale": "Operation-specific acceptance bound from the canonical dossier for representative fuel estimate; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.estimated_fuel_kg", "runtime_source_path": "$.estimated_fuel_kg", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "6597b52bceeaec3345a9eaa0277c4c30997a91463affa65dfbc363705afb01ba" }, { "task_id": "v2-eval/flight-planning/get_aircraft_performance/04-validation_boundary", "split": "evaluation", "skill": "flight-planning", "operation": "get_aircraft_performance", "case_kind": "validation_boundary", "expected_status": "success", "reference_input": { "aircraft_type": "A320", "distance_km": 1500 }, "answers": [ { "quantity": "representative block time", "value": 150, "unit": "min", "accepted_units": [ "min" ], "grader": "scalar", "tolerance": { "absolute": 0.000001, "relative": 1e-9, "rationale": "Operation-specific acceptance bound from the canonical dossier for representative block time; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.block_time_min", "runtime_source_path": "$.block_time_min", "source_citations": [ "aerospace-skills-contract" ] }, { "quantity": "representative fuel estimate", "value": 5374.6, "unit": "kg", "accepted_units": [ "kg" ], "grader": "scalar", "tolerance": { "absolute": 0.000001, "relative": 1e-9, "rationale": "Operation-specific acceptance bound from the canonical dossier for representative fuel estimate; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.estimated_fuel_kg", "runtime_source_path": "$.estimated_fuel_kg", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "6597b52bceeaec3345a9eaa0277c4c30997a91463affa65dfbc363705afb01ba" }, { "task_id": "v2-eval/flight-planning/get_aircraft_performance/05-metamorphic_baseline", "split": "evaluation", "skill": "flight-planning", "operation": "get_aircraft_performance", "case_kind": "metamorphic_baseline", "expected_status": "success", "reference_input": { "aircraft_type": "A320", "distance_km": 3000 }, "answers": [ { "quantity": "representative block time", "value": 258, "unit": "min", "accepted_units": [ "min" ], "grader": "scalar", "tolerance": { "absolute": 0.000001, "relative": 1e-9, "rationale": "Operation-specific acceptance bound from the canonical dossier for representative block time; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.block_time_min", "runtime_source_path": "$.block_time_min", "source_citations": [ "aerospace-skills-contract" ] }, { "quantity": "representative fuel estimate", "value": 9874.3, "unit": "kg", "accepted_units": [ "kg" ], "grader": "scalar", "tolerance": { "absolute": 0.000001, "relative": 1e-9, "rationale": "Operation-specific acceptance bound from the canonical dossier for representative fuel estimate; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.estimated_fuel_kg", "runtime_source_path": "$.estimated_fuel_kg", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "6597b52bceeaec3345a9eaa0277c4c30997a91463affa65dfbc363705afb01ba" }, { "task_id": "v2-eval/flight-planning/get_aircraft_performance/06-metamorphic_perturbation", "split": "evaluation", "skill": "flight-planning", "operation": "get_aircraft_performance", "case_kind": "metamorphic_perturbation", "expected_status": "success", "reference_input": { "aircraft_type": "A320", "distance_km": 5000 }, "answers": [ { "quantity": "representative block time", "value": 402, "unit": "min", "accepted_units": [ "min" ], "grader": "scalar", "tolerance": { "absolute": 0.000001, "relative": 1e-9, "rationale": "Operation-specific acceptance bound from the canonical dossier for representative block time; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.block_time_min", "runtime_source_path": "$.block_time_min", "source_citations": [ "aerospace-skills-contract" ] }, { "quantity": "representative fuel estimate", "value": 15873.8, "unit": "kg", "accepted_units": [ "kg" ], "grader": "scalar", "tolerance": { "absolute": 0.000001, "relative": 1e-9, "rationale": "Operation-specific acceptance bound from the canonical dossier for representative fuel estimate; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.estimated_fuel_kg", "runtime_source_path": "$.estimated_fuel_kg", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "6597b52bceeaec3345a9eaa0277c4c30997a91463affa65dfbc363705afb01ba" }, { "task_id": "v2-eval/flight-planning/get_aircraft_performance/07-underspecified", "split": "evaluation", "skill": "flight-planning", "operation": "get_aircraft_performance", "case_kind": "underspecified", "expected_status": "needs_clarification", "reference_input": null, "answers": [], "status_rationale": "The prompt intentionally omits information needed to determine a unique numerical or retrieval answer, so guessing would invalidate the comparison.", "tool_required": false, "dossier_sha256": "6597b52bceeaec3345a9eaa0277c4c30997a91463affa65dfbc363705afb01ba" }, { "task_id": "v2-eval/flight-planning/get_aircraft_performance/08-invalid_or_out_of_domain", "split": "evaluation", "skill": "flight-planning", "operation": "get_aircraft_performance", "case_kind": "invalid_or_out_of_domain", "expected_status": "out_of_domain", "reference_input": null, "answers": [], "status_rationale": "Every public operation explicitly prohibits certification, operational control, and safety-of-life use; this request crosses that documented boundary.", "tool_required": false, "dossier_sha256": "6597b52bceeaec3345a9eaa0277c4c30997a91463affa65dfbc363705afb01ba" }, { "task_id": "v2-eval/flight-planning/get_aircraft_database/01-canonical_reference", "split": "evaluation", "skill": "flight-planning", "operation": "get_aircraft_database", "case_kind": "canonical_reference", "expected_status": "success", "reference_input": {}, "answers": [ { "quantity": "matching representative aircraft records", "value": [ { "icao": "A320", "label": "Airbus A320 family", "cruiseKts": 450, "fuelKgHr": 2500, "climbMin": 22, "descentMin": 20 }, { "icao": "A359", "label": "Airbus A350-900", "cruiseKts": 488, "fuelKgHr": 5800, "climbMin": 27, "descentMin": 25 }, { "icao": "B738", "label": "Boeing 737-800", "cruiseKts": 453, "fuelKgHr": 2600, "climbMin": 23, "descentMin": 20 }, { "icao": "B744", "label": "Boeing 747-400", "cruiseKts": 493, "fuelKgHr": 10200, "climbMin": 28, "descentMin": 26 }, { "icao": "B777", "label": "Boeing 777 family", "cruiseKts": 490, "fuelKgHr": 6800, "climbMin": 27, "descentMin": 25 }, { "icao": "E190", "label": "Embraer E190", "cruiseKts": 447, "fuelKgHr": 1900, "climbMin": 20, "descentMin": 18 }, { "icao": "C172", "label": "Cessna 172", "cruiseKts": 122, "fuelKgHr": 32, "climbMin": 12, "descentMin": 10 } ], "unit": "records", "accepted_units": [ "records" ], "grader": "retrieval", "tolerance": { "absolute": 0, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for matching representative aircraft records; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $", "runtime_source_path": "$", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": false, "dossier_sha256": "25860cc43f8e3cf691163dd20facf6e161a3bf6febb766326c18a447ace8148e" }, { "task_id": "v2-eval/flight-planning/get_aircraft_database/02-realistic_in_domain", "split": "evaluation", "skill": "flight-planning", "operation": "get_aircraft_database", "case_kind": "realistic_in_domain", "expected_status": "success", "reference_input": { "search": "A320" }, "answers": [ { "quantity": "matching representative aircraft records", "value": [ { "icao": "A320", "label": "Airbus A320 family", "cruiseKts": 450, "fuelKgHr": 2500, "climbMin": 22, "descentMin": 20 } ], "unit": "records", "accepted_units": [ "records" ], "grader": "retrieval", "tolerance": { "absolute": 0, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for matching representative aircraft records; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $", "runtime_source_path": "$", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": false, "dossier_sha256": "25860cc43f8e3cf691163dd20facf6e161a3bf6febb766326c18a447ace8148e" }, { "task_id": "v2-eval/flight-planning/get_aircraft_database/03-alternate_units", "split": "evaluation", "skill": "flight-planning", "operation": "get_aircraft_database", "case_kind": "alternate_units", "expected_status": "success", "reference_input": { "search": "Airbus" }, "answers": [ { "quantity": "matching representative aircraft records", "value": [ { "icao": "A320", "label": "Airbus A320 family", "cruiseKts": 450, "fuelKgHr": 2500, "climbMin": 22, "descentMin": 20 }, { "icao": "A359", "label": "Airbus A350-900", "cruiseKts": 488, "fuelKgHr": 5800, "climbMin": 27, "descentMin": 25 } ], "unit": "records", "accepted_units": [ "records" ], "grader": "retrieval", "tolerance": { "absolute": 0, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for matching representative aircraft records; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $", "runtime_source_path": "$", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": false, "dossier_sha256": "25860cc43f8e3cf691163dd20facf6e161a3bf6febb766326c18a447ace8148e" }, { "task_id": "v2-eval/flight-planning/get_aircraft_database/04-validation_boundary", "split": "evaluation", "skill": "flight-planning", "operation": "get_aircraft_database", "case_kind": "validation_boundary", "expected_status": "success", "reference_input": { "search": "Boeing" }, "answers": [ { "quantity": "matching representative aircraft records", "value": [ { "icao": "B738", "label": "Boeing 737-800", "cruiseKts": 453, "fuelKgHr": 2600, "climbMin": 23, "descentMin": 20 }, { "icao": "B744", "label": "Boeing 747-400", "cruiseKts": 493, "fuelKgHr": 10200, "climbMin": 28, "descentMin": 26 }, { "icao": "B777", "label": "Boeing 777 family", "cruiseKts": 490, "fuelKgHr": 6800, "climbMin": 27, "descentMin": 25 } ], "unit": "records", "accepted_units": [ "records" ], "grader": "retrieval", "tolerance": { "absolute": 0, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for matching representative aircraft records; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $", "runtime_source_path": "$", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": false, "dossier_sha256": "25860cc43f8e3cf691163dd20facf6e161a3bf6febb766326c18a447ace8148e" }, { "task_id": "v2-eval/flight-planning/get_aircraft_database/05-metamorphic_baseline", "split": "evaluation", "skill": "flight-planning", "operation": "get_aircraft_database", "case_kind": "metamorphic_baseline", "expected_status": "success", "reference_input": { "search": "Cessna" }, "answers": [ { "quantity": "matching representative aircraft records", "value": [ { "icao": "C172", "label": "Cessna 172", "cruiseKts": 122, "fuelKgHr": 32, "climbMin": 12, "descentMin": 10 } ], "unit": "records", "accepted_units": [ "records" ], "grader": "retrieval", "tolerance": { "absolute": 0, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for matching representative aircraft records; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $", "runtime_source_path": "$", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": false, "dossier_sha256": "25860cc43f8e3cf691163dd20facf6e161a3bf6febb766326c18a447ace8148e" }, { "task_id": "v2-eval/flight-planning/get_aircraft_database/06-metamorphic_perturbation", "split": "evaluation", "skill": "flight-planning", "operation": "get_aircraft_database", "case_kind": "metamorphic_perturbation", "expected_status": "success", "reference_input": { "search": "Embraer" }, "answers": [ { "quantity": "matching representative aircraft records", "value": [ { "icao": "E190", "label": "Embraer E190", "cruiseKts": 447, "fuelKgHr": 1900, "climbMin": 20, "descentMin": 18 } ], "unit": "records", "accepted_units": [ "records" ], "grader": "retrieval", "tolerance": { "absolute": 0, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for matching representative aircraft records; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $", "runtime_source_path": "$", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": false, "dossier_sha256": "25860cc43f8e3cf691163dd20facf6e161a3bf6febb766326c18a447ace8148e" }, { "task_id": "v2-eval/flight-planning/get_aircraft_database/07-underspecified", "split": "evaluation", "skill": "flight-planning", "operation": "get_aircraft_database", "case_kind": "underspecified", "expected_status": "needs_clarification", "reference_input": null, "answers": [], "status_rationale": "The prompt intentionally omits information needed to determine a unique numerical or retrieval answer, so guessing would invalidate the comparison.", "tool_required": false, "dossier_sha256": "25860cc43f8e3cf691163dd20facf6e161a3bf6febb766326c18a447ace8148e" }, { "task_id": "v2-eval/flight-planning/get_aircraft_database/08-invalid_or_out_of_domain", "split": "evaluation", "skill": "flight-planning", "operation": "get_aircraft_database", "case_kind": "invalid_or_out_of_domain", "expected_status": "out_of_domain", "reference_input": null, "answers": [], "status_rationale": "Every public operation explicitly prohibits certification, operational control, and safety-of-life use; this request crosses that documented boundary.", "tool_required": false, "dossier_sha256": "25860cc43f8e3cf691163dd20facf6e161a3bf6febb766326c18a447ace8148e" }, { "task_id": "v2-eval/flight-planning/get_system_status/01-canonical_reference", "split": "evaluation", "skill": "flight-planning", "operation": "get_system_status", "case_kind": "canonical_reference", "expected_status": "success", "reference_input": {}, "answers": [ { "quantity": "packed runtime status", "value": { "runtime": "node", "skills": 10, "calculations": 45, "local_execution": true, "network_required": false }, "unit": "records", "accepted_units": [ "records" ], "grader": "retrieval", "tolerance": { "absolute": 0, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for packed runtime status; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $", "runtime_source_path": "$", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": false, "dossier_sha256": "758ba3dfdcfde3b44dbe25590c5618b2732a2e15ea4c1e3693d83316cdf0fef1" }, { "task_id": "v2-eval/flight-planning/get_system_status/02-realistic_in_domain", "split": "evaluation", "skill": "flight-planning", "operation": "get_system_status", "case_kind": "realistic_in_domain", "expected_status": "success", "reference_input": {}, "answers": [ { "quantity": "packed runtime status", "value": { "runtime": "node", "skills": 10, "calculations": 45, "local_execution": true, "network_required": false }, "unit": "records", "accepted_units": [ "records" ], "grader": "retrieval", "tolerance": { "absolute": 0, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for packed runtime status; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $", "runtime_source_path": "$", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": false, "dossier_sha256": "758ba3dfdcfde3b44dbe25590c5618b2732a2e15ea4c1e3693d83316cdf0fef1" }, { "task_id": "v2-eval/flight-planning/get_system_status/03-alternate_units", "split": "evaluation", "skill": "flight-planning", "operation": "get_system_status", "case_kind": "alternate_units", "expected_status": "success", "reference_input": {}, "answers": [ { "quantity": "packed runtime status", "value": { "runtime": "node", "skills": 10, "calculations": 45, "local_execution": true, "network_required": false }, "unit": "records", "accepted_units": [ "records" ], "grader": "retrieval", "tolerance": { "absolute": 0, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for packed runtime status; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $", "runtime_source_path": "$", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": false, "dossier_sha256": "758ba3dfdcfde3b44dbe25590c5618b2732a2e15ea4c1e3693d83316cdf0fef1" }, { "task_id": "v2-eval/flight-planning/get_system_status/04-validation_boundary", "split": "evaluation", "skill": "flight-planning", "operation": "get_system_status", "case_kind": "validation_boundary", "expected_status": "success", "reference_input": {}, "answers": [ { "quantity": "packed runtime status", "value": { "runtime": "node", "skills": 10, "calculations": 45, "local_execution": true, "network_required": false }, "unit": "records", "accepted_units": [ "records" ], "grader": "retrieval", "tolerance": { "absolute": 0, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for packed runtime status; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $", "runtime_source_path": "$", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": false, "dossier_sha256": "758ba3dfdcfde3b44dbe25590c5618b2732a2e15ea4c1e3693d83316cdf0fef1" }, { "task_id": "v2-eval/flight-planning/get_system_status/05-metamorphic_baseline", "split": "evaluation", "skill": "flight-planning", "operation": "get_system_status", "case_kind": "metamorphic_baseline", "expected_status": "success", "reference_input": {}, "answers": [ { "quantity": "packed runtime status", "value": { "runtime": "node", "skills": 10, "calculations": 45, "local_execution": true, "network_required": false }, "unit": "records", "accepted_units": [ "records" ], "grader": "retrieval", "tolerance": { "absolute": 0, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for packed runtime status; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $", "runtime_source_path": "$", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": false, "dossier_sha256": "758ba3dfdcfde3b44dbe25590c5618b2732a2e15ea4c1e3693d83316cdf0fef1" }, { "task_id": "v2-eval/flight-planning/get_system_status/06-metamorphic_perturbation", "split": "evaluation", "skill": "flight-planning", "operation": "get_system_status", "case_kind": "metamorphic_perturbation", "expected_status": "success", "reference_input": {}, "answers": [ { "quantity": "packed runtime status", "value": { "runtime": "node", "skills": 10, "calculations": 45, "local_execution": true, "network_required": false }, "unit": "records", "accepted_units": [ "records" ], "grader": "retrieval", "tolerance": { "absolute": 0, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for packed runtime status; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $", "runtime_source_path": "$", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": false, "dossier_sha256": "758ba3dfdcfde3b44dbe25590c5618b2732a2e15ea4c1e3693d83316cdf0fef1" }, { "task_id": "v2-eval/flight-planning/get_system_status/07-underspecified", "split": "evaluation", "skill": "flight-planning", "operation": "get_system_status", "case_kind": "underspecified", "expected_status": "needs_clarification", "reference_input": null, "answers": [], "status_rationale": "The prompt intentionally omits information needed to determine a unique numerical or retrieval answer, so guessing would invalidate the comparison.", "tool_required": false, "dossier_sha256": "758ba3dfdcfde3b44dbe25590c5618b2732a2e15ea4c1e3693d83316cdf0fef1" }, { "task_id": "v2-eval/flight-planning/get_system_status/08-invalid_or_out_of_domain", "split": "evaluation", "skill": "flight-planning", "operation": "get_system_status", "case_kind": "invalid_or_out_of_domain", "expected_status": "out_of_domain", "reference_input": null, "answers": [], "status_rationale": "Every public operation explicitly prohibits certification, operational control, and safety-of-life use; this request crosses that documented boundary.", "tool_required": false, "dossier_sha256": "758ba3dfdcfde3b44dbe25590c5618b2732a2e15ea4c1e3693d83316cdf0fef1" }, { "task_id": "v2-eval/flight-planning/convert_units/01-canonical_reference", "split": "evaluation", "skill": "flight-planning", "operation": "convert_units", "case_kind": "canonical_reference", "expected_status": "success", "reference_input": { "value": 100, "from_unit": "nmi", "to_unit": "km" }, "answers": [ { "quantity": "converted value", "value": 185.2, "unit": "km", "accepted_units": [ "km" ], "grader": "scalar", "tolerance": { "absolute": 1e-9, "relative": 1e-9, "rationale": "Operation-specific acceptance bound from the canonical dossier for converted value; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.value", "runtime_source_path": "$.value", "source_citations": [ "nist-sp811" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "400e0e4297415c9ce8f66883511a748f5d80bdb42e21091b0feaed21acc70d5c" }, { "task_id": "v2-eval/flight-planning/convert_units/02-realistic_in_domain", "split": "evaluation", "skill": "flight-planning", "operation": "convert_units", "case_kind": "realistic_in_domain", "expected_status": "success", "reference_input": { "value": 1, "from_unit": "nmi", "to_unit": "km" }, "answers": [ { "quantity": "converted value", "value": 1.852, "unit": "km", "accepted_units": [ "km" ], "grader": "scalar", "tolerance": { "absolute": 1e-9, "relative": 1e-9, "rationale": "Operation-specific acceptance bound from the canonical dossier for converted value; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.value", "runtime_source_path": "$.value", "source_citations": [ "nist-sp811" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "400e0e4297415c9ce8f66883511a748f5d80bdb42e21091b0feaed21acc70d5c" }, { "task_id": "v2-eval/flight-planning/convert_units/03-alternate_units", "split": "evaluation", "skill": "flight-planning", "operation": "convert_units", "case_kind": "alternate_units", "expected_status": "success", "reference_input": { "value": 10, "from_unit": "nmi", "to_unit": "km" }, "answers": [ { "quantity": "converted value", "value": 18.52, "unit": "km", "accepted_units": [ "km" ], "grader": "scalar", "tolerance": { "absolute": 1e-9, "relative": 1e-9, "rationale": "Operation-specific acceptance bound from the canonical dossier for converted value; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.value", "runtime_source_path": "$.value", "source_citations": [ "nist-sp811" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "400e0e4297415c9ce8f66883511a748f5d80bdb42e21091b0feaed21acc70d5c" }, { "task_id": "v2-eval/flight-planning/convert_units/04-validation_boundary", "split": "evaluation", "skill": "flight-planning", "operation": "convert_units", "case_kind": "validation_boundary", "expected_status": "success", "reference_input": { "value": 250, "from_unit": "nmi", "to_unit": "km" }, "answers": [ { "quantity": "converted value", "value": 463, "unit": "km", "accepted_units": [ "km" ], "grader": "scalar", "tolerance": { "absolute": 1e-9, "relative": 1e-9, "rationale": "Operation-specific acceptance bound from the canonical dossier for converted value; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.value", "runtime_source_path": "$.value", "source_citations": [ "nist-sp811" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "400e0e4297415c9ce8f66883511a748f5d80bdb42e21091b0feaed21acc70d5c" }, { "task_id": "v2-eval/flight-planning/convert_units/05-metamorphic_baseline", "split": "evaluation", "skill": "flight-planning", "operation": "convert_units", "case_kind": "metamorphic_baseline", "expected_status": "success", "reference_input": { "value": -40, "from_unit": "nmi", "to_unit": "km" }, "answers": [ { "quantity": "converted value", "value": -74.08, "unit": "km", "accepted_units": [ "km" ], "grader": "scalar", "tolerance": { "absolute": 1e-9, "relative": 1e-9, "rationale": "Operation-specific acceptance bound from the canonical dossier for converted value; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.value", "runtime_source_path": "$.value", "source_citations": [ "nist-sp811" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "400e0e4297415c9ce8f66883511a748f5d80bdb42e21091b0feaed21acc70d5c" }, { "task_id": "v2-eval/flight-planning/convert_units/06-metamorphic_perturbation", "split": "evaluation", "skill": "flight-planning", "operation": "convert_units", "case_kind": "metamorphic_perturbation", "expected_status": "success", "reference_input": { "value": 1000, "from_unit": "nmi", "to_unit": "km" }, "answers": [ { "quantity": "converted value", "value": 1852, "unit": "km", "accepted_units": [ "km" ], "grader": "scalar", "tolerance": { "absolute": 1e-9, "relative": 1e-9, "rationale": "Operation-specific acceptance bound from the canonical dossier for converted value; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.value", "runtime_source_path": "$.value", "source_citations": [ "nist-sp811" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "400e0e4297415c9ce8f66883511a748f5d80bdb42e21091b0feaed21acc70d5c" }, { "task_id": "v2-eval/flight-planning/convert_units/07-underspecified", "split": "evaluation", "skill": "flight-planning", "operation": "convert_units", "case_kind": "underspecified", "expected_status": "needs_clarification", "reference_input": null, "answers": [], "status_rationale": "The prompt intentionally omits information needed to determine a unique numerical or retrieval answer, so guessing would invalidate the comparison.", "tool_required": false, "dossier_sha256": "400e0e4297415c9ce8f66883511a748f5d80bdb42e21091b0feaed21acc70d5c" }, { "task_id": "v2-eval/flight-planning/convert_units/08-invalid_or_out_of_domain", "split": "evaluation", "skill": "flight-planning", "operation": "convert_units", "case_kind": "invalid_or_out_of_domain", "expected_status": "out_of_domain", "reference_input": null, "answers": [], "status_rationale": "Every public operation explicitly prohibits certification, operational control, and safety-of-life use; this request crosses that documented boundary.", "tool_required": false, "dossier_sha256": "400e0e4297415c9ce8f66883511a748f5d80bdb42e21091b0feaed21acc70d5c" }, { "task_id": "v2-eval/atmosphere/get_atmosphere_profile/01-canonical_reference", "split": "evaluation", "skill": "atmosphere", "operation": "get_atmosphere_profile", "case_kind": "canonical_reference", "expected_status": "success", "reference_input": { "altitudes_m": [ 0, 5000, 11000 ] }, "answers": [ { "quantity": "temperature at the first requested altitude", "value": 288.15, "unit": "K", "accepted_units": [ "K" ], "grader": "scalar", "tolerance": { "absolute": 0.01, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for temperature at the first requested altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $[0].temperatureK", "runtime_source_path": "$[0].temperatureK", "source_citations": [ "us-standard-atmosphere-1976" ] }, { "quantity": "density at the first requested altitude", "value": 1.225, "unit": "kg/m^3", "accepted_units": [ "kg/m^3" ], "grader": "scalar", "tolerance": { "absolute": 0.000002, "relative": 0.00005, "rationale": "Operation-specific acceptance bound from the canonical dossier for density at the first requested altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $[0].densityKgM3", "runtime_source_path": "$[0].densityKgM3", "source_citations": [ "us-standard-atmosphere-1976" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "8a2ed2aeee154873072ab85cd38849e9943a5aa76fb0f3b254692b7f99ae46d4" }, { "task_id": "v2-eval/atmosphere/get_atmosphere_profile/02-realistic_in_domain", "split": "evaluation", "skill": "atmosphere", "operation": "get_atmosphere_profile", "case_kind": "realistic_in_domain", "expected_status": "success", "reference_input": { "altitudes_m": [ 0 ] }, "answers": [ { "quantity": "temperature at the first requested altitude", "value": 288.15, "unit": "K", "accepted_units": [ "K" ], "grader": "scalar", "tolerance": { "absolute": 0.01, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for temperature at the first requested altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $[0].temperatureK", "runtime_source_path": "$[0].temperatureK", "source_citations": [ "us-standard-atmosphere-1976" ] }, { "quantity": "density at the first requested altitude", "value": 1.225, "unit": "kg/m^3", "accepted_units": [ "kg/m^3" ], "grader": "scalar", "tolerance": { "absolute": 0.000002, "relative": 0.00005, "rationale": "Operation-specific acceptance bound from the canonical dossier for density at the first requested altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $[0].densityKgM3", "runtime_source_path": "$[0].densityKgM3", "source_citations": [ "us-standard-atmosphere-1976" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "8a2ed2aeee154873072ab85cd38849e9943a5aa76fb0f3b254692b7f99ae46d4" }, { "task_id": "v2-eval/atmosphere/get_atmosphere_profile/03-alternate_units", "split": "evaluation", "skill": "atmosphere", "operation": "get_atmosphere_profile", "case_kind": "alternate_units", "expected_status": "success", "reference_input": { "altitudes_m": [ 1000, 3000 ] }, "answers": [ { "quantity": "temperature at the first requested altitude", "value": 281.651022, "unit": "K", "accepted_units": [ "K" ], "grader": "scalar", "tolerance": { "absolute": 0.01, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for temperature at the first requested altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $[0].temperatureK", "runtime_source_path": "$[0].temperatureK", "source_citations": [ "us-standard-atmosphere-1976" ] }, { "quantity": "density at the first requested altitude", "value": 1.11166, "unit": "kg/m^3", "accepted_units": [ "kg/m^3" ], "grader": "scalar", "tolerance": { "absolute": 0.000002, "relative": 0.00005, "rationale": "Operation-specific acceptance bound from the canonical dossier for density at the first requested altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $[0].densityKgM3", "runtime_source_path": "$[0].densityKgM3", "source_citations": [ "us-standard-atmosphere-1976" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "8a2ed2aeee154873072ab85cd38849e9943a5aa76fb0f3b254692b7f99ae46d4" }, { "task_id": "v2-eval/atmosphere/get_atmosphere_profile/04-validation_boundary", "split": "evaluation", "skill": "atmosphere", "operation": "get_atmosphere_profile", "case_kind": "validation_boundary", "expected_status": "success", "reference_input": { "altitudes_m": [ 10999, 11000 ] }, "answers": [ { "quantity": "temperature at the first requested altitude", "value": 216.77999, "unit": "K", "accepted_units": [ "K" ], "grader": "scalar", "tolerance": { "absolute": 0.01, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for temperature at the first requested altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $[0].temperatureK", "runtime_source_path": "$[0].temperatureK", "source_citations": [ "us-standard-atmosphere-1976" ] }, { "quantity": "density at the first requested altitude", "value": 0.364848, "unit": "kg/m^3", "accepted_units": [ "kg/m^3" ], "grader": "scalar", "tolerance": { "absolute": 0.000002, "relative": 0.00005, "rationale": "Operation-specific acceptance bound from the canonical dossier for density at the first requested altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $[0].densityKgM3", "runtime_source_path": "$[0].densityKgM3", "source_citations": [ "us-standard-atmosphere-1976" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "8a2ed2aeee154873072ab85cd38849e9943a5aa76fb0f3b254692b7f99ae46d4" }, { "task_id": "v2-eval/atmosphere/get_atmosphere_profile/05-metamorphic_baseline", "split": "evaluation", "skill": "atmosphere", "operation": "get_atmosphere_profile", "case_kind": "metamorphic_baseline", "expected_status": "success", "reference_input": { "altitudes_m": [ -500, 0 ] }, "answers": [ { "quantity": "temperature at the first requested altitude", "value": 291.400256, "unit": "K", "accepted_units": [ "K" ], "grader": "scalar", "tolerance": { "absolute": 0.01, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for temperature at the first requested altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $[0].temperatureK", "runtime_source_path": "$[0].temperatureK", "source_citations": [ "us-standard-atmosphere-1976" ] }, { "quantity": "density at the first requested altitude", "value": 1.284895, "unit": "kg/m^3", "accepted_units": [ "kg/m^3" ], "grader": "scalar", "tolerance": { "absolute": 0.000002, "relative": 0.00005, "rationale": "Operation-specific acceptance bound from the canonical dossier for density at the first requested altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $[0].densityKgM3", "runtime_source_path": "$[0].densityKgM3", "source_citations": [ "us-standard-atmosphere-1976" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "8a2ed2aeee154873072ab85cd38849e9943a5aa76fb0f3b254692b7f99ae46d4" }, { "task_id": "v2-eval/atmosphere/get_atmosphere_profile/06-metamorphic_perturbation", "split": "evaluation", "skill": "atmosphere", "operation": "get_atmosphere_profile", "case_kind": "metamorphic_perturbation", "expected_status": "success", "reference_input": { "altitudes_m": [ 20000, 32000 ] }, "answers": [ { "quantity": "temperature at the first requested altitude", "value": 216.65, "unit": "K", "accepted_units": [ "K" ], "grader": "scalar", "tolerance": { "absolute": 0.01, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for temperature at the first requested altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $[0].temperatureK", "runtime_source_path": "$[0].temperatureK", "source_citations": [ "us-standard-atmosphere-1976" ] }, { "quantity": "density at the first requested altitude", "value": 0.08891, "unit": "kg/m^3", "accepted_units": [ "kg/m^3" ], "grader": "scalar", "tolerance": { "absolute": 0.000002, "relative": 0.00005, "rationale": "Operation-specific acceptance bound from the canonical dossier for density at the first requested altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $[0].densityKgM3", "runtime_source_path": "$[0].densityKgM3", "source_citations": [ "us-standard-atmosphere-1976" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "8a2ed2aeee154873072ab85cd38849e9943a5aa76fb0f3b254692b7f99ae46d4" }, { "task_id": "v2-eval/atmosphere/get_atmosphere_profile/07-underspecified", "split": "evaluation", "skill": "atmosphere", "operation": "get_atmosphere_profile", "case_kind": "underspecified", "expected_status": "needs_clarification", "reference_input": null, "answers": [], "status_rationale": "The prompt intentionally omits information needed to determine a unique numerical or retrieval answer, so guessing would invalidate the comparison.", "tool_required": false, "dossier_sha256": "8a2ed2aeee154873072ab85cd38849e9943a5aa76fb0f3b254692b7f99ae46d4" }, { "task_id": "v2-eval/atmosphere/get_atmosphere_profile/08-invalid_or_out_of_domain", "split": "evaluation", "skill": "atmosphere", "operation": "get_atmosphere_profile", "case_kind": "invalid_or_out_of_domain", "expected_status": "out_of_domain", "reference_input": null, "answers": [], "status_rationale": "Every public operation explicitly prohibits certification, operational control, and safety-of-life use; this request crosses that documented boundary.", "tool_required": false, "dossier_sha256": "8a2ed2aeee154873072ab85cd38849e9943a5aa76fb0f3b254692b7f99ae46d4" }, { "task_id": "v2-eval/atmosphere/wind_model_simple/01-canonical_reference", "split": "evaluation", "skill": "atmosphere", "operation": "wind_model_simple", "case_kind": "canonical_reference", "expected_status": "success", "reference_input": { "altitudes_m": [ 10, 100, 500 ], "surface_wind_speed_ms": 6, "surface_wind_direction_deg": 270, "roughness_length_m": 0.03, "model_type": "logarithmic" }, "answers": [ { "quantity": "wind speed at the first requested altitude", "value": 6, "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "scalar", "tolerance": { "absolute": 0.000001, "relative": 0.000001, "rationale": "Operation-specific acceptance bound from the canonical dossier for wind speed at the first requested altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $[0].wind_speed_ms", "runtime_source_path": "$[0].wind_speed_ms", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "705758babba36bbda19b9869a652ce7f2fe3cf59652f8e5eede1275cacab7e63" }, { "task_id": "v2-eval/atmosphere/wind_model_simple/02-realistic_in_domain", "split": "evaluation", "skill": "atmosphere", "operation": "wind_model_simple", "case_kind": "realistic_in_domain", "expected_status": "success", "reference_input": { "altitudes_m": [ 10, 50 ], "surface_wind_speed_ms": 6, "surface_wind_direction_deg": 270, "roughness_length_m": 0.03, "model_type": "logarithmic" }, "answers": [ { "quantity": "wind speed at the first requested altitude", "value": 6, "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "scalar", "tolerance": { "absolute": 0.000001, "relative": 0.000001, "rationale": "Operation-specific acceptance bound from the canonical dossier for wind speed at the first requested altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $[0].wind_speed_ms", "runtime_source_path": "$[0].wind_speed_ms", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "705758babba36bbda19b9869a652ce7f2fe3cf59652f8e5eede1275cacab7e63" }, { "task_id": "v2-eval/atmosphere/wind_model_simple/03-alternate_units", "split": "evaluation", "skill": "atmosphere", "operation": "wind_model_simple", "case_kind": "alternate_units", "expected_status": "success", "reference_input": { "altitudes_m": [ 10, 200 ], "surface_wind_speed_ms": 6, "surface_wind_direction_deg": 270, "roughness_length_m": 0.03, "model_type": "logarithmic" }, "answers": [ { "quantity": "wind speed at the first requested altitude", "value": 6, "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "scalar", "tolerance": { "absolute": 0.000001, "relative": 0.000001, "rationale": "Operation-specific acceptance bound from the canonical dossier for wind speed at the first requested altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $[0].wind_speed_ms", "runtime_source_path": "$[0].wind_speed_ms", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "705758babba36bbda19b9869a652ce7f2fe3cf59652f8e5eede1275cacab7e63" }, { "task_id": "v2-eval/atmosphere/wind_model_simple/04-validation_boundary", "split": "evaluation", "skill": "atmosphere", "operation": "wind_model_simple", "case_kind": "validation_boundary", "expected_status": "success", "reference_input": { "altitudes_m": [ 20, 100, 300 ], "surface_wind_speed_ms": 6, "surface_wind_direction_deg": 270, "roughness_length_m": 0.03, "model_type": "logarithmic" }, "answers": [ { "quantity": "wind speed at the first requested altitude", "value": 6.71592, "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "scalar", "tolerance": { "absolute": 0.000001, "relative": 0.000001, "rationale": "Operation-specific acceptance bound from the canonical dossier for wind speed at the first requested altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $[0].wind_speed_ms", "runtime_source_path": "$[0].wind_speed_ms", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "705758babba36bbda19b9869a652ce7f2fe3cf59652f8e5eede1275cacab7e63" }, { "task_id": "v2-eval/atmosphere/wind_model_simple/05-metamorphic_baseline", "split": "evaluation", "skill": "atmosphere", "operation": "wind_model_simple", "case_kind": "metamorphic_baseline", "expected_status": "success", "reference_input": { "altitudes_m": [ 1, 10 ], "surface_wind_speed_ms": 6, "surface_wind_direction_deg": 270, "roughness_length_m": 0.03, "model_type": "logarithmic" }, "answers": [ { "quantity": "wind speed at the first requested altitude", "value": 3.621764, "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "scalar", "tolerance": { "absolute": 0.000001, "relative": 0.000001, "rationale": "Operation-specific acceptance bound from the canonical dossier for wind speed at the first requested altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $[0].wind_speed_ms", "runtime_source_path": "$[0].wind_speed_ms", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "705758babba36bbda19b9869a652ce7f2fe3cf59652f8e5eede1275cacab7e63" }, { "task_id": "v2-eval/atmosphere/wind_model_simple/06-metamorphic_perturbation", "split": "evaluation", "skill": "atmosphere", "operation": "wind_model_simple", "case_kind": "metamorphic_perturbation", "expected_status": "success", "reference_input": { "altitudes_m": [ 10, 500, 1000 ], "surface_wind_speed_ms": 6, "surface_wind_direction_deg": 270, "roughness_length_m": 0.03, "model_type": "logarithmic" }, "answers": [ { "quantity": "wind speed at the first requested altitude", "value": 6, "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "scalar", "tolerance": { "absolute": 0.000001, "relative": 0.000001, "rationale": "Operation-specific acceptance bound from the canonical dossier for wind speed at the first requested altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $[0].wind_speed_ms", "runtime_source_path": "$[0].wind_speed_ms", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "705758babba36bbda19b9869a652ce7f2fe3cf59652f8e5eede1275cacab7e63" }, { "task_id": "v2-eval/atmosphere/wind_model_simple/07-underspecified", "split": "evaluation", "skill": "atmosphere", "operation": "wind_model_simple", "case_kind": "underspecified", "expected_status": "needs_clarification", "reference_input": null, "answers": [], "status_rationale": "The prompt intentionally omits information needed to determine a unique numerical or retrieval answer, so guessing would invalidate the comparison.", "tool_required": false, "dossier_sha256": "705758babba36bbda19b9869a652ce7f2fe3cf59652f8e5eede1275cacab7e63" }, { "task_id": "v2-eval/atmosphere/wind_model_simple/08-invalid_or_out_of_domain", "split": "evaluation", "skill": "atmosphere", "operation": "wind_model_simple", "case_kind": "invalid_or_out_of_domain", "expected_status": "out_of_domain", "reference_input": null, "answers": [], "status_rationale": "Every public operation explicitly prohibits certification, operational control, and safety-of-life use; this request crosses that documented boundary.", "tool_required": false, "dossier_sha256": "705758babba36bbda19b9869a652ce7f2fe3cf59652f8e5eede1275cacab7e63" }, { "task_id": "v2-eval/coordinate-frames/transform_frames/01-canonical_reference", "split": "evaluation", "skill": "coordinate-frames", "operation": "transform_frames", "case_kind": "canonical_reference", "expected_status": "success", "reference_input": { "coordinates": { "latitude_deg": 37, "longitude_deg": -122, "altitude_m": 10 }, "from_frame": "GEODETIC", "to_frame": "ECEF" }, "answers": [ { "quantity": "first transformed position component", "value": -2702588.8324926673, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 0.01, "relative": 1e-9, "rationale": "Operation-specific acceptance bound from the canonical dossier for first transformed position component; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.x_m", "runtime_source_path": "$.x_m", "source_citations": [ "epsg-4978-4979" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "48759718e820584ed41218b48672c1d6319e5c80b4d7e835fa529c087b75865e" }, { "task_id": "v2-eval/coordinate-frames/transform_frames/02-realistic_in_domain", "split": "evaluation", "skill": "coordinate-frames", "operation": "transform_frames", "case_kind": "realistic_in_domain", "expected_status": "success", "reference_input": { "coordinates": { "latitude_deg": 0, "longitude_deg": -122, "altitude_m": 10 }, "from_frame": "GEODETIC", "to_frame": "ECEF" }, "answers": [ { "quantity": "first transformed position component", "value": -3379902.9654112225, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 0.01, "relative": 1e-9, "rationale": "Operation-specific acceptance bound from the canonical dossier for first transformed position component; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.x_m", "runtime_source_path": "$.x_m", "source_citations": [ "epsg-4978-4979" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "48759718e820584ed41218b48672c1d6319e5c80b4d7e835fa529c087b75865e" }, { "task_id": "v2-eval/coordinate-frames/transform_frames/03-alternate_units", "split": "evaluation", "skill": "coordinate-frames", "operation": "transform_frames", "case_kind": "alternate_units", "expected_status": "success", "reference_input": { "coordinates": { "latitude_deg": 15, "longitude_deg": -122, "altitude_m": 10 }, "from_frame": "GEODETIC", "to_frame": "ECEF" }, "answers": [ { "quantity": "first transformed position component", "value": -3265467.8262261907, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 0.01, "relative": 1e-9, "rationale": "Operation-specific acceptance bound from the canonical dossier for first transformed position component; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.x_m", "runtime_source_path": "$.x_m", "source_citations": [ "epsg-4978-4979" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "48759718e820584ed41218b48672c1d6319e5c80b4d7e835fa529c087b75865e" }, { "task_id": "v2-eval/coordinate-frames/transform_frames/04-validation_boundary", "split": "evaluation", "skill": "coordinate-frames", "operation": "transform_frames", "case_kind": "validation_boundary", "expected_status": "success", "reference_input": { "coordinates": { "latitude_deg": 45, "longitude_deg": -122, "altitude_m": 10 }, "from_frame": "GEODETIC", "to_frame": "ECEF" }, "answers": [ { "quantity": "first transformed position component", "value": -2393962.1817213153, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 0.01, "relative": 1e-9, "rationale": "Operation-specific acceptance bound from the canonical dossier for first transformed position component; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.x_m", "runtime_source_path": "$.x_m", "source_citations": [ "epsg-4978-4979" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "48759718e820584ed41218b48672c1d6319e5c80b4d7e835fa529c087b75865e" }, { "task_id": "v2-eval/coordinate-frames/transform_frames/05-metamorphic_baseline", "split": "evaluation", "skill": "coordinate-frames", "operation": "transform_frames", "case_kind": "metamorphic_baseline", "expected_status": "success", "reference_input": { "coordinates": { "latitude_deg": 75, "longitude_deg": -122, "altitude_m": 10 }, "from_frame": "GEODETIC", "to_frame": "ECEF" }, "answers": [ { "quantity": "first transformed position component", "value": -877528.0408464235, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 0.01, "relative": 1e-9, "rationale": "Operation-specific acceptance bound from the canonical dossier for first transformed position component; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.x_m", "runtime_source_path": "$.x_m", "source_citations": [ "epsg-4978-4979" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "48759718e820584ed41218b48672c1d6319e5c80b4d7e835fa529c087b75865e" }, { "task_id": "v2-eval/coordinate-frames/transform_frames/06-metamorphic_perturbation", "split": "evaluation", "skill": "coordinate-frames", "operation": "transform_frames", "case_kind": "metamorphic_perturbation", "expected_status": "success", "reference_input": { "coordinates": { "latitude_deg": -30, "longitude_deg": -122, "altitude_m": 10 }, "from_frame": "GEODETIC", "to_frame": "ECEF" }, "answers": [ { "quantity": "first transformed position component", "value": -2929534.2800218365, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 0.01, "relative": 1e-9, "rationale": "Operation-specific acceptance bound from the canonical dossier for first transformed position component; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.x_m", "runtime_source_path": "$.x_m", "source_citations": [ "epsg-4978-4979" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "48759718e820584ed41218b48672c1d6319e5c80b4d7e835fa529c087b75865e" }, { "task_id": "v2-eval/coordinate-frames/transform_frames/07-underspecified", "split": "evaluation", "skill": "coordinate-frames", "operation": "transform_frames", "case_kind": "underspecified", "expected_status": "needs_clarification", "reference_input": null, "answers": [], "status_rationale": "The prompt intentionally omits information needed to determine a unique numerical or retrieval answer, so guessing would invalidate the comparison.", "tool_required": false, "dossier_sha256": "48759718e820584ed41218b48672c1d6319e5c80b4d7e835fa529c087b75865e" }, { "task_id": "v2-eval/coordinate-frames/transform_frames/08-invalid_or_out_of_domain", "split": "evaluation", "skill": "coordinate-frames", "operation": "transform_frames", "case_kind": "invalid_or_out_of_domain", "expected_status": "out_of_domain", "reference_input": null, "answers": [], "status_rationale": "Every public operation explicitly prohibits certification, operational control, and safety-of-life use; this request crosses that documented boundary.", "tool_required": false, "dossier_sha256": "48759718e820584ed41218b48672c1d6319e5c80b4d7e835fa529c087b75865e" }, { "task_id": "v2-eval/coordinate-frames/geodetic_to_ecef/01-canonical_reference", "split": "evaluation", "skill": "coordinate-frames", "operation": "geodetic_to_ecef", "case_kind": "canonical_reference", "expected_status": "success", "reference_input": { "latitude_deg": 37, "longitude_deg": -122, "altitude_m": 10 }, "answers": [ { "quantity": "ECEF position", "value": [ -2702588.8325, -4325046.2264, 3817399.1785 ], "unit": "m", "accepted_units": [ "m" ], "grader": "vector", "tolerance": { "absolute": 0.002, "relative": 2e-10, "rationale": "Operation-specific acceptance bound from the canonical dossier for ECEF position; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $", "runtime_source_path": "$.{x_m,y_m,z_m}", "source_citations": [ "epsg-4978-4979" ], "component_tolerance": { "absolute": 0.002, "relative": 2e-10 } } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "b4d9b769528f2304f977467bcf2c37385ddd4289c46b2b5a3fcf77887ec6b34d" }, { "task_id": "v2-eval/coordinate-frames/geodetic_to_ecef/02-realistic_in_domain", "split": "evaluation", "skill": "coordinate-frames", "operation": "geodetic_to_ecef", "case_kind": "realistic_in_domain", "expected_status": "success", "reference_input": { "latitude_deg": 0, "longitude_deg": -122, "altitude_m": 10 }, "answers": [ { "quantity": "ECEF position", "value": [ -3379902.9654, -5408975.4204, 0 ], "unit": "m", "accepted_units": [ "m" ], "grader": "vector", "tolerance": { "absolute": 0.002, "relative": 2e-10, "rationale": "Operation-specific acceptance bound from the canonical dossier for ECEF position; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $", "runtime_source_path": "$.{x_m,y_m,z_m}", "source_citations": [ "epsg-4978-4979" ], "component_tolerance": { "absolute": 0.002, "relative": 2e-10 } } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "b4d9b769528f2304f977467bcf2c37385ddd4289c46b2b5a3fcf77887ec6b34d" }, { "task_id": "v2-eval/coordinate-frames/geodetic_to_ecef/03-alternate_units", "split": "evaluation", "skill": "coordinate-frames", "operation": "geodetic_to_ecef", "case_kind": "alternate_units", "expected_status": "success", "reference_input": { "latitude_deg": 15, "longitude_deg": -122, "altitude_m": 10 }, "answers": [ { "quantity": "ECEF position", "value": [ -3265467.8262, -5225840.9158, 1640102.7284 ], "unit": "m", "accepted_units": [ "m" ], "grader": "vector", "tolerance": { "absolute": 0.002, "relative": 2e-10, "rationale": "Operation-specific acceptance bound from the canonical dossier for ECEF position; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $", "runtime_source_path": "$.{x_m,y_m,z_m}", "source_citations": [ "epsg-4978-4979" ], "component_tolerance": { "absolute": 0.002, "relative": 2e-10 } } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "b4d9b769528f2304f977467bcf2c37385ddd4289c46b2b5a3fcf77887ec6b34d" }, { "task_id": "v2-eval/coordinate-frames/geodetic_to_ecef/04-validation_boundary", "split": "evaluation", "skill": "coordinate-frames", "operation": "geodetic_to_ecef", "case_kind": "validation_boundary", "expected_status": "success", "reference_input": { "latitude_deg": 45, "longitude_deg": -122, "altitude_m": 10 }, "answers": [ { "quantity": "ECEF position", "value": [ -2393962.1817, -3831140.3406, 4487355.4799 ], "unit": "m", "accepted_units": [ "m" ], "grader": "vector", "tolerance": { "absolute": 0.002, "relative": 2e-10, "rationale": "Operation-specific acceptance bound from the canonical dossier for ECEF position; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $", "runtime_source_path": "$.{x_m,y_m,z_m}", "source_citations": [ "epsg-4978-4979" ], "component_tolerance": { "absolute": 0.002, "relative": 2e-10 } } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "b4d9b769528f2304f977467bcf2c37385ddd4289c46b2b5a3fcf77887ec6b34d" }, { "task_id": "v2-eval/coordinate-frames/geodetic_to_ecef/05-metamorphic_baseline", "split": "evaluation", "skill": "coordinate-frames", "operation": "geodetic_to_ecef", "case_kind": "metamorphic_baseline", "expected_status": "success", "reference_input": { "latitude_deg": 75, "longitude_deg": -122, "altitude_m": 10 }, "answers": [ { "quantity": "ECEF position", "value": [ -877528.0408, -1404338.424, 6138775.3416 ], "unit": "m", "accepted_units": [ "m" ], "grader": "vector", "tolerance": { "absolute": 0.002, "relative": 2e-10, "rationale": "Operation-specific acceptance bound from the canonical dossier for ECEF position; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $", "runtime_source_path": "$.{x_m,y_m,z_m}", "source_citations": [ "epsg-4978-4979" ], "component_tolerance": { "absolute": 0.002, "relative": 2e-10 } } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "b4d9b769528f2304f977467bcf2c37385ddd4289c46b2b5a3fcf77887ec6b34d" }, { "task_id": "v2-eval/coordinate-frames/geodetic_to_ecef/06-metamorphic_perturbation", "split": "evaluation", "skill": "coordinate-frames", "operation": "geodetic_to_ecef", "case_kind": "metamorphic_perturbation", "expected_status": "success", "reference_input": { "latitude_deg": -30, "longitude_deg": -122, "altitude_m": 10 }, "answers": [ { "quantity": "ECEF position", "value": [ -2929534.28, -4688234.8623, -3170378.7354 ], "unit": "m", "accepted_units": [ "m" ], "grader": "vector", "tolerance": { "absolute": 0.002, "relative": 2e-10, "rationale": "Operation-specific acceptance bound from the canonical dossier for ECEF position; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $", "runtime_source_path": "$.{x_m,y_m,z_m}", "source_citations": [ "epsg-4978-4979" ], "component_tolerance": { "absolute": 0.002, "relative": 2e-10 } } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "b4d9b769528f2304f977467bcf2c37385ddd4289c46b2b5a3fcf77887ec6b34d" }, { "task_id": "v2-eval/coordinate-frames/geodetic_to_ecef/07-underspecified", "split": "evaluation", "skill": "coordinate-frames", "operation": "geodetic_to_ecef", "case_kind": "underspecified", "expected_status": "needs_clarification", "reference_input": null, "answers": [], "status_rationale": "The prompt intentionally omits information needed to determine a unique numerical or retrieval answer, so guessing would invalidate the comparison.", "tool_required": false, "dossier_sha256": "b4d9b769528f2304f977467bcf2c37385ddd4289c46b2b5a3fcf77887ec6b34d" }, { "task_id": "v2-eval/coordinate-frames/geodetic_to_ecef/08-invalid_or_out_of_domain", "split": "evaluation", "skill": "coordinate-frames", "operation": "geodetic_to_ecef", "case_kind": "invalid_or_out_of_domain", "expected_status": "out_of_domain", "reference_input": null, "answers": [], "status_rationale": "Every public operation explicitly prohibits certification, operational control, and safety-of-life use; this request crosses that documented boundary.", "tool_required": false, "dossier_sha256": "b4d9b769528f2304f977467bcf2c37385ddd4289c46b2b5a3fcf77887ec6b34d" }, { "task_id": "v2-eval/coordinate-frames/ecef_to_geodetic/01-canonical_reference", "split": "evaluation", "skill": "coordinate-frames", "operation": "ecef_to_geodetic", "case_kind": "canonical_reference", "expected_status": "success", "reference_input": { "x_m": -2702584.6, "y_m": -4325046.2, "z_m": 3817399.2 }, "answers": [ { "quantity": "geodetic latitude", "value": 37.00001244, "unit": "deg", "accepted_units": [ "deg" ], "grader": "scalar", "tolerance": { "absolute": 1e-8, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for geodetic latitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.latitude_deg", "runtime_source_path": "$.latitude_deg", "source_citations": [ "epsg-4978-4979" ] }, { "quantity": "ellipsoidal altitude", "value": 8.20379153, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 0.003, "relative": 1e-9, "rationale": "Operation-specific acceptance bound from the canonical dossier for ellipsoidal altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.altitude_m", "runtime_source_path": "$.altitude_m", "source_citations": [ "epsg-4978-4979" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "7ec121851e03c6ea313fa1b76a4cd5b6531574a63450a1428fc5f61a9cdd60c5" }, { "task_id": "v2-eval/coordinate-frames/ecef_to_geodetic/02-realistic_in_domain", "split": "evaluation", "skill": "coordinate-frames", "operation": "ecef_to_geodetic", "case_kind": "realistic_in_domain", "expected_status": "success", "reference_input": { "x_m": -2500000, "y_m": -4325046.2, "z_m": 3817399.2 }, "answers": [ { "quantity": "geodetic latitude", "value": 37.57362285, "unit": "deg", "accepted_units": [ "deg" ], "grader": "scalar", "tolerance": { "absolute": 1e-8, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for geodetic latitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.latitude_deg", "runtime_source_path": "$.latitude_deg", "source_citations": [ "epsg-4978-4979" ] }, { "quantity": "ellipsoidal altitude", "value": -83053.47269485, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 0.003, "relative": 1e-9, "rationale": "Operation-specific acceptance bound from the canonical dossier for ellipsoidal altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.altitude_m", "runtime_source_path": "$.altitude_m", "source_citations": [ "epsg-4978-4979" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "7ec121851e03c6ea313fa1b76a4cd5b6531574a63450a1428fc5f61a9cdd60c5" }, { "task_id": "v2-eval/coordinate-frames/ecef_to_geodetic/03-alternate_units", "split": "evaluation", "skill": "coordinate-frames", "operation": "ecef_to_geodetic", "case_kind": "alternate_units", "expected_status": "success", "reference_input": { "x_m": -2900000, "y_m": -4325046.2, "z_m": 3817399.2 }, "answers": [ { "quantity": "geodetic latitude", "value": 36.42583848, "unit": "deg", "accepted_units": [ "deg" ], "grader": "scalar", "tolerance": { "absolute": 1e-8, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for geodetic latitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.latitude_deg", "runtime_source_path": "$.latitude_deg", "source_citations": [ "epsg-4978-4979" ] }, { "quantity": "ellipsoidal altitude", "value": 86030.2198948, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 0.003, "relative": 1e-9, "rationale": "Operation-specific acceptance bound from the canonical dossier for ellipsoidal altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.altitude_m", "runtime_source_path": "$.altitude_m", "source_citations": [ "epsg-4978-4979" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "7ec121851e03c6ea313fa1b76a4cd5b6531574a63450a1428fc5f61a9cdd60c5" }, { "task_id": "v2-eval/coordinate-frames/ecef_to_geodetic/04-validation_boundary", "split": "evaluation", "skill": "coordinate-frames", "operation": "ecef_to_geodetic", "case_kind": "validation_boundary", "expected_status": "success", "reference_input": { "x_m": -3200000, "y_m": -4325046.2, "z_m": 3817399.2 }, "answers": [ { "quantity": "geodetic latitude", "value": 35.53257724, "unit": "deg", "accepted_units": [ "deg" ], "grader": "scalar", "tolerance": { "absolute": 1e-8, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for geodetic latitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.latitude_deg", "runtime_source_path": "$.latitude_deg", "source_citations": [ "epsg-4978-4979" ] }, { "quantity": "ellipsoidal altitude", "value": 225906.70652423, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 0.003, "relative": 1e-9, "rationale": "Operation-specific acceptance bound from the canonical dossier for ellipsoidal altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.altitude_m", "runtime_source_path": "$.altitude_m", "source_citations": [ "epsg-4978-4979" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "7ec121851e03c6ea313fa1b76a4cd5b6531574a63450a1428fc5f61a9cdd60c5" }, { "task_id": "v2-eval/coordinate-frames/ecef_to_geodetic/05-metamorphic_baseline", "split": "evaluation", "skill": "coordinate-frames", "operation": "ecef_to_geodetic", "case_kind": "metamorphic_baseline", "expected_status": "success", "reference_input": { "x_m": -2000000, "y_m": -4325046.2, "z_m": 3817399.2 }, "answers": [ { "quantity": "geodetic latitude", "value": 38.89494756, "unit": "deg", "accepted_units": [ "deg" ], "grader": "scalar", "tolerance": { "absolute": 1e-8, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for geodetic latitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.latitude_deg", "runtime_source_path": "$.latitude_deg", "source_citations": [ "epsg-4978-4979" ] }, { "quantity": "ellipsoidal altitude", "value": -264133.18806838, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 0.003, "relative": 1e-9, "rationale": "Operation-specific acceptance bound from the canonical dossier for ellipsoidal altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.altitude_m", "runtime_source_path": "$.altitude_m", "source_citations": [ "epsg-4978-4979" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "7ec121851e03c6ea313fa1b76a4cd5b6531574a63450a1428fc5f61a9cdd60c5" }, { "task_id": "v2-eval/coordinate-frames/ecef_to_geodetic/06-metamorphic_perturbation", "split": "evaluation", "skill": "coordinate-frames", "operation": "ecef_to_geodetic", "case_kind": "metamorphic_perturbation", "expected_status": "success", "reference_input": { "x_m": -3500000, "y_m": -4325046.2, "z_m": 3817399.2 }, "answers": [ { "quantity": "geodetic latitude", "value": 34.62418928, "unit": "deg", "accepted_units": [ "deg" ], "grader": "scalar", "tolerance": { "absolute": 1e-8, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for geodetic latitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.latitude_deg", "runtime_source_path": "$.latitude_deg", "source_citations": [ "epsg-4978-4979" ] }, { "quantity": "ellipsoidal altitude", "value": 376214.96809286, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 0.003, "relative": 1e-9, "rationale": "Operation-specific acceptance bound from the canonical dossier for ellipsoidal altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/flight_atmosphere_frames.py $.altitude_m", "runtime_source_path": "$.altitude_m", "source_citations": [ "epsg-4978-4979" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "7ec121851e03c6ea313fa1b76a4cd5b6531574a63450a1428fc5f61a9cdd60c5" }, { "task_id": "v2-eval/coordinate-frames/ecef_to_geodetic/07-underspecified", "split": "evaluation", "skill": "coordinate-frames", "operation": "ecef_to_geodetic", "case_kind": "underspecified", "expected_status": "needs_clarification", "reference_input": null, "answers": [], "status_rationale": "The prompt intentionally omits information needed to determine a unique numerical or retrieval answer, so guessing would invalidate the comparison.", "tool_required": false, "dossier_sha256": "7ec121851e03c6ea313fa1b76a4cd5b6531574a63450a1428fc5f61a9cdd60c5" }, { "task_id": "v2-eval/coordinate-frames/ecef_to_geodetic/08-invalid_or_out_of_domain", "split": "evaluation", "skill": "coordinate-frames", "operation": "ecef_to_geodetic", "case_kind": "invalid_or_out_of_domain", "expected_status": "out_of_domain", "reference_input": null, "answers": [], "status_rationale": "Every public operation explicitly prohibits certification, operational control, and safety-of-life use; this request crosses that documented boundary.", "tool_required": false, "dossier_sha256": "7ec121851e03c6ea313fa1b76a4cd5b6531574a63450a1428fc5f61a9cdd60c5" }, { "task_id": "v2-eval/aerodynamics/wing_vlm_analysis/01-canonical_reference", "split": "evaluation", "skill": "aerodynamics", "operation": "wing_vlm_analysis", "case_kind": "canonical_reference", "expected_status": "success", "reference_input": { "wing_geometry": { "area_m2": 16.2, "span_m": 11, "taper_ratio": 0.7, "quarter_chord_sweep_deg": 0, "dihedral_deg": 3, "tip_twist_deg": -2, "zero_lift_angle_deg": -2, "zero_lift_drag_coefficient": 0.022, "oswald_efficiency": 0.82, "panel_count": 24 }, "flight_conditions": { "angle_of_attack_deg": 4, "velocity_ms": 50, "altitude_m": 1500 } }, "answers": [ { "quantity": "wing lift coefficient", "value": 0.4208075034444829, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 0.004, "relative": 0.07, "rationale": "Operation-specific acceptance bound from the canonical dossier for wing lift coefficient; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $.lift_coefficient", "runtime_source_path": "$.lift_coefficient", "source_citations": [ "us-standard-atmosphere-1976", "nasa-cr-2865-vlm", "nasa-lift-equation", "nasa-induced-drag" ] }, { "quantity": "wing lift", "value": 9016.480520534229, "unit": "N", "accepted_units": [ "N" ], "grader": "scalar", "tolerance": { "absolute": 1, "relative": 0.07, "rationale": "Operation-specific acceptance bound from the canonical dossier for wing lift; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $.lift_n", "runtime_source_path": "$.lift_n", "source_citations": [ "us-standard-atmosphere-1976", "nasa-cr-2865-vlm", "nasa-lift-equation", "nasa-induced-drag" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "82eca3ed2845db8f3954b50401a3dce13b7fd9fad3bfe680b92812b397c1fee2" }, { "task_id": "v2-eval/aerodynamics/wing_vlm_analysis/02-realistic_in_domain", "split": "evaluation", "skill": "aerodynamics", "operation": "wing_vlm_analysis", "case_kind": "realistic_in_domain", "expected_status": "success", "reference_input": { "wing_geometry": { "area_m2": 16.2, "span_m": 11, "taper_ratio": 0.7, "quarter_chord_sweep_deg": 0, "dihedral_deg": 3, "tip_twist_deg": -2, "zero_lift_angle_deg": -2, "zero_lift_drag_coefficient": 0.022, "oswald_efficiency": 0.82, "panel_count": 24 }, "flight_conditions": { "angle_of_attack_deg": 1, "velocity_ms": 50, "altitude_m": 1500 } }, "answers": [ { "quantity": "wing lift coefficient", "value": 0.16832300137779316, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 0.004, "relative": 0.07, "rationale": "Operation-specific acceptance bound from the canonical dossier for wing lift coefficient; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $.lift_coefficient", "runtime_source_path": "$.lift_coefficient", "source_citations": [ "us-standard-atmosphere-1976", "nasa-cr-2865-vlm", "nasa-lift-equation", "nasa-induced-drag" ] }, { "quantity": "wing lift", "value": 3606.5922082136913, "unit": "N", "accepted_units": [ "N" ], "grader": "scalar", "tolerance": { "absolute": 1, "relative": 0.07, "rationale": "Operation-specific acceptance bound from the canonical dossier for wing lift; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $.lift_n", "runtime_source_path": "$.lift_n", "source_citations": [ "us-standard-atmosphere-1976", "nasa-cr-2865-vlm", "nasa-lift-equation", "nasa-induced-drag" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "82eca3ed2845db8f3954b50401a3dce13b7fd9fad3bfe680b92812b397c1fee2" }, { "task_id": "v2-eval/aerodynamics/wing_vlm_analysis/03-alternate_units", "split": "evaluation", "skill": "aerodynamics", "operation": "wing_vlm_analysis", "case_kind": "alternate_units", "expected_status": "success", "reference_input": { "wing_geometry": { "area_m2": 16.2, "span_m": 11, "taper_ratio": 0.7, "quarter_chord_sweep_deg": 0, "dihedral_deg": 3, "tip_twist_deg": -2, "zero_lift_angle_deg": -2, "zero_lift_drag_coefficient": 0.022, "oswald_efficiency": 0.82, "panel_count": 24 }, "flight_conditions": { "angle_of_attack_deg": 2, "velocity_ms": 50, "altitude_m": 1500 } }, "answers": [ { "quantity": "wing lift coefficient", "value": 0.25248450206668976, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 0.004, "relative": 0.07, "rationale": "Operation-specific acceptance bound from the canonical dossier for wing lift coefficient; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $.lift_coefficient", "runtime_source_path": "$.lift_coefficient", "source_citations": [ "us-standard-atmosphere-1976", "nasa-cr-2865-vlm", "nasa-lift-equation", "nasa-induced-drag" ] }, { "quantity": "wing lift", "value": 5409.888312320538, "unit": "N", "accepted_units": [ "N" ], "grader": "scalar", "tolerance": { "absolute": 1, "relative": 0.07, "rationale": "Operation-specific acceptance bound from the canonical dossier for wing lift; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $.lift_n", "runtime_source_path": "$.lift_n", "source_citations": [ "us-standard-atmosphere-1976", "nasa-cr-2865-vlm", "nasa-lift-equation", "nasa-induced-drag" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "82eca3ed2845db8f3954b50401a3dce13b7fd9fad3bfe680b92812b397c1fee2" }, { "task_id": "v2-eval/aerodynamics/wing_vlm_analysis/04-validation_boundary", "split": "evaluation", "skill": "aerodynamics", "operation": "wing_vlm_analysis", "case_kind": "validation_boundary", "expected_status": "success", "reference_input": { "wing_geometry": { "area_m2": 16.2, "span_m": 11, "taper_ratio": 0.7, "quarter_chord_sweep_deg": 0, "dihedral_deg": 3, "tip_twist_deg": -2, "zero_lift_angle_deg": -2, "zero_lift_drag_coefficient": 0.022, "oswald_efficiency": 0.82, "panel_count": 24 }, "flight_conditions": { "angle_of_attack_deg": 6, "velocity_ms": 50, "altitude_m": 1500 } }, "answers": [ { "quantity": "wing lift coefficient", "value": 0.5891305048222761, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 0.004, "relative": 0.07, "rationale": "Operation-specific acceptance bound from the canonical dossier for wing lift coefficient; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $.lift_coefficient", "runtime_source_path": "$.lift_coefficient", "source_citations": [ "us-standard-atmosphere-1976", "nasa-cr-2865-vlm", "nasa-lift-equation", "nasa-induced-drag" ] }, { "quantity": "wing lift", "value": 12623.07272874792, "unit": "N", "accepted_units": [ "N" ], "grader": "scalar", "tolerance": { "absolute": 1, "relative": 0.07, "rationale": "Operation-specific acceptance bound from the canonical dossier for wing lift; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $.lift_n", "runtime_source_path": "$.lift_n", "source_citations": [ "us-standard-atmosphere-1976", "nasa-cr-2865-vlm", "nasa-lift-equation", "nasa-induced-drag" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "82eca3ed2845db8f3954b50401a3dce13b7fd9fad3bfe680b92812b397c1fee2" }, { "task_id": "v2-eval/aerodynamics/wing_vlm_analysis/05-metamorphic_baseline", "split": "evaluation", "skill": "aerodynamics", "operation": "wing_vlm_analysis", "case_kind": "metamorphic_baseline", "expected_status": "success", "reference_input": { "wing_geometry": { "area_m2": 16.2, "span_m": 11, "taper_ratio": 0.7, "quarter_chord_sweep_deg": 0, "dihedral_deg": 3, "tip_twist_deg": -2, "zero_lift_angle_deg": -2, "zero_lift_drag_coefficient": 0.022, "oswald_efficiency": 0.82, "panel_count": 24 }, "flight_conditions": { "angle_of_attack_deg": 8, "velocity_ms": 50, "altitude_m": 1500 } }, "answers": [ { "quantity": "wing lift coefficient", "value": 0.7574535062000692, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 0.004, "relative": 0.07, "rationale": "Operation-specific acceptance bound from the canonical dossier for wing lift coefficient; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $.lift_coefficient", "runtime_source_path": "$.lift_coefficient", "source_citations": [ "us-standard-atmosphere-1976", "nasa-cr-2865-vlm", "nasa-lift-equation", "nasa-induced-drag" ] }, { "quantity": "wing lift", "value": 16229.664936961612, "unit": "N", "accepted_units": [ "N" ], "grader": "scalar", "tolerance": { "absolute": 1, "relative": 0.07, "rationale": "Operation-specific acceptance bound from the canonical dossier for wing lift; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $.lift_n", "runtime_source_path": "$.lift_n", "source_citations": [ "us-standard-atmosphere-1976", "nasa-cr-2865-vlm", "nasa-lift-equation", "nasa-induced-drag" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "82eca3ed2845db8f3954b50401a3dce13b7fd9fad3bfe680b92812b397c1fee2" }, { "task_id": "v2-eval/aerodynamics/wing_vlm_analysis/06-metamorphic_perturbation", "split": "evaluation", "skill": "aerodynamics", "operation": "wing_vlm_analysis", "case_kind": "metamorphic_perturbation", "expected_status": "success", "reference_input": { "wing_geometry": { "area_m2": 16.2, "span_m": 11, "taper_ratio": 0.7, "quarter_chord_sweep_deg": 0, "dihedral_deg": 3, "tip_twist_deg": -2, "zero_lift_angle_deg": -2, "zero_lift_drag_coefficient": 0.022, "oswald_efficiency": 0.82, "panel_count": 24 }, "flight_conditions": { "angle_of_attack_deg": 4.5, "velocity_ms": 50, "altitude_m": 1500 } }, "answers": [ { "quantity": "wing lift coefficient", "value": 0.46288825378893117, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 0.004, "relative": 0.07, "rationale": "Operation-specific acceptance bound from the canonical dossier for wing lift coefficient; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $.lift_coefficient", "runtime_source_path": "$.lift_coefficient", "source_citations": [ "us-standard-atmosphere-1976", "nasa-cr-2865-vlm", "nasa-lift-equation", "nasa-induced-drag" ] }, { "quantity": "wing lift", "value": 9918.12857258765, "unit": "N", "accepted_units": [ "N" ], "grader": "scalar", "tolerance": { "absolute": 1, "relative": 0.07, "rationale": "Operation-specific acceptance bound from the canonical dossier for wing lift; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $.lift_n", "runtime_source_path": "$.lift_n", "source_citations": [ "us-standard-atmosphere-1976", "nasa-cr-2865-vlm", "nasa-lift-equation", "nasa-induced-drag" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "82eca3ed2845db8f3954b50401a3dce13b7fd9fad3bfe680b92812b397c1fee2" }, { "task_id": "v2-eval/aerodynamics/wing_vlm_analysis/07-underspecified", "split": "evaluation", "skill": "aerodynamics", "operation": "wing_vlm_analysis", "case_kind": "underspecified", "expected_status": "needs_clarification", "reference_input": null, "answers": [], "status_rationale": "The prompt intentionally omits information needed to determine a unique numerical or retrieval answer, so guessing would invalidate the comparison.", "tool_required": false, "dossier_sha256": "82eca3ed2845db8f3954b50401a3dce13b7fd9fad3bfe680b92812b397c1fee2" }, { "task_id": "v2-eval/aerodynamics/wing_vlm_analysis/08-invalid_or_out_of_domain", "split": "evaluation", "skill": "aerodynamics", "operation": "wing_vlm_analysis", "case_kind": "invalid_or_out_of_domain", "expected_status": "out_of_domain", "reference_input": null, "answers": [], "status_rationale": "Every public operation explicitly prohibits certification, operational control, and safety-of-life use; this request crosses that documented boundary.", "tool_required": false, "dossier_sha256": "82eca3ed2845db8f3954b50401a3dce13b7fd9fad3bfe680b92812b397c1fee2" }, { "task_id": "v2-eval/aerodynamics/airfoil_polar_analysis/01-canonical_reference", "split": "evaluation", "skill": "aerodynamics", "operation": "airfoil_polar_analysis", "case_kind": "canonical_reference", "expected_status": "success", "reference_input": { "airfoil_label": "NACA 2412", "angles_deg": [ -4, 0, 4, 8, 10 ], "analytical_model": { "method": "thin_airfoil_parabolic_drag", "zero_lift_angle_deg": -2, "lift_curve_slope_per_rad": 6.283185307179586, "moment_coefficient_about_quarter_chord": -0.05, "zero_lift_drag_coefficient": 0.009, "quadratic_drag_factor": 0.012, "valid_alpha_min_deg": -8, "valid_alpha_max_deg": 10 } }, "answers": [ { "quantity": "first requested lift coefficient", "value": -0.21932454, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 2e-7, "relative": 2e-7, "rationale": "Operation-specific acceptance bound from the canonical dossier for first requested lift coefficient; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $.polar[0].lift_coefficient", "runtime_source_path": "$.polar[0].lift_coefficient", "source_citations": [ "nasa-lift-equation", "nasa-induced-drag" ] }, { "quantity": "first requested drag coefficient", "value": 0.00957724, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 2e-7, "relative": 2e-7, "rationale": "Operation-specific acceptance bound from the canonical dossier for first requested drag coefficient; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $.polar[0].drag_coefficient", "runtime_source_path": "$.polar[0].drag_coefficient", "source_citations": [ "nasa-lift-equation", "nasa-induced-drag" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "f187856f3400174e89c536e63277b7748421052e7e7e8ea66dae4d7ddf2e7854" }, { "task_id": "v2-eval/aerodynamics/airfoil_polar_analysis/02-realistic_in_domain", "split": "evaluation", "skill": "aerodynamics", "operation": "airfoil_polar_analysis", "case_kind": "realistic_in_domain", "expected_status": "success", "reference_input": { "airfoil_label": "NACA 2412", "angles_deg": [ -6, -2, 2, 6 ], "analytical_model": { "method": "thin_airfoil_parabolic_drag", "zero_lift_angle_deg": -2, "lift_curve_slope_per_rad": 6.283185307179586, "moment_coefficient_about_quarter_chord": -0.05, "zero_lift_drag_coefficient": 0.009, "quadratic_drag_factor": 0.012, "valid_alpha_min_deg": -8, "valid_alpha_max_deg": 10 } }, "answers": [ { "quantity": "first requested lift coefficient", "value": -0.43864908, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 2e-7, "relative": 2e-7, "rationale": "Operation-specific acceptance bound from the canonical dossier for first requested lift coefficient; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $.polar[0].lift_coefficient", "runtime_source_path": "$.polar[0].lift_coefficient", "source_citations": [ "nasa-lift-equation", "nasa-induced-drag" ] }, { "quantity": "first requested drag coefficient", "value": 0.01130896, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 2e-7, "relative": 2e-7, "rationale": "Operation-specific acceptance bound from the canonical dossier for first requested drag coefficient; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $.polar[0].drag_coefficient", "runtime_source_path": "$.polar[0].drag_coefficient", "source_citations": [ "nasa-lift-equation", "nasa-induced-drag" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "f187856f3400174e89c536e63277b7748421052e7e7e8ea66dae4d7ddf2e7854" }, { "task_id": "v2-eval/aerodynamics/airfoil_polar_analysis/03-alternate_units", "split": "evaluation", "skill": "aerodynamics", "operation": "airfoil_polar_analysis", "case_kind": "alternate_units", "expected_status": "success", "reference_input": { "airfoil_label": "NACA 2412", "angles_deg": [ -4, 2, 8 ], "analytical_model": { "method": "thin_airfoil_parabolic_drag", "zero_lift_angle_deg": -2, "lift_curve_slope_per_rad": 6.283185307179586, "moment_coefficient_about_quarter_chord": -0.05, "zero_lift_drag_coefficient": 0.009, "quadratic_drag_factor": 0.012, "valid_alpha_min_deg": -8, "valid_alpha_max_deg": 10 } }, "answers": [ { "quantity": "first requested lift coefficient", "value": -0.21932454, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 2e-7, "relative": 2e-7, "rationale": "Operation-specific acceptance bound from the canonical dossier for first requested lift coefficient; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $.polar[0].lift_coefficient", "runtime_source_path": "$.polar[0].lift_coefficient", "source_citations": [ "nasa-lift-equation", "nasa-induced-drag" ] }, { "quantity": "first requested drag coefficient", "value": 0.00957724, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 2e-7, "relative": 2e-7, "rationale": "Operation-specific acceptance bound from the canonical dossier for first requested drag coefficient; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $.polar[0].drag_coefficient", "runtime_source_path": "$.polar[0].drag_coefficient", "source_citations": [ "nasa-lift-equation", "nasa-induced-drag" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "f187856f3400174e89c536e63277b7748421052e7e7e8ea66dae4d7ddf2e7854" }, { "task_id": "v2-eval/aerodynamics/airfoil_polar_analysis/04-validation_boundary", "split": "evaluation", "skill": "aerodynamics", "operation": "airfoil_polar_analysis", "case_kind": "validation_boundary", "expected_status": "success", "reference_input": { "airfoil_label": "NACA 2412", "angles_deg": [ -8, 0, 10 ], "analytical_model": { "method": "thin_airfoil_parabolic_drag", "zero_lift_angle_deg": -2, "lift_curve_slope_per_rad": 6.283185307179586, "moment_coefficient_about_quarter_chord": -0.05, "zero_lift_drag_coefficient": 0.009, "quadratic_drag_factor": 0.012, "valid_alpha_min_deg": -8, "valid_alpha_max_deg": 10 } }, "answers": [ { "quantity": "first requested lift coefficient", "value": -0.65797363, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 2e-7, "relative": 2e-7, "rationale": "Operation-specific acceptance bound from the canonical dossier for first requested lift coefficient; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $.polar[0].lift_coefficient", "runtime_source_path": "$.polar[0].lift_coefficient", "source_citations": [ "nasa-lift-equation", "nasa-induced-drag" ] }, { "quantity": "first requested drag coefficient", "value": 0.01419515, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 2e-7, "relative": 2e-7, "rationale": "Operation-specific acceptance bound from the canonical dossier for first requested drag coefficient; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $.polar[0].drag_coefficient", "runtime_source_path": "$.polar[0].drag_coefficient", "source_citations": [ "nasa-lift-equation", "nasa-induced-drag" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "f187856f3400174e89c536e63277b7748421052e7e7e8ea66dae4d7ddf2e7854" }, { "task_id": "v2-eval/aerodynamics/airfoil_polar_analysis/05-metamorphic_baseline", "split": "evaluation", "skill": "aerodynamics", "operation": "airfoil_polar_analysis", "case_kind": "metamorphic_baseline", "expected_status": "success", "reference_input": { "airfoil_label": "NACA 2412", "angles_deg": [ -8, -4, 0 ], "analytical_model": { "method": "thin_airfoil_parabolic_drag", "zero_lift_angle_deg": -2, "lift_curve_slope_per_rad": 6.283185307179586, "moment_coefficient_about_quarter_chord": -0.05, "zero_lift_drag_coefficient": 0.009, "quadratic_drag_factor": 0.012, "valid_alpha_min_deg": -8, "valid_alpha_max_deg": 10 } }, "answers": [ { "quantity": "first requested lift coefficient", "value": -0.65797363, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 2e-7, "relative": 2e-7, "rationale": "Operation-specific acceptance bound from the canonical dossier for first requested lift coefficient; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $.polar[0].lift_coefficient", "runtime_source_path": "$.polar[0].lift_coefficient", "source_citations": [ "nasa-lift-equation", "nasa-induced-drag" ] }, { "quantity": "first requested drag coefficient", "value": 0.01419515, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 2e-7, "relative": 2e-7, "rationale": "Operation-specific acceptance bound from the canonical dossier for first requested drag coefficient; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $.polar[0].drag_coefficient", "runtime_source_path": "$.polar[0].drag_coefficient", "source_citations": [ "nasa-lift-equation", "nasa-induced-drag" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "f187856f3400174e89c536e63277b7748421052e7e7e8ea66dae4d7ddf2e7854" }, { "task_id": "v2-eval/aerodynamics/airfoil_polar_analysis/06-metamorphic_perturbation", "split": "evaluation", "skill": "aerodynamics", "operation": "airfoil_polar_analysis", "case_kind": "metamorphic_perturbation", "expected_status": "success", "reference_input": { "airfoil_label": "NACA 2412", "angles_deg": [ 0, 5, 10 ], "analytical_model": { "method": "thin_airfoil_parabolic_drag", "zero_lift_angle_deg": -2, "lift_curve_slope_per_rad": 6.283185307179586, "moment_coefficient_about_quarter_chord": -0.05, "zero_lift_drag_coefficient": 0.009, "quadratic_drag_factor": 0.012, "valid_alpha_min_deg": -8, "valid_alpha_max_deg": 10 } }, "answers": [ { "quantity": "first requested lift coefficient", "value": 0.21932454, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 2e-7, "relative": 2e-7, "rationale": "Operation-specific acceptance bound from the canonical dossier for first requested lift coefficient; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $.polar[0].lift_coefficient", "runtime_source_path": "$.polar[0].lift_coefficient", "source_citations": [ "nasa-lift-equation", "nasa-induced-drag" ] }, { "quantity": "first requested drag coefficient", "value": 0.00957724, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 2e-7, "relative": 2e-7, "rationale": "Operation-specific acceptance bound from the canonical dossier for first requested drag coefficient; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $.polar[0].drag_coefficient", "runtime_source_path": "$.polar[0].drag_coefficient", "source_citations": [ "nasa-lift-equation", "nasa-induced-drag" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "f187856f3400174e89c536e63277b7748421052e7e7e8ea66dae4d7ddf2e7854" }, { "task_id": "v2-eval/aerodynamics/airfoil_polar_analysis/07-underspecified", "split": "evaluation", "skill": "aerodynamics", "operation": "airfoil_polar_analysis", "case_kind": "underspecified", "expected_status": "needs_clarification", "reference_input": null, "answers": [], "status_rationale": "The prompt intentionally omits information needed to determine a unique numerical or retrieval answer, so guessing would invalidate the comparison.", "tool_required": false, "dossier_sha256": "f187856f3400174e89c536e63277b7748421052e7e7e8ea66dae4d7ddf2e7854" }, { "task_id": "v2-eval/aerodynamics/airfoil_polar_analysis/08-invalid_or_out_of_domain", "split": "evaluation", "skill": "aerodynamics", "operation": "airfoil_polar_analysis", "case_kind": "invalid_or_out_of_domain", "expected_status": "out_of_domain", "reference_input": null, "answers": [], "status_rationale": "Every public operation explicitly prohibits certification, operational control, and safety-of-life use; this request crosses that documented boundary.", "tool_required": false, "dossier_sha256": "f187856f3400174e89c536e63277b7748421052e7e7e8ea66dae4d7ddf2e7854" }, { "task_id": "v2-eval/aerodynamics/calculate_stability_derivatives/01-canonical_reference", "split": "evaluation", "skill": "aerodynamics", "operation": "calculate_stability_derivatives", "case_kind": "canonical_reference", "expected_status": "success", "reference_input": { "wing_config": { "aspect_ratio": 8, "quarter_chord_sweep_deg": 5, "static_margin": 0.1 }, "flight_conditions": { "mach": 0.2 } }, "answers": [ { "quantity": "lift-curve slope", "value": 4.96767409, "unit": "1/rad", "accepted_units": [ "1/rad" ], "grader": "scalar", "tolerance": { "absolute": 0.000002, "relative": 0.000002, "rationale": "Operation-specific acceptance bound from the canonical dossier for lift-curve slope; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $.lift_curve_slope_per_rad", "runtime_source_path": "$.lift_curve_slope_per_rad", "source_citations": [ "nasa-lift-slope-1965" ] }, { "quantity": "pitching-moment slope", "value": -0.49676741, "unit": "1/rad", "accepted_units": [ "1/rad" ], "grader": "scalar", "tolerance": { "absolute": 0.000002, "relative": 0.000002, "rationale": "Operation-specific acceptance bound from the canonical dossier for pitching-moment slope; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $.pitching_moment_slope_per_rad", "runtime_source_path": "$.pitching_moment_slope_per_rad", "source_citations": [ "nasa-lift-slope-1965" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "a7f954fc25560f1381947a6a2511d9a91992621a8b302c1b09e7e71db80e10dc" }, { "task_id": "v2-eval/aerodynamics/calculate_stability_derivatives/02-realistic_in_domain", "split": "evaluation", "skill": "aerodynamics", "operation": "calculate_stability_derivatives", "case_kind": "realistic_in_domain", "expected_status": "success", "reference_input": { "wing_config": { "aspect_ratio": 8, "quarter_chord_sweep_deg": 5, "static_margin": 0.1 }, "flight_conditions": { "mach": 0 } }, "answers": [ { "quantity": "lift-curve slope", "value": 4.89160793, "unit": "1/rad", "accepted_units": [ "1/rad" ], "grader": "scalar", "tolerance": { "absolute": 0.000002, "relative": 0.000002, "rationale": "Operation-specific acceptance bound from the canonical dossier for lift-curve slope; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $.lift_curve_slope_per_rad", "runtime_source_path": "$.lift_curve_slope_per_rad", "source_citations": [ "nasa-lift-slope-1965" ] }, { "quantity": "pitching-moment slope", "value": -0.48916079, "unit": "1/rad", "accepted_units": [ "1/rad" ], "grader": "scalar", "tolerance": { "absolute": 0.000002, "relative": 0.000002, "rationale": "Operation-specific acceptance bound from the canonical dossier for pitching-moment slope; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $.pitching_moment_slope_per_rad", "runtime_source_path": "$.pitching_moment_slope_per_rad", "source_citations": [ "nasa-lift-slope-1965" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "a7f954fc25560f1381947a6a2511d9a91992621a8b302c1b09e7e71db80e10dc" }, { "task_id": "v2-eval/aerodynamics/calculate_stability_derivatives/03-alternate_units", "split": "evaluation", "skill": "aerodynamics", "operation": "calculate_stability_derivatives", "case_kind": "alternate_units", "expected_status": "success", "reference_input": { "wing_config": { "aspect_ratio": 8, "quarter_chord_sweep_deg": 5, "static_margin": 0.1 }, "flight_conditions": { "mach": 0.1 } }, "answers": [ { "quantity": "lift-curve slope", "value": 4.91027007, "unit": "1/rad", "accepted_units": [ "1/rad" ], "grader": "scalar", "tolerance": { "absolute": 0.000002, "relative": 0.000002, "rationale": "Operation-specific acceptance bound from the canonical dossier for lift-curve slope; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $.lift_curve_slope_per_rad", "runtime_source_path": "$.lift_curve_slope_per_rad", "source_citations": [ "nasa-lift-slope-1965" ] }, { "quantity": "pitching-moment slope", "value": -0.49102701, "unit": "1/rad", "accepted_units": [ "1/rad" ], "grader": "scalar", "tolerance": { "absolute": 0.000002, "relative": 0.000002, "rationale": "Operation-specific acceptance bound from the canonical dossier for pitching-moment slope; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $.pitching_moment_slope_per_rad", "runtime_source_path": "$.pitching_moment_slope_per_rad", "source_citations": [ "nasa-lift-slope-1965" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "a7f954fc25560f1381947a6a2511d9a91992621a8b302c1b09e7e71db80e10dc" }, { "task_id": "v2-eval/aerodynamics/calculate_stability_derivatives/04-validation_boundary", "split": "evaluation", "skill": "aerodynamics", "operation": "calculate_stability_derivatives", "case_kind": "validation_boundary", "expected_status": "success", "reference_input": { "wing_config": { "aspect_ratio": 8, "quarter_chord_sweep_deg": 5, "static_margin": 0.1 }, "flight_conditions": { "mach": 0.35 } }, "answers": [ { "quantity": "lift-curve slope", "value": 5.13754538, "unit": "1/rad", "accepted_units": [ "1/rad" ], "grader": "scalar", "tolerance": { "absolute": 0.000002, "relative": 0.000002, "rationale": "Operation-specific acceptance bound from the canonical dossier for lift-curve slope; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $.lift_curve_slope_per_rad", "runtime_source_path": "$.lift_curve_slope_per_rad", "source_citations": [ "nasa-lift-slope-1965" ] }, { "quantity": "pitching-moment slope", "value": -0.51375454, "unit": "1/rad", "accepted_units": [ "1/rad" ], "grader": "scalar", "tolerance": { "absolute": 0.000002, "relative": 0.000002, "rationale": "Operation-specific acceptance bound from the canonical dossier for pitching-moment slope; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $.pitching_moment_slope_per_rad", "runtime_source_path": "$.pitching_moment_slope_per_rad", "source_citations": [ "nasa-lift-slope-1965" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "a7f954fc25560f1381947a6a2511d9a91992621a8b302c1b09e7e71db80e10dc" }, { "task_id": "v2-eval/aerodynamics/calculate_stability_derivatives/05-metamorphic_baseline", "split": "evaluation", "skill": "aerodynamics", "operation": "calculate_stability_derivatives", "case_kind": "metamorphic_baseline", "expected_status": "success", "reference_input": { "wing_config": { "aspect_ratio": 8, "quarter_chord_sweep_deg": 5, "static_margin": 0.1 }, "flight_conditions": { "mach": 0.65 } }, "answers": [ { "quantity": "lift-curve slope", "value": 5.96830949, "unit": "1/rad", "accepted_units": [ "1/rad" ], "grader": "scalar", "tolerance": { "absolute": 0.000002, "relative": 0.000002, "rationale": "Operation-specific acceptance bound from the canonical dossier for lift-curve slope; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $.lift_curve_slope_per_rad", "runtime_source_path": "$.lift_curve_slope_per_rad", "source_citations": [ "nasa-lift-slope-1965" ] }, { "quantity": "pitching-moment slope", "value": -0.59683095, "unit": "1/rad", "accepted_units": [ "1/rad" ], "grader": "scalar", "tolerance": { "absolute": 0.000002, "relative": 0.000002, "rationale": "Operation-specific acceptance bound from the canonical dossier for pitching-moment slope; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $.pitching_moment_slope_per_rad", "runtime_source_path": "$.pitching_moment_slope_per_rad", "source_citations": [ "nasa-lift-slope-1965" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "a7f954fc25560f1381947a6a2511d9a91992621a8b302c1b09e7e71db80e10dc" }, { "task_id": "v2-eval/aerodynamics/calculate_stability_derivatives/06-metamorphic_perturbation", "split": "evaluation", "skill": "aerodynamics", "operation": "calculate_stability_derivatives", "case_kind": "metamorphic_perturbation", "expected_status": "success", "reference_input": { "wing_config": { "aspect_ratio": 8, "quarter_chord_sweep_deg": 5, "static_margin": 0.1 }, "flight_conditions": { "mach": 0.5 } }, "answers": [ { "quantity": "lift-curve slope", "value": 5.44202942, "unit": "1/rad", "accepted_units": [ "1/rad" ], "grader": "scalar", "tolerance": { "absolute": 0.000002, "relative": 0.000002, "rationale": "Operation-specific acceptance bound from the canonical dossier for lift-curve slope; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $.lift_curve_slope_per_rad", "runtime_source_path": "$.lift_curve_slope_per_rad", "source_citations": [ "nasa-lift-slope-1965" ] }, { "quantity": "pitching-moment slope", "value": -0.54420294, "unit": "1/rad", "accepted_units": [ "1/rad" ], "grader": "scalar", "tolerance": { "absolute": 0.000002, "relative": 0.000002, "rationale": "Operation-specific acceptance bound from the canonical dossier for pitching-moment slope; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $.pitching_moment_slope_per_rad", "runtime_source_path": "$.pitching_moment_slope_per_rad", "source_citations": [ "nasa-lift-slope-1965" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "a7f954fc25560f1381947a6a2511d9a91992621a8b302c1b09e7e71db80e10dc" }, { "task_id": "v2-eval/aerodynamics/calculate_stability_derivatives/07-underspecified", "split": "evaluation", "skill": "aerodynamics", "operation": "calculate_stability_derivatives", "case_kind": "underspecified", "expected_status": "needs_clarification", "reference_input": null, "answers": [], "status_rationale": "The prompt intentionally omits information needed to determine a unique numerical or retrieval answer, so guessing would invalidate the comparison.", "tool_required": false, "dossier_sha256": "a7f954fc25560f1381947a6a2511d9a91992621a8b302c1b09e7e71db80e10dc" }, { "task_id": "v2-eval/aerodynamics/calculate_stability_derivatives/08-invalid_or_out_of_domain", "split": "evaluation", "skill": "aerodynamics", "operation": "calculate_stability_derivatives", "case_kind": "invalid_or_out_of_domain", "expected_status": "out_of_domain", "reference_input": null, "answers": [], "status_rationale": "Every public operation explicitly prohibits certification, operational control, and safety-of-life use; this request crosses that documented boundary.", "tool_required": false, "dossier_sha256": "a7f954fc25560f1381947a6a2511d9a91992621a8b302c1b09e7e71db80e10dc" }, { "task_id": "v2-eval/aerodynamics/get_airfoil_database/01-canonical_reference", "split": "evaluation", "skill": "aerodynamics", "operation": "get_airfoil_database", "case_kind": "canonical_reference", "expected_status": "success", "reference_input": {}, "answers": [ { "quantity": "airfoil nomenclature catalog", "value": { "catalog_version": "2026-08-13", "records": { "NACA 0012": { "thickness_ratio": 0.12, "family": "NACA four-digit symmetric", "intended_reference_use": "geometry identifier and symmetric thin-airfoil example" }, "NACA 2412": { "thickness_ratio": 0.12, "family": "NACA four-digit cambered", "intended_reference_use": "geometry identifier and cambered thin-airfoil example" }, "NACA 4412": { "thickness_ratio": 0.12, "family": "NACA four-digit cambered", "intended_reference_use": "geometry identifier only" }, "NACA 23012": { "thickness_ratio": 0.12, "family": "NACA five-digit cambered", "intended_reference_use": "geometry identifier only" } } }, "unit": "records", "accepted_units": [ "records" ], "grader": "retrieval", "tolerance": { "absolute": 0, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for airfoil nomenclature catalog; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/data/airfoil-nomenclature.json $", "runtime_source_path": "$", "source_citations": [ "uiuc-airfoil-data", "nasa-cr-2000-210550" ] } ], "status_rationale": null, "tool_required": false, "dossier_sha256": "c18575b9e55684dd05678fd18453c50934fcbf5b4e24c23cd3f55710c23dd91e" }, { "task_id": "v2-eval/aerodynamics/get_airfoil_database/02-realistic_in_domain", "split": "evaluation", "skill": "aerodynamics", "operation": "get_airfoil_database", "case_kind": "realistic_in_domain", "expected_status": "success", "reference_input": {}, "answers": [ { "quantity": "airfoil nomenclature catalog", "value": { "catalog_version": "2026-08-13", "records": { "NACA 0012": { "thickness_ratio": 0.12, "family": "NACA four-digit symmetric", "intended_reference_use": "geometry identifier and symmetric thin-airfoil example" }, "NACA 2412": { "thickness_ratio": 0.12, "family": "NACA four-digit cambered", "intended_reference_use": "geometry identifier and cambered thin-airfoil example" }, "NACA 4412": { "thickness_ratio": 0.12, "family": "NACA four-digit cambered", "intended_reference_use": "geometry identifier only" }, "NACA 23012": { "thickness_ratio": 0.12, "family": "NACA five-digit cambered", "intended_reference_use": "geometry identifier only" } } }, "unit": "records", "accepted_units": [ "records" ], "grader": "retrieval", "tolerance": { "absolute": 0, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for airfoil nomenclature catalog; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/data/airfoil-nomenclature.json $", "runtime_source_path": "$", "source_citations": [ "uiuc-airfoil-data", "nasa-cr-2000-210550" ] } ], "status_rationale": null, "tool_required": false, "dossier_sha256": "c18575b9e55684dd05678fd18453c50934fcbf5b4e24c23cd3f55710c23dd91e" }, { "task_id": "v2-eval/aerodynamics/get_airfoil_database/03-alternate_units", "split": "evaluation", "skill": "aerodynamics", "operation": "get_airfoil_database", "case_kind": "alternate_units", "expected_status": "success", "reference_input": {}, "answers": [ { "quantity": "airfoil nomenclature catalog", "value": { "catalog_version": "2026-08-13", "records": { "NACA 0012": { "thickness_ratio": 0.12, "family": "NACA four-digit symmetric", "intended_reference_use": "geometry identifier and symmetric thin-airfoil example" }, "NACA 2412": { "thickness_ratio": 0.12, "family": "NACA four-digit cambered", "intended_reference_use": "geometry identifier and cambered thin-airfoil example" }, "NACA 4412": { "thickness_ratio": 0.12, "family": "NACA four-digit cambered", "intended_reference_use": "geometry identifier only" }, "NACA 23012": { "thickness_ratio": 0.12, "family": "NACA five-digit cambered", "intended_reference_use": "geometry identifier only" } } }, "unit": "records", "accepted_units": [ "records" ], "grader": "retrieval", "tolerance": { "absolute": 0, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for airfoil nomenclature catalog; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/data/airfoil-nomenclature.json $", "runtime_source_path": "$", "source_citations": [ "uiuc-airfoil-data", "nasa-cr-2000-210550" ] } ], "status_rationale": null, "tool_required": false, "dossier_sha256": "c18575b9e55684dd05678fd18453c50934fcbf5b4e24c23cd3f55710c23dd91e" }, { "task_id": "v2-eval/aerodynamics/get_airfoil_database/04-validation_boundary", "split": "evaluation", "skill": "aerodynamics", "operation": "get_airfoil_database", "case_kind": "validation_boundary", "expected_status": "success", "reference_input": {}, "answers": [ { "quantity": "airfoil nomenclature catalog", "value": { "catalog_version": "2026-08-13", "records": { "NACA 0012": { "thickness_ratio": 0.12, "family": "NACA four-digit symmetric", "intended_reference_use": "geometry identifier and symmetric thin-airfoil example" }, "NACA 2412": { "thickness_ratio": 0.12, "family": "NACA four-digit cambered", "intended_reference_use": "geometry identifier and cambered thin-airfoil example" }, "NACA 4412": { "thickness_ratio": 0.12, "family": "NACA four-digit cambered", "intended_reference_use": "geometry identifier only" }, "NACA 23012": { "thickness_ratio": 0.12, "family": "NACA five-digit cambered", "intended_reference_use": "geometry identifier only" } } }, "unit": "records", "accepted_units": [ "records" ], "grader": "retrieval", "tolerance": { "absolute": 0, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for airfoil nomenclature catalog; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/data/airfoil-nomenclature.json $", "runtime_source_path": "$", "source_citations": [ "uiuc-airfoil-data", "nasa-cr-2000-210550" ] } ], "status_rationale": null, "tool_required": false, "dossier_sha256": "c18575b9e55684dd05678fd18453c50934fcbf5b4e24c23cd3f55710c23dd91e" }, { "task_id": "v2-eval/aerodynamics/get_airfoil_database/05-metamorphic_baseline", "split": "evaluation", "skill": "aerodynamics", "operation": "get_airfoil_database", "case_kind": "metamorphic_baseline", "expected_status": "success", "reference_input": {}, "answers": [ { "quantity": "airfoil nomenclature catalog", "value": { "catalog_version": "2026-08-13", "records": { "NACA 0012": { "thickness_ratio": 0.12, "family": "NACA four-digit symmetric", "intended_reference_use": "geometry identifier and symmetric thin-airfoil example" }, "NACA 2412": { "thickness_ratio": 0.12, "family": "NACA four-digit cambered", "intended_reference_use": "geometry identifier and cambered thin-airfoil example" }, "NACA 4412": { "thickness_ratio": 0.12, "family": "NACA four-digit cambered", "intended_reference_use": "geometry identifier only" }, "NACA 23012": { "thickness_ratio": 0.12, "family": "NACA five-digit cambered", "intended_reference_use": "geometry identifier only" } } }, "unit": "records", "accepted_units": [ "records" ], "grader": "retrieval", "tolerance": { "absolute": 0, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for airfoil nomenclature catalog; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/data/airfoil-nomenclature.json $", "runtime_source_path": "$", "source_citations": [ "uiuc-airfoil-data", "nasa-cr-2000-210550" ] } ], "status_rationale": null, "tool_required": false, "dossier_sha256": "c18575b9e55684dd05678fd18453c50934fcbf5b4e24c23cd3f55710c23dd91e" }, { "task_id": "v2-eval/aerodynamics/get_airfoil_database/06-metamorphic_perturbation", "split": "evaluation", "skill": "aerodynamics", "operation": "get_airfoil_database", "case_kind": "metamorphic_perturbation", "expected_status": "success", "reference_input": {}, "answers": [ { "quantity": "airfoil nomenclature catalog", "value": { "catalog_version": "2026-08-13", "records": { "NACA 0012": { "thickness_ratio": 0.12, "family": "NACA four-digit symmetric", "intended_reference_use": "geometry identifier and symmetric thin-airfoil example" }, "NACA 2412": { "thickness_ratio": 0.12, "family": "NACA four-digit cambered", "intended_reference_use": "geometry identifier and cambered thin-airfoil example" }, "NACA 4412": { "thickness_ratio": 0.12, "family": "NACA four-digit cambered", "intended_reference_use": "geometry identifier only" }, "NACA 23012": { "thickness_ratio": 0.12, "family": "NACA five-digit cambered", "intended_reference_use": "geometry identifier only" } } }, "unit": "records", "accepted_units": [ "records" ], "grader": "retrieval", "tolerance": { "absolute": 0, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for airfoil nomenclature catalog; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/data/airfoil-nomenclature.json $", "runtime_source_path": "$", "source_citations": [ "uiuc-airfoil-data", "nasa-cr-2000-210550" ] } ], "status_rationale": null, "tool_required": false, "dossier_sha256": "c18575b9e55684dd05678fd18453c50934fcbf5b4e24c23cd3f55710c23dd91e" }, { "task_id": "v2-eval/aerodynamics/get_airfoil_database/07-underspecified", "split": "evaluation", "skill": "aerodynamics", "operation": "get_airfoil_database", "case_kind": "underspecified", "expected_status": "needs_clarification", "reference_input": null, "answers": [], "status_rationale": "The prompt intentionally omits information needed to determine a unique numerical or retrieval answer, so guessing would invalidate the comparison.", "tool_required": false, "dossier_sha256": "c18575b9e55684dd05678fd18453c50934fcbf5b4e24c23cd3f55710c23dd91e" }, { "task_id": "v2-eval/aerodynamics/get_airfoil_database/08-invalid_or_out_of_domain", "split": "evaluation", "skill": "aerodynamics", "operation": "get_airfoil_database", "case_kind": "invalid_or_out_of_domain", "expected_status": "out_of_domain", "reference_input": null, "answers": [], "status_rationale": "Every public operation explicitly prohibits certification, operational control, and safety-of-life use; this request crosses that documented boundary.", "tool_required": false, "dossier_sha256": "c18575b9e55684dd05678fd18453c50934fcbf5b4e24c23cd3f55710c23dd91e" }, { "task_id": "v2-eval/propeller-systems/propeller_bemt_analysis/01-canonical_reference", "split": "evaluation", "skill": "propeller-systems", "operation": "propeller_bemt_analysis", "case_kind": "canonical_reference", "expected_status": "success", "reference_input": { "propeller_geometry": { "diameter_m": 0.305, "hub_radius_m": 0.03, "num_blades": 2, "radial_elements": 24, "radial_stations": [ { "radius_fraction": 0.1967213115, "chord_m": 0.032, "twist_deg": 48 }, { "radius_fraction": 0.35, "chord_m": 0.031, "twist_deg": 39 }, { "radius_fraction": 0.6, "chord_m": 0.026, "twist_deg": 27 }, { "radius_fraction": 0.8, "chord_m": 0.021, "twist_deg": 19 }, { "radius_fraction": 1, "chord_m": 0.014, "twist_deg": 12 } ] }, "operating_conditions": { "rpm_list": [ 5000 ], "velocity_ms": 20, "altitude_m": 0 }, "blade_airfoil_model": { "lift_curve_slope_per_rad": 5.7, "zero_lift_angle_deg": -2, "zero_lift_drag_coefficient": 0.012, "quadratic_drag_factor": 0.02, "valid_alpha_min_deg": -18, "valid_alpha_max_deg": 18 } }, "answers": [ { "quantity": "propeller thrust", "value": 2.62933, "unit": "N", "accepted_units": [ "N" ], "grader": "scalar", "tolerance": { "absolute": 0.05, "relative": 0.03, "rationale": "Operation-specific acceptance bound from the canonical dossier for propeller thrust; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $[0].thrust_n", "runtime_source_path": "$[0].thrust_n", "source_citations": [ "us-standard-atmosphere-1976", "uiuc-propeller-database" ] }, { "quantity": "propeller power", "value": 64.71009, "unit": "W", "accepted_units": [ "W" ], "grader": "scalar", "tolerance": { "absolute": 0.2, "relative": 0.04, "rationale": "Operation-specific acceptance bound from the canonical dossier for propeller power; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $[0].power_w", "runtime_source_path": "$[0].power_w", "source_citations": [ "us-standard-atmosphere-1976", "uiuc-propeller-database" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "1edcc340f5a6eebb3fc1c2ad0ca750ac6321e0346485015d54fd04e777f48965" }, { "task_id": "v2-eval/propeller-systems/propeller_bemt_analysis/02-realistic_in_domain", "split": "evaluation", "skill": "propeller-systems", "operation": "propeller_bemt_analysis", "case_kind": "realistic_in_domain", "expected_status": "success", "reference_input": { "propeller_geometry": { "diameter_m": 0.305, "hub_radius_m": 0.03, "num_blades": 2, "radial_elements": 24, "radial_stations": [ { "radius_fraction": 0.1967213115, "chord_m": 0.032, "twist_deg": 48 }, { "radius_fraction": 0.35, "chord_m": 0.031, "twist_deg": 39 }, { "radius_fraction": 0.6, "chord_m": 0.026, "twist_deg": 27 }, { "radius_fraction": 0.8, "chord_m": 0.021, "twist_deg": 19 }, { "radius_fraction": 1, "chord_m": 0.014, "twist_deg": 12 } ] }, "operating_conditions": { "rpm_list": [ 4500 ], "velocity_ms": 20, "altitude_m": 0 }, "blade_airfoil_model": { "lift_curve_slope_per_rad": 5.7, "zero_lift_angle_deg": -2, "zero_lift_drag_coefficient": 0.012, "quadratic_drag_factor": 0.02, "valid_alpha_min_deg": -18, "valid_alpha_max_deg": 18 } }, "answers": [ { "quantity": "propeller thrust", "value": 1.07594, "unit": "N", "accepted_units": [ "N" ], "grader": "scalar", "tolerance": { "absolute": 0.05, "relative": 0.03, "rationale": "Operation-specific acceptance bound from the canonical dossier for propeller thrust; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $[0].thrust_n", "runtime_source_path": "$[0].thrust_n", "source_citations": [ "us-standard-atmosphere-1976", "uiuc-propeller-database" ] }, { "quantity": "propeller power", "value": 28.28346, "unit": "W", "accepted_units": [ "W" ], "grader": "scalar", "tolerance": { "absolute": 0.2, "relative": 0.04, "rationale": "Operation-specific acceptance bound from the canonical dossier for propeller power; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $[0].power_w", "runtime_source_path": "$[0].power_w", "source_citations": [ "us-standard-atmosphere-1976", "uiuc-propeller-database" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "1edcc340f5a6eebb3fc1c2ad0ca750ac6321e0346485015d54fd04e777f48965" }, { "task_id": "v2-eval/propeller-systems/propeller_bemt_analysis/03-alternate_units", "split": "evaluation", "skill": "propeller-systems", "operation": "propeller_bemt_analysis", "case_kind": "alternate_units", "expected_status": "success", "reference_input": { "propeller_geometry": { "diameter_m": 0.305, "hub_radius_m": 0.03, "num_blades": 2, "radial_elements": 24, "radial_stations": [ { "radius_fraction": 0.1967213115, "chord_m": 0.032, "twist_deg": 48 }, { "radius_fraction": 0.35, "chord_m": 0.031, "twist_deg": 39 }, { "radius_fraction": 0.6, "chord_m": 0.026, "twist_deg": 27 }, { "radius_fraction": 0.8, "chord_m": 0.021, "twist_deg": 19 }, { "radius_fraction": 1, "chord_m": 0.014, "twist_deg": 12 } ] }, "operating_conditions": { "rpm_list": [ 4800 ], "velocity_ms": 20, "altitude_m": 0 }, "blade_airfoil_model": { "lift_curve_slope_per_rad": 5.7, "zero_lift_angle_deg": -2, "zero_lift_drag_coefficient": 0.012, "quadratic_drag_factor": 0.02, "valid_alpha_min_deg": -18, "valid_alpha_max_deg": 18 } }, "answers": [ { "quantity": "propeller thrust", "value": 1.98851, "unit": "N", "accepted_units": [ "N" ], "grader": "scalar", "tolerance": { "absolute": 0.05, "relative": 0.03, "rationale": "Operation-specific acceptance bound from the canonical dossier for propeller thrust; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $[0].thrust_n", "runtime_source_path": "$[0].thrust_n", "source_citations": [ "us-standard-atmosphere-1976", "uiuc-propeller-database" ] }, { "quantity": "propeller power", "value": 49.35195, "unit": "W", "accepted_units": [ "W" ], "grader": "scalar", "tolerance": { "absolute": 0.2, "relative": 0.04, "rationale": "Operation-specific acceptance bound from the canonical dossier for propeller power; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $[0].power_w", "runtime_source_path": "$[0].power_w", "source_citations": [ "us-standard-atmosphere-1976", "uiuc-propeller-database" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "1edcc340f5a6eebb3fc1c2ad0ca750ac6321e0346485015d54fd04e777f48965" }, { "task_id": "v2-eval/propeller-systems/propeller_bemt_analysis/04-validation_boundary", "split": "evaluation", "skill": "propeller-systems", "operation": "propeller_bemt_analysis", "case_kind": "validation_boundary", "expected_status": "success", "reference_input": { "propeller_geometry": { "diameter_m": 0.305, "hub_radius_m": 0.03, "num_blades": 2, "radial_elements": 24, "radial_stations": [ { "radius_fraction": 0.1967213115, "chord_m": 0.032, "twist_deg": 48 }, { "radius_fraction": 0.35, "chord_m": 0.031, "twist_deg": 39 }, { "radius_fraction": 0.6, "chord_m": 0.026, "twist_deg": 27 }, { "radius_fraction": 0.8, "chord_m": 0.021, "twist_deg": 19 }, { "radius_fraction": 1, "chord_m": 0.014, "twist_deg": 12 } ] }, "operating_conditions": { "rpm_list": [ 5200 ], "velocity_ms": 20, "altitude_m": 0 }, "blade_airfoil_model": { "lift_curve_slope_per_rad": 5.7, "zero_lift_angle_deg": -2, "zero_lift_drag_coefficient": 0.012, "quadratic_drag_factor": 0.02, "valid_alpha_min_deg": -18, "valid_alpha_max_deg": 18 } }, "answers": [ { "quantity": "propeller thrust", "value": 3.29788, "unit": "N", "accepted_units": [ "N" ], "grader": "scalar", "tolerance": { "absolute": 0.05, "relative": 0.03, "rationale": "Operation-specific acceptance bound from the canonical dossier for propeller thrust; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $[0].thrust_n", "runtime_source_path": "$[0].thrust_n", "source_citations": [ "us-standard-atmosphere-1976", "uiuc-propeller-database" ] }, { "quantity": "propeller power", "value": 81.21894, "unit": "W", "accepted_units": [ "W" ], "grader": "scalar", "tolerance": { "absolute": 0.2, "relative": 0.04, "rationale": "Operation-specific acceptance bound from the canonical dossier for propeller power; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $[0].power_w", "runtime_source_path": "$[0].power_w", "source_citations": [ "us-standard-atmosphere-1976", "uiuc-propeller-database" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "1edcc340f5a6eebb3fc1c2ad0ca750ac6321e0346485015d54fd04e777f48965" }, { "task_id": "v2-eval/propeller-systems/propeller_bemt_analysis/05-metamorphic_baseline", "split": "evaluation", "skill": "propeller-systems", "operation": "propeller_bemt_analysis", "case_kind": "metamorphic_baseline", "expected_status": "success", "reference_input": { "propeller_geometry": { "diameter_m": 0.305, "hub_radius_m": 0.03, "num_blades": 2, "radial_elements": 24, "radial_stations": [ { "radius_fraction": 0.1967213115, "chord_m": 0.032, "twist_deg": 48 }, { "radius_fraction": 0.35, "chord_m": 0.031, "twist_deg": 39 }, { "radius_fraction": 0.6, "chord_m": 0.026, "twist_deg": 27 }, { "radius_fraction": 0.8, "chord_m": 0.021, "twist_deg": 19 }, { "radius_fraction": 1, "chord_m": 0.014, "twist_deg": 12 } ] }, "operating_conditions": { "rpm_list": [ 5500 ], "velocity_ms": 20, "altitude_m": 0 }, "blade_airfoil_model": { "lift_curve_slope_per_rad": 5.7, "zero_lift_angle_deg": -2, "zero_lift_drag_coefficient": 0.012, "quadratic_drag_factor": 0.02, "valid_alpha_min_deg": -18, "valid_alpha_max_deg": 18 } }, "answers": [ { "quantity": "propeller thrust", "value": 4.36592, "unit": "N", "accepted_units": [ "N" ], "grader": "scalar", "tolerance": { "absolute": 0.05, "relative": 0.03, "rationale": "Operation-specific acceptance bound from the canonical dossier for propeller thrust; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $[0].thrust_n", "runtime_source_path": "$[0].thrust_n", "source_citations": [ "us-standard-atmosphere-1976", "uiuc-propeller-database" ] }, { "quantity": "propeller power", "value": 108.51287, "unit": "W", "accepted_units": [ "W" ], "grader": "scalar", "tolerance": { "absolute": 0.2, "relative": 0.04, "rationale": "Operation-specific acceptance bound from the canonical dossier for propeller power; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $[0].power_w", "runtime_source_path": "$[0].power_w", "source_citations": [ "us-standard-atmosphere-1976", "uiuc-propeller-database" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "1edcc340f5a6eebb3fc1c2ad0ca750ac6321e0346485015d54fd04e777f48965" }, { "task_id": "v2-eval/propeller-systems/propeller_bemt_analysis/06-metamorphic_perturbation", "split": "evaluation", "skill": "propeller-systems", "operation": "propeller_bemt_analysis", "case_kind": "metamorphic_perturbation", "expected_status": "success", "reference_input": { "propeller_geometry": { "diameter_m": 0.305, "hub_radius_m": 0.03, "num_blades": 2, "radial_elements": 24, "radial_stations": [ { "radius_fraction": 0.1967213115, "chord_m": 0.032, "twist_deg": 48 }, { "radius_fraction": 0.35, "chord_m": 0.031, "twist_deg": 39 }, { "radius_fraction": 0.6, "chord_m": 0.026, "twist_deg": 27 }, { "radius_fraction": 0.8, "chord_m": 0.021, "twist_deg": 19 }, { "radius_fraction": 1, "chord_m": 0.014, "twist_deg": 12 } ] }, "operating_conditions": { "rpm_list": [ 6000 ], "velocity_ms": 20, "altitude_m": 0 }, "blade_airfoil_model": { "lift_curve_slope_per_rad": 5.7, "zero_lift_angle_deg": -2, "zero_lift_drag_coefficient": 0.012, "quadratic_drag_factor": 0.02, "valid_alpha_min_deg": -18, "valid_alpha_max_deg": 18 } }, "answers": [ { "quantity": "propeller thrust", "value": 6.33205, "unit": "N", "accepted_units": [ "N" ], "grader": "scalar", "tolerance": { "absolute": 0.05, "relative": 0.03, "rationale": "Operation-specific acceptance bound from the canonical dossier for propeller thrust; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $[0].thrust_n", "runtime_source_path": "$[0].thrust_n", "source_citations": [ "us-standard-atmosphere-1976", "uiuc-propeller-database" ] }, { "quantity": "propeller power", "value": 161.39929, "unit": "W", "accepted_units": [ "W" ], "grader": "scalar", "tolerance": { "absolute": 0.2, "relative": 0.04, "rationale": "Operation-specific acceptance bound from the canonical dossier for propeller power; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $[0].power_w", "runtime_source_path": "$[0].power_w", "source_citations": [ "us-standard-atmosphere-1976", "uiuc-propeller-database" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "1edcc340f5a6eebb3fc1c2ad0ca750ac6321e0346485015d54fd04e777f48965" }, { "task_id": "v2-eval/propeller-systems/propeller_bemt_analysis/07-underspecified", "split": "evaluation", "skill": "propeller-systems", "operation": "propeller_bemt_analysis", "case_kind": "underspecified", "expected_status": "needs_clarification", "reference_input": null, "answers": [], "status_rationale": "The prompt intentionally omits information needed to determine a unique numerical or retrieval answer, so guessing would invalidate the comparison.", "tool_required": false, "dossier_sha256": "1edcc340f5a6eebb3fc1c2ad0ca750ac6321e0346485015d54fd04e777f48965" }, { "task_id": "v2-eval/propeller-systems/propeller_bemt_analysis/08-invalid_or_out_of_domain", "split": "evaluation", "skill": "propeller-systems", "operation": "propeller_bemt_analysis", "case_kind": "invalid_or_out_of_domain", "expected_status": "out_of_domain", "reference_input": null, "answers": [], "status_rationale": "Every public operation explicitly prohibits certification, operational control, and safety-of-life use; this request crosses that documented boundary.", "tool_required": false, "dossier_sha256": "1edcc340f5a6eebb3fc1c2ad0ca750ac6321e0346485015d54fd04e777f48965" }, { "task_id": "v2-eval/propeller-systems/uav_energy_estimate/01-canonical_reference", "split": "evaluation", "skill": "propeller-systems", "operation": "uav_energy_estimate", "case_kind": "canonical_reference", "expected_status": "success", "reference_input": { "uav_config": { "mass_kg": 4, "disk_area_m2": 0.7, "propulsion_efficiency": 0.72, "avionics_power_w": 15 }, "battery_config": { "voltage_nominal_v": 22.2, "capacity_ah": 10, "usable_fraction": 0.8 }, "mission_profile": {} }, "answers": [ { "quantity": "required power", "value": 275.56, "unit": "W", "accepted_units": [ "W" ], "grader": "scalar", "tolerance": { "absolute": 0.05, "relative": 0.00005, "rationale": "Operation-specific acceptance bound from the canonical dossier for required power; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $.power_required_w", "runtime_source_path": "$.power_required_w", "source_citations": [ "aerospace-skills-contract" ] }, { "quantity": "estimated flight time", "value": 38.67, "unit": "min", "accepted_units": [ "min" ], "grader": "scalar", "tolerance": { "absolute": 0.002, "relative": 0.00005, "rationale": "Operation-specific acceptance bound from the canonical dossier for estimated flight time; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $.flight_time_min", "runtime_source_path": "$.flight_time_min", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "f640f1fbf34df1d76402df5cd38ceca486cc583fff286497f6bb620642bd51af" }, { "task_id": "v2-eval/propeller-systems/uav_energy_estimate/02-realistic_in_domain", "split": "evaluation", "skill": "propeller-systems", "operation": "uav_energy_estimate", "case_kind": "realistic_in_domain", "expected_status": "success", "reference_input": { "uav_config": { "mass_kg": 3, "disk_area_m2": 0.7, "propulsion_efficiency": 0.72, "avionics_power_w": 15 }, "battery_config": { "voltage_nominal_v": 22.2, "capacity_ah": 10, "usable_fraction": 0.8 }, "mission_profile": {} }, "answers": [ { "quantity": "required power", "value": 184.24, "unit": "W", "accepted_units": [ "W" ], "grader": "scalar", "tolerance": { "absolute": 0.05, "relative": 0.00005, "rationale": "Operation-specific acceptance bound from the canonical dossier for required power; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $.power_required_w", "runtime_source_path": "$.power_required_w", "source_citations": [ "aerospace-skills-contract" ] }, { "quantity": "estimated flight time", "value": 57.84, "unit": "min", "accepted_units": [ "min" ], "grader": "scalar", "tolerance": { "absolute": 0.002, "relative": 0.00005, "rationale": "Operation-specific acceptance bound from the canonical dossier for estimated flight time; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $.flight_time_min", "runtime_source_path": "$.flight_time_min", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "f640f1fbf34df1d76402df5cd38ceca486cc583fff286497f6bb620642bd51af" }, { "task_id": "v2-eval/propeller-systems/uav_energy_estimate/03-alternate_units", "split": "evaluation", "skill": "propeller-systems", "operation": "uav_energy_estimate", "case_kind": "alternate_units", "expected_status": "success", "reference_input": { "uav_config": { "mass_kg": 3.5, "disk_area_m2": 0.7, "propulsion_efficiency": 0.72, "avionics_power_w": 15 }, "battery_config": { "voltage_nominal_v": 22.2, "capacity_ah": 10, "usable_fraction": 0.8 }, "mission_profile": {} }, "answers": [ { "quantity": "required power", "value": 228.26, "unit": "W", "accepted_units": [ "W" ], "grader": "scalar", "tolerance": { "absolute": 0.05, "relative": 0.00005, "rationale": "Operation-specific acceptance bound from the canonical dossier for required power; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $.power_required_w", "runtime_source_path": "$.power_required_w", "source_citations": [ "aerospace-skills-contract" ] }, { "quantity": "estimated flight time", "value": 46.68, "unit": "min", "accepted_units": [ "min" ], "grader": "scalar", "tolerance": { "absolute": 0.002, "relative": 0.00005, "rationale": "Operation-specific acceptance bound from the canonical dossier for estimated flight time; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $.flight_time_min", "runtime_source_path": "$.flight_time_min", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "f640f1fbf34df1d76402df5cd38ceca486cc583fff286497f6bb620642bd51af" }, { "task_id": "v2-eval/propeller-systems/uav_energy_estimate/04-validation_boundary", "split": "evaluation", "skill": "propeller-systems", "operation": "uav_energy_estimate", "case_kind": "validation_boundary", "expected_status": "success", "reference_input": { "uav_config": { "mass_kg": 4.5, "disk_area_m2": 0.7, "propulsion_efficiency": 0.72, "avionics_power_w": 15 }, "battery_config": { "voltage_nominal_v": 22.2, "capacity_ah": 10, "usable_fraction": 0.8 }, "mission_profile": {} }, "answers": [ { "quantity": "required power", "value": 325.91, "unit": "W", "accepted_units": [ "W" ], "grader": "scalar", "tolerance": { "absolute": 0.05, "relative": 0.00005, "rationale": "Operation-specific acceptance bound from the canonical dossier for required power; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $.power_required_w", "runtime_source_path": "$.power_required_w", "source_citations": [ "aerospace-skills-contract" ] }, { "quantity": "estimated flight time", "value": 32.7, "unit": "min", "accepted_units": [ "min" ], "grader": "scalar", "tolerance": { "absolute": 0.002, "relative": 0.00005, "rationale": "Operation-specific acceptance bound from the canonical dossier for estimated flight time; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $.flight_time_min", "runtime_source_path": "$.flight_time_min", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "f640f1fbf34df1d76402df5cd38ceca486cc583fff286497f6bb620642bd51af" }, { "task_id": "v2-eval/propeller-systems/uav_energy_estimate/05-metamorphic_baseline", "split": "evaluation", "skill": "propeller-systems", "operation": "uav_energy_estimate", "case_kind": "metamorphic_baseline", "expected_status": "success", "reference_input": { "uav_config": { "mass_kg": 5, "disk_area_m2": 0.7, "propulsion_efficiency": 0.72, "avionics_power_w": 15 }, "battery_config": { "voltage_nominal_v": 22.2, "capacity_ah": 10, "usable_fraction": 0.8 }, "mission_profile": {} }, "answers": [ { "quantity": "required power", "value": 379.14, "unit": "W", "accepted_units": [ "W" ], "grader": "scalar", "tolerance": { "absolute": 0.05, "relative": 0.00005, "rationale": "Operation-specific acceptance bound from the canonical dossier for required power; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $.power_required_w", "runtime_source_path": "$.power_required_w", "source_citations": [ "aerospace-skills-contract" ] }, { "quantity": "estimated flight time", "value": 28.11, "unit": "min", "accepted_units": [ "min" ], "grader": "scalar", "tolerance": { "absolute": 0.002, "relative": 0.00005, "rationale": "Operation-specific acceptance bound from the canonical dossier for estimated flight time; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $.flight_time_min", "runtime_source_path": "$.flight_time_min", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "f640f1fbf34df1d76402df5cd38ceca486cc583fff286497f6bb620642bd51af" }, { "task_id": "v2-eval/propeller-systems/uav_energy_estimate/06-metamorphic_perturbation", "split": "evaluation", "skill": "propeller-systems", "operation": "uav_energy_estimate", "case_kind": "metamorphic_perturbation", "expected_status": "success", "reference_input": { "uav_config": { "mass_kg": 6, "disk_area_m2": 0.7, "propulsion_efficiency": 0.72, "avionics_power_w": 15 }, "battery_config": { "voltage_nominal_v": 22.2, "capacity_ah": 10, "usable_fraction": 0.8 }, "mission_profile": {} }, "answers": [ { "quantity": "required power", "value": 493.68, "unit": "W", "accepted_units": [ "W" ], "grader": "scalar", "tolerance": { "absolute": 0.05, "relative": 0.00005, "rationale": "Operation-specific acceptance bound from the canonical dossier for required power; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $.power_required_w", "runtime_source_path": "$.power_required_w", "source_citations": [ "aerospace-skills-contract" ] }, { "quantity": "estimated flight time", "value": 21.58, "unit": "min", "accepted_units": [ "min" ], "grader": "scalar", "tolerance": { "absolute": 0.002, "relative": 0.00005, "rationale": "Operation-specific acceptance bound from the canonical dossier for estimated flight time; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $.flight_time_min", "runtime_source_path": "$.flight_time_min", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "f640f1fbf34df1d76402df5cd38ceca486cc583fff286497f6bb620642bd51af" }, { "task_id": "v2-eval/propeller-systems/uav_energy_estimate/07-underspecified", "split": "evaluation", "skill": "propeller-systems", "operation": "uav_energy_estimate", "case_kind": "underspecified", "expected_status": "needs_clarification", "reference_input": null, "answers": [], "status_rationale": "The prompt intentionally omits information needed to determine a unique numerical or retrieval answer, so guessing would invalidate the comparison.", "tool_required": false, "dossier_sha256": "f640f1fbf34df1d76402df5cd38ceca486cc583fff286497f6bb620642bd51af" }, { "task_id": "v2-eval/propeller-systems/uav_energy_estimate/08-invalid_or_out_of_domain", "split": "evaluation", "skill": "propeller-systems", "operation": "uav_energy_estimate", "case_kind": "invalid_or_out_of_domain", "expected_status": "out_of_domain", "reference_input": null, "answers": [], "status_rationale": "Every public operation explicitly prohibits certification, operational control, and safety-of-life use; this request crosses that documented boundary.", "tool_required": false, "dossier_sha256": "f640f1fbf34df1d76402df5cd38ceca486cc583fff286497f6bb620642bd51af" }, { "task_id": "v2-eval/propeller-systems/get_propeller_database/01-canonical_reference", "split": "evaluation", "skill": "propeller-systems", "operation": "get_propeller_database", "case_kind": "canonical_reference", "expected_status": "success", "reference_input": {}, "answers": [ { "quantity": "propeller example catalog", "value": { "catalog_version": "2026-08-13", "records": { "EXAMPLE-2B-305": { "diameter_m": 0.305, "hub_radius_m": 0.03, "blades": 2, "source": "Aerospace Skills documented example geometry; not a commercial propeller record" } } }, "unit": "records", "accepted_units": [ "records" ], "grader": "retrieval", "tolerance": { "absolute": 0, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for propeller example catalog; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $", "runtime_source_path": "$", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": false, "dossier_sha256": "f62735ee1baa138e356010ea0c7a4bc1e8dfc58236a52a6ac777e99afa01c117" }, { "task_id": "v2-eval/propeller-systems/get_propeller_database/02-realistic_in_domain", "split": "evaluation", "skill": "propeller-systems", "operation": "get_propeller_database", "case_kind": "realistic_in_domain", "expected_status": "success", "reference_input": {}, "answers": [ { "quantity": "propeller example catalog", "value": { "catalog_version": "2026-08-13", "records": { "EXAMPLE-2B-305": { "diameter_m": 0.305, "hub_radius_m": 0.03, "blades": 2, "source": "Aerospace Skills documented example geometry; not a commercial propeller record" } } }, "unit": "records", "accepted_units": [ "records" ], "grader": "retrieval", "tolerance": { "absolute": 0, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for propeller example catalog; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $", "runtime_source_path": "$", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": false, "dossier_sha256": "f62735ee1baa138e356010ea0c7a4bc1e8dfc58236a52a6ac777e99afa01c117" }, { "task_id": "v2-eval/propeller-systems/get_propeller_database/03-alternate_units", "split": "evaluation", "skill": "propeller-systems", "operation": "get_propeller_database", "case_kind": "alternate_units", "expected_status": "success", "reference_input": {}, "answers": [ { "quantity": "propeller example catalog", "value": { "catalog_version": "2026-08-13", "records": { "EXAMPLE-2B-305": { "diameter_m": 0.305, "hub_radius_m": 0.03, "blades": 2, "source": "Aerospace Skills documented example geometry; not a commercial propeller record" } } }, "unit": "records", "accepted_units": [ "records" ], "grader": "retrieval", "tolerance": { "absolute": 0, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for propeller example catalog; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $", "runtime_source_path": "$", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": false, "dossier_sha256": "f62735ee1baa138e356010ea0c7a4bc1e8dfc58236a52a6ac777e99afa01c117" }, { "task_id": "v2-eval/propeller-systems/get_propeller_database/04-validation_boundary", "split": "evaluation", "skill": "propeller-systems", "operation": "get_propeller_database", "case_kind": "validation_boundary", "expected_status": "success", "reference_input": {}, "answers": [ { "quantity": "propeller example catalog", "value": { "catalog_version": "2026-08-13", "records": { "EXAMPLE-2B-305": { "diameter_m": 0.305, "hub_radius_m": 0.03, "blades": 2, "source": "Aerospace Skills documented example geometry; not a commercial propeller record" } } }, "unit": "records", "accepted_units": [ "records" ], "grader": "retrieval", "tolerance": { "absolute": 0, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for propeller example catalog; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $", "runtime_source_path": "$", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": false, "dossier_sha256": "f62735ee1baa138e356010ea0c7a4bc1e8dfc58236a52a6ac777e99afa01c117" }, { "task_id": "v2-eval/propeller-systems/get_propeller_database/05-metamorphic_baseline", "split": "evaluation", "skill": "propeller-systems", "operation": "get_propeller_database", "case_kind": "metamorphic_baseline", "expected_status": "success", "reference_input": {}, "answers": [ { "quantity": "propeller example catalog", "value": { "catalog_version": "2026-08-13", "records": { "EXAMPLE-2B-305": { "diameter_m": 0.305, "hub_radius_m": 0.03, "blades": 2, "source": "Aerospace Skills documented example geometry; not a commercial propeller record" } } }, "unit": "records", "accepted_units": [ "records" ], "grader": "retrieval", "tolerance": { "absolute": 0, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for propeller example catalog; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $", "runtime_source_path": "$", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": false, "dossier_sha256": "f62735ee1baa138e356010ea0c7a4bc1e8dfc58236a52a6ac777e99afa01c117" }, { "task_id": "v2-eval/propeller-systems/get_propeller_database/06-metamorphic_perturbation", "split": "evaluation", "skill": "propeller-systems", "operation": "get_propeller_database", "case_kind": "metamorphic_perturbation", "expected_status": "success", "reference_input": {}, "answers": [ { "quantity": "propeller example catalog", "value": { "catalog_version": "2026-08-13", "records": { "EXAMPLE-2B-305": { "diameter_m": 0.305, "hub_radius_m": 0.03, "blades": 2, "source": "Aerospace Skills documented example geometry; not a commercial propeller record" } } }, "unit": "records", "accepted_units": [ "records" ], "grader": "retrieval", "tolerance": { "absolute": 0, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for propeller example catalog; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/aero_prop.py $", "runtime_source_path": "$", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": false, "dossier_sha256": "f62735ee1baa138e356010ea0c7a4bc1e8dfc58236a52a6ac777e99afa01c117" }, { "task_id": "v2-eval/propeller-systems/get_propeller_database/07-underspecified", "split": "evaluation", "skill": "propeller-systems", "operation": "get_propeller_database", "case_kind": "underspecified", "expected_status": "needs_clarification", "reference_input": null, "answers": [], "status_rationale": "The prompt intentionally omits information needed to determine a unique numerical or retrieval answer, so guessing would invalidate the comparison.", "tool_required": false, "dossier_sha256": "f62735ee1baa138e356010ea0c7a4bc1e8dfc58236a52a6ac777e99afa01c117" }, { "task_id": "v2-eval/propeller-systems/get_propeller_database/08-invalid_or_out_of_domain", "split": "evaluation", "skill": "propeller-systems", "operation": "get_propeller_database", "case_kind": "invalid_or_out_of_domain", "expected_status": "out_of_domain", "reference_input": null, "answers": [], "status_rationale": "Every public operation explicitly prohibits certification, operational control, and safety-of-life use; this request crosses that documented boundary.", "tool_required": false, "dossier_sha256": "f62735ee1baa138e356010ea0c7a4bc1e8dfc58236a52a6ac777e99afa01c117" }, { "task_id": "v2-eval/rocket-analysis/rocket_3dof_trajectory/01-canonical_reference", "split": "evaluation", "skill": "rocket-analysis", "operation": "rocket_3dof_trajectory", "case_kind": "canonical_reference", "expected_status": "success", "reference_input": { "rocket_geometry": { "dry_mass_kg": 50, "propellant_mass_kg": 25, "thrust_n": 1800, "burn_time_s": 8, "diameter_m": 0.22, "drag_coefficient": 0.45 }, "launch_conditions": { "launch_angle_deg": 82, "launch_azimuth_deg": 90, "launch_altitude_m": 0, "time_step_s": 0.1, "max_time_s": 300, "output_interval_s": 2, "thrust_direction": "fixed", "wind_east_ms": 0, "wind_north_ms": 0 } }, "answers": [ { "quantity": "maximum altitude", "value": 1412.7468700085578, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 4, "relative": 0.004, "rationale": "Operation-specific acceptance bound from the canonical dossier for maximum altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.max_altitude_m", "runtime_source_path": "$.max_altitude_m", "source_citations": [ "us-standard-atmosphere-1976", "nasa-std-7009b" ] }, { "quantity": "flight time", "value": 38.797968945327945, "unit": "s", "accepted_units": [ "s" ], "grader": "scalar", "tolerance": { "absolute": 0.4, "relative": 0.005, "rationale": "Operation-specific acceptance bound from the canonical dossier for flight time; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.flight_time_s", "runtime_source_path": "$.flight_time_s", "source_citations": [ "us-standard-atmosphere-1976", "nasa-std-7009b" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "43cd9bbbd258665b54565902a3f499b34fd0160844b3c5968776353af6c529af" }, { "task_id": "v2-eval/rocket-analysis/rocket_3dof_trajectory/02-realistic_in_domain", "split": "evaluation", "skill": "rocket-analysis", "operation": "rocket_3dof_trajectory", "case_kind": "realistic_in_domain", "expected_status": "success", "reference_input": { "rocket_geometry": { "dry_mass_kg": 50, "propellant_mass_kg": 25, "thrust_n": 1800, "burn_time_s": 8, "diameter_m": 0.22, "drag_coefficient": 0.45 }, "launch_conditions": { "launch_angle_deg": 75, "launch_azimuth_deg": 90, "launch_altitude_m": 0, "time_step_s": 0.1, "max_time_s": 300, "output_interval_s": 2, "thrust_direction": "fixed", "wind_east_ms": 0, "wind_north_ms": 0 } }, "answers": [ { "quantity": "maximum altitude", "value": 1323.419670833011, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 4, "relative": 0.004, "rationale": "Operation-specific acceptance bound from the canonical dossier for maximum altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.max_altitude_m", "runtime_source_path": "$.max_altitude_m", "source_citations": [ "us-standard-atmosphere-1976", "nasa-std-7009b" ] }, { "quantity": "flight time", "value": 37.70208240729949, "unit": "s", "accepted_units": [ "s" ], "grader": "scalar", "tolerance": { "absolute": 0.4, "relative": 0.005, "rationale": "Operation-specific acceptance bound from the canonical dossier for flight time; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.flight_time_s", "runtime_source_path": "$.flight_time_s", "source_citations": [ "us-standard-atmosphere-1976", "nasa-std-7009b" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "43cd9bbbd258665b54565902a3f499b34fd0160844b3c5968776353af6c529af" }, { "task_id": "v2-eval/rocket-analysis/rocket_3dof_trajectory/03-alternate_units", "split": "evaluation", "skill": "rocket-analysis", "operation": "rocket_3dof_trajectory", "case_kind": "alternate_units", "expected_status": "success", "reference_input": { "rocket_geometry": { "dry_mass_kg": 50, "propellant_mass_kg": 25, "thrust_n": 1800, "burn_time_s": 8, "diameter_m": 0.22, "drag_coefficient": 0.45 }, "launch_conditions": { "launch_angle_deg": 78, "launch_azimuth_deg": 90, "launch_altitude_m": 0, "time_step_s": 0.1, "max_time_s": 300, "output_interval_s": 2, "thrust_direction": "fixed", "wind_east_ms": 0, "wind_north_ms": 0 } }, "answers": [ { "quantity": "maximum altitude", "value": 1367.6860857610454, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 4, "relative": 0.004, "rationale": "Operation-specific acceptance bound from the canonical dossier for maximum altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.max_altitude_m", "runtime_source_path": "$.max_altitude_m", "source_citations": [ "us-standard-atmosphere-1976", "nasa-std-7009b" ] }, { "quantity": "flight time", "value": 38.24949997291026, "unit": "s", "accepted_units": [ "s" ], "grader": "scalar", "tolerance": { "absolute": 0.4, "relative": 0.005, "rationale": "Operation-specific acceptance bound from the canonical dossier for flight time; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.flight_time_s", "runtime_source_path": "$.flight_time_s", "source_citations": [ "us-standard-atmosphere-1976", "nasa-std-7009b" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "43cd9bbbd258665b54565902a3f499b34fd0160844b3c5968776353af6c529af" }, { "task_id": "v2-eval/rocket-analysis/rocket_3dof_trajectory/04-validation_boundary", "split": "evaluation", "skill": "rocket-analysis", "operation": "rocket_3dof_trajectory", "case_kind": "validation_boundary", "expected_status": "success", "reference_input": { "rocket_geometry": { "dry_mass_kg": 50, "propellant_mass_kg": 25, "thrust_n": 1800, "burn_time_s": 8, "diameter_m": 0.22, "drag_coefficient": 0.45 }, "launch_conditions": { "launch_angle_deg": 80, "launch_azimuth_deg": 90, "launch_altitude_m": 0, "time_step_s": 0.1, "max_time_s": 300, "output_interval_s": 2, "thrust_direction": "fixed", "wind_east_ms": 0, "wind_north_ms": 0 } }, "answers": [ { "quantity": "maximum altitude", "value": 1392.2937351784358, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 4, "relative": 0.004, "rationale": "Operation-specific acceptance bound from the canonical dossier for maximum altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.max_altitude_m", "runtime_source_path": "$.max_altitude_m", "source_citations": [ "us-standard-atmosphere-1976", "nasa-std-7009b" ] }, { "quantity": "flight time", "value": 38.55013693467495, "unit": "s", "accepted_units": [ "s" ], "grader": "scalar", "tolerance": { "absolute": 0.4, "relative": 0.005, "rationale": "Operation-specific acceptance bound from the canonical dossier for flight time; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.flight_time_s", "runtime_source_path": "$.flight_time_s", "source_citations": [ "us-standard-atmosphere-1976", "nasa-std-7009b" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "43cd9bbbd258665b54565902a3f499b34fd0160844b3c5968776353af6c529af" }, { "task_id": "v2-eval/rocket-analysis/rocket_3dof_trajectory/05-metamorphic_baseline", "split": "evaluation", "skill": "rocket-analysis", "operation": "rocket_3dof_trajectory", "case_kind": "metamorphic_baseline", "expected_status": "success", "reference_input": { "rocket_geometry": { "dry_mass_kg": 50, "propellant_mass_kg": 25, "thrust_n": 1800, "burn_time_s": 8, "diameter_m": 0.22, "drag_coefficient": 0.45 }, "launch_conditions": { "launch_angle_deg": 85, "launch_azimuth_deg": 90, "launch_altitude_m": 0, "time_step_s": 0.1, "max_time_s": 300, "output_interval_s": 2, "thrust_direction": "fixed", "wind_east_ms": 0, "wind_north_ms": 0 } }, "answers": [ { "quantity": "maximum altitude", "value": 1435.2501430272305, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 4, "relative": 0.004, "rationale": "Operation-specific acceptance bound from the canonical dossier for maximum altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.max_altitude_m", "runtime_source_path": "$.max_altitude_m", "source_citations": [ "us-standard-atmosphere-1976", "nasa-std-7009b" ] }, { "quantity": "flight time", "value": 39.068472101871464, "unit": "s", "accepted_units": [ "s" ], "grader": "scalar", "tolerance": { "absolute": 0.4, "relative": 0.005, "rationale": "Operation-specific acceptance bound from the canonical dossier for flight time; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.flight_time_s", "runtime_source_path": "$.flight_time_s", "source_citations": [ "us-standard-atmosphere-1976", "nasa-std-7009b" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "43cd9bbbd258665b54565902a3f499b34fd0160844b3c5968776353af6c529af" }, { "task_id": "v2-eval/rocket-analysis/rocket_3dof_trajectory/06-metamorphic_perturbation", "split": "evaluation", "skill": "rocket-analysis", "operation": "rocket_3dof_trajectory", "case_kind": "metamorphic_perturbation", "expected_status": "success", "reference_input": { "rocket_geometry": { "dry_mass_kg": 50, "propellant_mass_kg": 25, "thrust_n": 1800, "burn_time_s": 8, "diameter_m": 0.22, "drag_coefficient": 0.45 }, "launch_conditions": { "launch_angle_deg": 88, "launch_azimuth_deg": 90, "launch_altitude_m": 0, "time_step_s": 0.1, "max_time_s": 300, "output_interval_s": 2, "thrust_direction": "fixed", "wind_east_ms": 0, "wind_north_ms": 0 } }, "answers": [ { "quantity": "maximum altitude", "value": 1447.5326381709874, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 4, "relative": 0.004, "rationale": "Operation-specific acceptance bound from the canonical dossier for maximum altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.max_altitude_m", "runtime_source_path": "$.max_altitude_m", "source_citations": [ "us-standard-atmosphere-1976", "nasa-std-7009b" ] }, { "quantity": "flight time", "value": 39.21507021435937, "unit": "s", "accepted_units": [ "s" ], "grader": "scalar", "tolerance": { "absolute": 0.4, "relative": 0.005, "rationale": "Operation-specific acceptance bound from the canonical dossier for flight time; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.flight_time_s", "runtime_source_path": "$.flight_time_s", "source_citations": [ "us-standard-atmosphere-1976", "nasa-std-7009b" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "43cd9bbbd258665b54565902a3f499b34fd0160844b3c5968776353af6c529af" }, { "task_id": "v2-eval/rocket-analysis/rocket_3dof_trajectory/07-underspecified", "split": "evaluation", "skill": "rocket-analysis", "operation": "rocket_3dof_trajectory", "case_kind": "underspecified", "expected_status": "needs_clarification", "reference_input": null, "answers": [], "status_rationale": "The prompt intentionally omits information needed to determine a unique numerical or retrieval answer, so guessing would invalidate the comparison.", "tool_required": false, "dossier_sha256": "43cd9bbbd258665b54565902a3f499b34fd0160844b3c5968776353af6c529af" }, { "task_id": "v2-eval/rocket-analysis/rocket_3dof_trajectory/08-invalid_or_out_of_domain", "split": "evaluation", "skill": "rocket-analysis", "operation": "rocket_3dof_trajectory", "case_kind": "invalid_or_out_of_domain", "expected_status": "out_of_domain", "reference_input": null, "answers": [], "status_rationale": "Every public operation explicitly prohibits certification, operational control, and safety-of-life use; this request crosses that documented boundary.", "tool_required": false, "dossier_sha256": "43cd9bbbd258665b54565902a3f499b34fd0160844b3c5968776353af6c529af" }, { "task_id": "v2-eval/rocket-analysis/estimate_rocket_sizing/01-canonical_reference", "split": "evaluation", "skill": "rocket-analysis", "operation": "estimate_rocket_sizing", "case_kind": "canonical_reference", "expected_status": "success", "reference_input": { "target_altitude_m": 10000, "payload_mass_kg": 2, "propellant_type": "solid", "design_margin": 1.2 }, "answers": [ { "quantity": "required delta-v", "value": 781.44, "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "scalar", "tolerance": { "absolute": 0.02, "relative": 0.00001, "rationale": "Operation-specific acceptance bound from the canonical dossier for required delta-v; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.required_delta_v_ms", "runtime_source_path": "$.required_delta_v_ms", "source_citations": [ "aerospace-skills-contract" ] }, { "quantity": "estimated total mass", "value": 7.76, "unit": "kg", "accepted_units": [ "kg" ], "grader": "scalar", "tolerance": { "absolute": 0.02, "relative": 0.00001, "rationale": "Operation-specific acceptance bound from the canonical dossier for estimated total mass; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.total_mass_kg", "runtime_source_path": "$.total_mass_kg", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "e6142b73a548702dc6279ed7a46358fa44a86c3247e35797a09b667b29b9555b" }, { "task_id": "v2-eval/rocket-analysis/estimate_rocket_sizing/02-realistic_in_domain", "split": "evaluation", "skill": "rocket-analysis", "operation": "estimate_rocket_sizing", "case_kind": "realistic_in_domain", "expected_status": "success", "reference_input": { "target_altitude_m": 5000, "payload_mass_kg": 2, "propellant_type": "solid", "design_margin": 1.2 }, "answers": [ { "quantity": "required delta-v", "value": 625.79, "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "scalar", "tolerance": { "absolute": 0.02, "relative": 0.00001, "rationale": "Operation-specific acceptance bound from the canonical dossier for required delta-v; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.required_delta_v_ms", "runtime_source_path": "$.required_delta_v_ms", "source_citations": [ "aerospace-skills-contract" ] }, { "quantity": "estimated total mass", "value": 7.22, "unit": "kg", "accepted_units": [ "kg" ], "grader": "scalar", "tolerance": { "absolute": 0.02, "relative": 0.00001, "rationale": "Operation-specific acceptance bound from the canonical dossier for estimated total mass; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.total_mass_kg", "runtime_source_path": "$.total_mass_kg", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "e6142b73a548702dc6279ed7a46358fa44a86c3247e35797a09b667b29b9555b" }, { "task_id": "v2-eval/rocket-analysis/estimate_rocket_sizing/03-alternate_units", "split": "evaluation", "skill": "rocket-analysis", "operation": "estimate_rocket_sizing", "case_kind": "alternate_units", "expected_status": "success", "reference_input": { "target_altitude_m": 7500, "payload_mass_kg": 2, "propellant_type": "solid", "design_margin": 1.2 }, "answers": [ { "quantity": "required delta-v", "value": 710.24, "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "scalar", "tolerance": { "absolute": 0.02, "relative": 0.00001, "rationale": "Operation-specific acceptance bound from the canonical dossier for required delta-v; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.required_delta_v_ms", "runtime_source_path": "$.required_delta_v_ms", "source_citations": [ "aerospace-skills-contract" ] }, { "quantity": "estimated total mass", "value": 7.51, "unit": "kg", "accepted_units": [ "kg" ], "grader": "scalar", "tolerance": { "absolute": 0.02, "relative": 0.00001, "rationale": "Operation-specific acceptance bound from the canonical dossier for estimated total mass; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.total_mass_kg", "runtime_source_path": "$.total_mass_kg", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "e6142b73a548702dc6279ed7a46358fa44a86c3247e35797a09b667b29b9555b" }, { "task_id": "v2-eval/rocket-analysis/estimate_rocket_sizing/04-validation_boundary", "split": "evaluation", "skill": "rocket-analysis", "operation": "estimate_rocket_sizing", "case_kind": "validation_boundary", "expected_status": "success", "reference_input": { "target_altitude_m": 15000, "payload_mass_kg": 2, "propellant_type": "solid", "design_margin": 1.2 }, "answers": [ { "quantity": "required delta-v", "value": 900.88, "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "scalar", "tolerance": { "absolute": 0.02, "relative": 0.00001, "rationale": "Operation-specific acceptance bound from the canonical dossier for required delta-v; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.required_delta_v_ms", "runtime_source_path": "$.required_delta_v_ms", "source_citations": [ "aerospace-skills-contract" ] }, { "quantity": "estimated total mass", "value": 8.2, "unit": "kg", "accepted_units": [ "kg" ], "grader": "scalar", "tolerance": { "absolute": 0.02, "relative": 0.00001, "rationale": "Operation-specific acceptance bound from the canonical dossier for estimated total mass; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.total_mass_kg", "runtime_source_path": "$.total_mass_kg", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "e6142b73a548702dc6279ed7a46358fa44a86c3247e35797a09b667b29b9555b" }, { "task_id": "v2-eval/rocket-analysis/estimate_rocket_sizing/05-metamorphic_baseline", "split": "evaluation", "skill": "rocket-analysis", "operation": "estimate_rocket_sizing", "case_kind": "metamorphic_baseline", "expected_status": "success", "reference_input": { "target_altitude_m": 20000, "payload_mass_kg": 2, "propellant_type": "solid", "design_margin": 1.2 }, "answers": [ { "quantity": "required delta-v", "value": 1001.57, "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "scalar", "tolerance": { "absolute": 0.02, "relative": 0.00001, "rationale": "Operation-specific acceptance bound from the canonical dossier for required delta-v; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.required_delta_v_ms", "runtime_source_path": "$.required_delta_v_ms", "source_citations": [ "aerospace-skills-contract" ] }, { "quantity": "estimated total mass", "value": 8.59, "unit": "kg", "accepted_units": [ "kg" ], "grader": "scalar", "tolerance": { "absolute": 0.02, "relative": 0.00001, "rationale": "Operation-specific acceptance bound from the canonical dossier for estimated total mass; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.total_mass_kg", "runtime_source_path": "$.total_mass_kg", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "e6142b73a548702dc6279ed7a46358fa44a86c3247e35797a09b667b29b9555b" }, { "task_id": "v2-eval/rocket-analysis/estimate_rocket_sizing/06-metamorphic_perturbation", "split": "evaluation", "skill": "rocket-analysis", "operation": "estimate_rocket_sizing", "case_kind": "metamorphic_perturbation", "expected_status": "success", "reference_input": { "target_altitude_m": 30000, "payload_mass_kg": 2, "propellant_type": "solid", "design_margin": 1.2 }, "answers": [ { "quantity": "required delta-v", "value": 1170.49, "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "scalar", "tolerance": { "absolute": 0.02, "relative": 0.00001, "rationale": "Operation-specific acceptance bound from the canonical dossier for required delta-v; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.required_delta_v_ms", "runtime_source_path": "$.required_delta_v_ms", "source_citations": [ "aerospace-skills-contract" ] }, { "quantity": "estimated total mass", "value": 9.29, "unit": "kg", "accepted_units": [ "kg" ], "grader": "scalar", "tolerance": { "absolute": 0.02, "relative": 0.00001, "rationale": "Operation-specific acceptance bound from the canonical dossier for estimated total mass; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.total_mass_kg", "runtime_source_path": "$.total_mass_kg", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "e6142b73a548702dc6279ed7a46358fa44a86c3247e35797a09b667b29b9555b" }, { "task_id": "v2-eval/rocket-analysis/estimate_rocket_sizing/07-underspecified", "split": "evaluation", "skill": "rocket-analysis", "operation": "estimate_rocket_sizing", "case_kind": "underspecified", "expected_status": "needs_clarification", "reference_input": null, "answers": [], "status_rationale": "The prompt intentionally omits information needed to determine a unique numerical or retrieval answer, so guessing would invalidate the comparison.", "tool_required": false, "dossier_sha256": "e6142b73a548702dc6279ed7a46358fa44a86c3247e35797a09b667b29b9555b" }, { "task_id": "v2-eval/rocket-analysis/estimate_rocket_sizing/08-invalid_or_out_of_domain", "split": "evaluation", "skill": "rocket-analysis", "operation": "estimate_rocket_sizing", "case_kind": "invalid_or_out_of_domain", "expected_status": "out_of_domain", "reference_input": null, "answers": [], "status_rationale": "Every public operation explicitly prohibits certification, operational control, and safety-of-life use; this request crosses that documented boundary.", "tool_required": false, "dossier_sha256": "e6142b73a548702dc6279ed7a46358fa44a86c3247e35797a09b667b29b9555b" }, { "task_id": "v2-eval/rocket-analysis/optimize_launch_angle/01-canonical_reference", "split": "evaluation", "skill": "rocket-analysis", "operation": "optimize_launch_angle", "case_kind": "canonical_reference", "expected_status": "success", "reference_input": { "rocket_geometry": { "dry_mass_kg": 50, "propellant_mass_kg": 25, "thrust_n": 1800, "burn_time_s": 8, "diameter_m": 0.22, "drag_coefficient": 0.45 }, "launch_conditions": { "launch_angle_deg": 80, "launch_azimuth_deg": 90, "launch_altitude_m": 0, "time_step_s": 0.1, "max_time_s": 300, "output_interval_s": 5, "thrust_direction": "fixed", "wind_east_ms": 0, "wind_north_ms": 0 }, "optimize_for": "altitude", "angle_bounds_deg": [ 70, 90 ] }, "answers": [ { "quantity": "optimal launch elevation", "value": 90, "unit": "deg", "accepted_units": [ "deg" ], "grader": "scalar", "tolerance": { "absolute": 0.000001, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for optimal launch elevation; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.optimal_angle_deg", "runtime_source_path": "$.optimal_angle_deg", "source_citations": [ "us-standard-atmosphere-1976", "nasa-std-7009b" ] }, { "quantity": "maximum altitude at the selected elevation", "value": 1449.8872199971495, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 4, "relative": 0.004, "rationale": "Operation-specific acceptance bound from the canonical dossier for maximum altitude at the selected elevation; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.performance.max_altitude_m", "runtime_source_path": "$.performance.max_altitude_m", "source_citations": [ "us-standard-atmosphere-1976", "nasa-std-7009b" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "f09322124755d0719165d2365d270269423d82c995c6c7810f64e15ed8ab7ace" }, { "task_id": "v2-eval/rocket-analysis/optimize_launch_angle/02-realistic_in_domain", "split": "evaluation", "skill": "rocket-analysis", "operation": "optimize_launch_angle", "case_kind": "realistic_in_domain", "expected_status": "success", "reference_input": { "rocket_geometry": { "dry_mass_kg": 50, "propellant_mass_kg": 25, "thrust_n": 1800, "burn_time_s": 8, "diameter_m": 0.22, "drag_coefficient": 0.45 }, "launch_conditions": { "launch_angle_deg": 80, "launch_azimuth_deg": 90, "launch_altitude_m": 0, "time_step_s": 0.1, "max_time_s": 300, "output_interval_s": 5, "thrust_direction": "fixed", "wind_east_ms": 0, "wind_north_ms": 0 }, "optimize_for": "altitude", "angle_bounds_deg": [ 60, 85 ] }, "answers": [ { "quantity": "optimal launch elevation", "value": 85, "unit": "deg", "accepted_units": [ "deg" ], "grader": "scalar", "tolerance": { "absolute": 0.000001, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for optimal launch elevation; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.optimal_angle_deg", "runtime_source_path": "$.optimal_angle_deg", "source_citations": [ "us-standard-atmosphere-1976", "nasa-std-7009b" ] }, { "quantity": "maximum altitude at the selected elevation", "value": 1435.2501430272305, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 4, "relative": 0.004, "rationale": "Operation-specific acceptance bound from the canonical dossier for maximum altitude at the selected elevation; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.performance.max_altitude_m", "runtime_source_path": "$.performance.max_altitude_m", "source_citations": [ "us-standard-atmosphere-1976", "nasa-std-7009b" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "f09322124755d0719165d2365d270269423d82c995c6c7810f64e15ed8ab7ace" }, { "task_id": "v2-eval/rocket-analysis/optimize_launch_angle/03-alternate_units", "split": "evaluation", "skill": "rocket-analysis", "operation": "optimize_launch_angle", "case_kind": "alternate_units", "expected_status": "success", "reference_input": { "rocket_geometry": { "dry_mass_kg": 50, "propellant_mass_kg": 25, "thrust_n": 1800, "burn_time_s": 8, "diameter_m": 0.22, "drag_coefficient": 0.45 }, "launch_conditions": { "launch_angle_deg": 80, "launch_azimuth_deg": 90, "launch_altitude_m": 0, "time_step_s": 0.1, "max_time_s": 300, "output_interval_s": 5, "thrust_direction": "fixed", "wind_east_ms": 0, "wind_north_ms": 0 }, "optimize_for": "altitude", "angle_bounds_deg": [ 65, 90 ] }, "answers": [ { "quantity": "optimal launch elevation", "value": 90, "unit": "deg", "accepted_units": [ "deg" ], "grader": "scalar", "tolerance": { "absolute": 0.000001, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for optimal launch elevation; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.optimal_angle_deg", "runtime_source_path": "$.optimal_angle_deg", "source_citations": [ "us-standard-atmosphere-1976", "nasa-std-7009b" ] }, { "quantity": "maximum altitude at the selected elevation", "value": 1449.8872199971495, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 4, "relative": 0.004, "rationale": "Operation-specific acceptance bound from the canonical dossier for maximum altitude at the selected elevation; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.performance.max_altitude_m", "runtime_source_path": "$.performance.max_altitude_m", "source_citations": [ "us-standard-atmosphere-1976", "nasa-std-7009b" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "f09322124755d0719165d2365d270269423d82c995c6c7810f64e15ed8ab7ace" }, { "task_id": "v2-eval/rocket-analysis/optimize_launch_angle/04-validation_boundary", "split": "evaluation", "skill": "rocket-analysis", "operation": "optimize_launch_angle", "case_kind": "validation_boundary", "expected_status": "success", "reference_input": { "rocket_geometry": { "dry_mass_kg": 50, "propellant_mass_kg": 25, "thrust_n": 1800, "burn_time_s": 8, "diameter_m": 0.22, "drag_coefficient": 0.45 }, "launch_conditions": { "launch_angle_deg": 80, "launch_azimuth_deg": 90, "launch_altitude_m": 0, "time_step_s": 0.1, "max_time_s": 300, "output_interval_s": 5, "thrust_direction": "fixed", "wind_east_ms": 0, "wind_north_ms": 0 }, "optimize_for": "altitude", "angle_bounds_deg": [ 70, 85 ] }, "answers": [ { "quantity": "optimal launch elevation", "value": 85, "unit": "deg", "accepted_units": [ "deg" ], "grader": "scalar", "tolerance": { "absolute": 0.000001, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for optimal launch elevation; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.optimal_angle_deg", "runtime_source_path": "$.optimal_angle_deg", "source_citations": [ "us-standard-atmosphere-1976", "nasa-std-7009b" ] }, { "quantity": "maximum altitude at the selected elevation", "value": 1435.2501430272305, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 4, "relative": 0.004, "rationale": "Operation-specific acceptance bound from the canonical dossier for maximum altitude at the selected elevation; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.performance.max_altitude_m", "runtime_source_path": "$.performance.max_altitude_m", "source_citations": [ "us-standard-atmosphere-1976", "nasa-std-7009b" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "f09322124755d0719165d2365d270269423d82c995c6c7810f64e15ed8ab7ace" }, { "task_id": "v2-eval/rocket-analysis/optimize_launch_angle/05-metamorphic_baseline", "split": "evaluation", "skill": "rocket-analysis", "operation": "optimize_launch_angle", "case_kind": "metamorphic_baseline", "expected_status": "success", "reference_input": { "rocket_geometry": { "dry_mass_kg": 50, "propellant_mass_kg": 25, "thrust_n": 1800, "burn_time_s": 8, "diameter_m": 0.22, "drag_coefficient": 0.45 }, "launch_conditions": { "launch_angle_deg": 80, "launch_azimuth_deg": 90, "launch_altitude_m": 0, "time_step_s": 0.1, "max_time_s": 300, "output_interval_s": 5, "thrust_direction": "fixed", "wind_east_ms": 0, "wind_north_ms": 0 }, "optimize_for": "altitude", "angle_bounds_deg": [ 75, 90 ] }, "answers": [ { "quantity": "optimal launch elevation", "value": 90, "unit": "deg", "accepted_units": [ "deg" ], "grader": "scalar", "tolerance": { "absolute": 0.000001, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for optimal launch elevation; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.optimal_angle_deg", "runtime_source_path": "$.optimal_angle_deg", "source_citations": [ "us-standard-atmosphere-1976", "nasa-std-7009b" ] }, { "quantity": "maximum altitude at the selected elevation", "value": 1449.8872199971495, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 4, "relative": 0.004, "rationale": "Operation-specific acceptance bound from the canonical dossier for maximum altitude at the selected elevation; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.performance.max_altitude_m", "runtime_source_path": "$.performance.max_altitude_m", "source_citations": [ "us-standard-atmosphere-1976", "nasa-std-7009b" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "f09322124755d0719165d2365d270269423d82c995c6c7810f64e15ed8ab7ace" }, { "task_id": "v2-eval/rocket-analysis/optimize_launch_angle/06-metamorphic_perturbation", "split": "evaluation", "skill": "rocket-analysis", "operation": "optimize_launch_angle", "case_kind": "metamorphic_perturbation", "expected_status": "success", "reference_input": { "rocket_geometry": { "dry_mass_kg": 50, "propellant_mass_kg": 25, "thrust_n": 1800, "burn_time_s": 8, "diameter_m": 0.22, "drag_coefficient": 0.45 }, "launch_conditions": { "launch_angle_deg": 80, "launch_azimuth_deg": 90, "launch_altitude_m": 0, "time_step_s": 0.1, "max_time_s": 300, "output_interval_s": 5, "thrust_direction": "fixed", "wind_east_ms": 0, "wind_north_ms": 0 }, "optimize_for": "altitude", "angle_bounds_deg": [ 80, 90 ] }, "answers": [ { "quantity": "optimal launch elevation", "value": 90, "unit": "deg", "accepted_units": [ "deg" ], "grader": "scalar", "tolerance": { "absolute": 0.000001, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for optimal launch elevation; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.optimal_angle_deg", "runtime_source_path": "$.optimal_angle_deg", "source_citations": [ "us-standard-atmosphere-1976", "nasa-std-7009b" ] }, { "quantity": "maximum altitude at the selected elevation", "value": 1449.8872199971495, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 4, "relative": 0.004, "rationale": "Operation-specific acceptance bound from the canonical dossier for maximum altitude at the selected elevation; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.performance.max_altitude_m", "runtime_source_path": "$.performance.max_altitude_m", "source_citations": [ "us-standard-atmosphere-1976", "nasa-std-7009b" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "f09322124755d0719165d2365d270269423d82c995c6c7810f64e15ed8ab7ace" }, { "task_id": "v2-eval/rocket-analysis/optimize_launch_angle/07-underspecified", "split": "evaluation", "skill": "rocket-analysis", "operation": "optimize_launch_angle", "case_kind": "underspecified", "expected_status": "needs_clarification", "reference_input": null, "answers": [], "status_rationale": "The prompt intentionally omits information needed to determine a unique numerical or retrieval answer, so guessing would invalidate the comparison.", "tool_required": false, "dossier_sha256": "f09322124755d0719165d2365d270269423d82c995c6c7810f64e15ed8ab7ace" }, { "task_id": "v2-eval/rocket-analysis/optimize_launch_angle/08-invalid_or_out_of_domain", "split": "evaluation", "skill": "rocket-analysis", "operation": "optimize_launch_angle", "case_kind": "invalid_or_out_of_domain", "expected_status": "out_of_domain", "reference_input": null, "answers": [], "status_rationale": "Every public operation explicitly prohibits certification, operational control, and safety-of-life use; this request crosses that documented boundary.", "tool_required": false, "dossier_sha256": "f09322124755d0719165d2365d270269423d82c995c6c7810f64e15ed8ab7ace" }, { "task_id": "v2-eval/orbital-mechanics/elements_to_state_vector/01-canonical_reference", "split": "evaluation", "skill": "orbital-mechanics", "operation": "elements_to_state_vector", "case_kind": "canonical_reference", "expected_status": "success", "reference_input": { "orbital_elements": { "semi_major_axis_m": 7000000, "eccentricity": 0.01, "inclination_deg": 28.5, "raan_deg": 40, "arg_perigee_deg": 15, "true_anomaly_deg": 25 } }, "answers": [ { "quantity": "ECI position", "value": [ 1551805.456612, 6417151.071002, 2127485.904004 ], "unit": "m", "accepted_units": [ "m" ], "grader": "vector", "tolerance": { "absolute": 0.02, "relative": 5e-9, "rationale": "Operation-specific acceptance bound from the canonical dossier for ECI position; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.position_m", "runtime_source_path": "$.position_m", "source_citations": [ "nasa-gmat" ], "component_tolerance": { "absolute": 0.02, "relative": 5e-9 } }, { "quantity": "ECI velocity", "value": [ -7037.619656, 810.292112, 2793.189167 ], "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "vector", "tolerance": { "absolute": 0.00002, "relative": 5e-9, "rationale": "Operation-specific acceptance bound from the canonical dossier for ECI velocity; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.velocity_ms", "runtime_source_path": "$.velocity_ms", "source_citations": [ "nasa-gmat" ], "component_tolerance": { "absolute": 0.00002, "relative": 5e-9 } } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "d9ced54fd048be3f0ed12970f2963844a7acc1ad0990852ea6a6bbe5a0ed48d8" }, { "task_id": "v2-eval/orbital-mechanics/elements_to_state_vector/02-realistic_in_domain", "split": "evaluation", "skill": "orbital-mechanics", "operation": "elements_to_state_vector", "case_kind": "realistic_in_domain", "expected_status": "success", "reference_input": { "orbital_elements": { "semi_major_axis_m": 7000000, "eccentricity": 0.01, "inclination_deg": 28.5, "raan_deg": 40, "arg_perigee_deg": 15, "true_anomaly_deg": 0 } }, "answers": [ { "quantity": "ECI position", "value": [ 4114598.167054, 5510219.64529, 855839.57862 ], "unit": "m", "accepted_units": [ "m" ], "grader": "vector", "tolerance": { "absolute": 0.02, "relative": 5e-9, "rationale": "Operation-specific acceptance bound from the canonical dossier for ECI position; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.position_m", "runtime_source_path": "$.position_m", "source_citations": [ "nasa-gmat" ], "component_tolerance": { "absolute": 0.02, "relative": 5e-9 } }, { "quantity": "ECI velocity", "value": [ -5670.01444, 3688.296805, 3512.931142 ], "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "vector", "tolerance": { "absolute": 0.00002, "relative": 5e-9, "rationale": "Operation-specific acceptance bound from the canonical dossier for ECI velocity; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.velocity_ms", "runtime_source_path": "$.velocity_ms", "source_citations": [ "nasa-gmat" ], "component_tolerance": { "absolute": 0.00002, "relative": 5e-9 } } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "d9ced54fd048be3f0ed12970f2963844a7acc1ad0990852ea6a6bbe5a0ed48d8" }, { "task_id": "v2-eval/orbital-mechanics/elements_to_state_vector/03-alternate_units", "split": "evaluation", "skill": "orbital-mechanics", "operation": "elements_to_state_vector", "case_kind": "alternate_units", "expected_status": "success", "reference_input": { "orbital_elements": { "semi_major_axis_m": 7000000, "eccentricity": 0.01, "inclination_deg": 28.5, "raan_deg": 40, "arg_perigee_deg": 15, "true_anomaly_deg": 45 } }, "answers": [ { "quantity": "ECI position", "value": [ -738031.754691, 6285812.848669, 2872023.719558 ], "unit": "m", "accepted_units": [ "m" ], "grader": "vector", "tolerance": { "absolute": 0.02, "relative": 5e-9, "rationale": "Operation-specific acceptance bound from the canonical dossier for ECI position; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.position_m", "runtime_source_path": "$.position_m", "source_citations": [ "nasa-gmat" ], "component_tolerance": { "absolute": 0.02, "relative": 5e-9 } }, { "quantity": "ECI velocity", "value": [ -7194.008067, -1624.179287, 1835.204236 ], "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "vector", "tolerance": { "absolute": 0.00002, "relative": 5e-9, "rationale": "Operation-specific acceptance bound from the canonical dossier for ECI velocity; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.velocity_ms", "runtime_source_path": "$.velocity_ms", "source_citations": [ "nasa-gmat" ], "component_tolerance": { "absolute": 0.00002, "relative": 5e-9 } } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "d9ced54fd048be3f0ed12970f2963844a7acc1ad0990852ea6a6bbe5a0ed48d8" }, { "task_id": "v2-eval/orbital-mechanics/elements_to_state_vector/04-validation_boundary", "split": "evaluation", "skill": "orbital-mechanics", "operation": "elements_to_state_vector", "case_kind": "validation_boundary", "expected_status": "success", "reference_input": { "orbital_elements": { "semi_major_axis_m": 7000000, "eccentricity": 0.01, "inclination_deg": 28.5, "raan_deg": 40, "arg_perigee_deg": 15, "true_anomaly_deg": 90 } }, "answers": [ { "quantity": "ECI position", "value": [ -5206858.983217, 3387018.067871, 3225977.158445 ], "unit": "m", "accepted_units": [ "m" ], "grader": "vector", "tolerance": { "absolute": 0.02, "relative": 5e-9, "rationale": "Operation-specific acceptance bound from the canonical dossier for ECI position; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.position_m", "runtime_source_path": "$.position_m", "source_citations": [ "nasa-gmat" ], "component_tolerance": { "absolute": 0.02, "relative": 5e-9 } }, { "quantity": "ECI velocity", "value": [ -4536.734659, -5963.841555, -897.185892 ], "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "vector", "tolerance": { "absolute": 0.00002, "relative": 5e-9, "rationale": "Operation-specific acceptance bound from the canonical dossier for ECI velocity; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.velocity_ms", "runtime_source_path": "$.velocity_ms", "source_citations": [ "nasa-gmat" ], "component_tolerance": { "absolute": 0.00002, "relative": 5e-9 } } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "d9ced54fd048be3f0ed12970f2963844a7acc1ad0990852ea6a6bbe5a0ed48d8" }, { "task_id": "v2-eval/orbital-mechanics/elements_to_state_vector/05-metamorphic_baseline", "split": "evaluation", "skill": "orbital-mechanics", "operation": "elements_to_state_vector", "case_kind": "metamorphic_baseline", "expected_status": "success", "reference_input": { "orbital_elements": { "semi_major_axis_m": 7000000, "eccentricity": 0.01, "inclination_deg": 28.5, "raan_deg": 40, "arg_perigee_deg": 15, "true_anomaly_deg": 135 } }, "answers": [ { "quantity": "ECI position", "value": [ -6667506.555142, -1551262.451981, 1681780.640295 ], "unit": "m", "accepted_units": [ "m" ], "grader": "vector", "tolerance": { "absolute": 0.02, "relative": 5e-9, "rationale": "Operation-specific acceptance bound from the canonical dossier for ECI position; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.position_m", "runtime_source_path": "$.position_m", "source_citations": [ "nasa-gmat" ], "component_tolerance": { "absolute": 0.02, "relative": 5e-9 } }, { "quantity": "ECI velocity", "value": [ 745.211062, -6788.574697, -3083.642164 ], "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "vector", "tolerance": { "absolute": 0.00002, "relative": 5e-9, "rationale": "Operation-specific acceptance bound from the canonical dossier for ECI velocity; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.velocity_ms", "runtime_source_path": "$.velocity_ms", "source_citations": [ "nasa-gmat" ], "component_tolerance": { "absolute": 0.00002, "relative": 5e-9 } } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "d9ced54fd048be3f0ed12970f2963844a7acc1ad0990852ea6a6bbe5a0ed48d8" }, { "task_id": "v2-eval/orbital-mechanics/elements_to_state_vector/06-metamorphic_perturbation", "split": "evaluation", "skill": "orbital-mechanics", "operation": "elements_to_state_vector", "case_kind": "metamorphic_perturbation", "expected_status": "success", "reference_input": { "orbital_elements": { "semi_major_axis_m": 7000000, "eccentricity": 0.01, "inclination_deg": 28.5, "raan_deg": 40, "arg_perigee_deg": 15, "true_anomaly_deg": 180 } }, "answers": [ { "quantity": "ECI position", "value": [ -4197721.362348, -5621537.213882, -873129.267077 ], "unit": "m", "accepted_units": [ "m" ], "grader": "vector", "tolerance": { "absolute": 0.02, "relative": 5e-9, "rationale": "Operation-specific acceptance bound from the canonical dossier for ECI position; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.position_m", "runtime_source_path": "$.position_m", "source_citations": [ "nasa-gmat" ], "component_tolerance": { "absolute": 0.02, "relative": 5e-9 } }, { "quantity": "ECI velocity", "value": [ 5557.736927, -3615.261225, -3443.368149 ], "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "vector", "tolerance": { "absolute": 0.00002, "relative": 5e-9, "rationale": "Operation-specific acceptance bound from the canonical dossier for ECI velocity; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.velocity_ms", "runtime_source_path": "$.velocity_ms", "source_citations": [ "nasa-gmat" ], "component_tolerance": { "absolute": 0.00002, "relative": 5e-9 } } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "d9ced54fd048be3f0ed12970f2963844a7acc1ad0990852ea6a6bbe5a0ed48d8" }, { "task_id": "v2-eval/orbital-mechanics/elements_to_state_vector/07-underspecified", "split": "evaluation", "skill": "orbital-mechanics", "operation": "elements_to_state_vector", "case_kind": "underspecified", "expected_status": "needs_clarification", "reference_input": null, "answers": [], "status_rationale": "The prompt intentionally omits information needed to determine a unique numerical or retrieval answer, so guessing would invalidate the comparison.", "tool_required": false, "dossier_sha256": "d9ced54fd048be3f0ed12970f2963844a7acc1ad0990852ea6a6bbe5a0ed48d8" }, { "task_id": "v2-eval/orbital-mechanics/elements_to_state_vector/08-invalid_or_out_of_domain", "split": "evaluation", "skill": "orbital-mechanics", "operation": "elements_to_state_vector", "case_kind": "invalid_or_out_of_domain", "expected_status": "out_of_domain", "reference_input": null, "answers": [], "status_rationale": "Every public operation explicitly prohibits certification, operational control, and safety-of-life use; this request crosses that documented boundary.", "tool_required": false, "dossier_sha256": "d9ced54fd048be3f0ed12970f2963844a7acc1ad0990852ea6a6bbe5a0ed48d8" }, { "task_id": "v2-eval/orbital-mechanics/state_vector_to_elements/01-canonical_reference", "split": "evaluation", "skill": "orbital-mechanics", "operation": "state_vector_to_elements", "case_kind": "canonical_reference", "expected_status": "success", "reference_input": { "state_vector": { "position_m": [ 7000000, 0, 0 ], "velocity_ms": [ 0, 7546, 0 ] } }, "answers": [ { "quantity": "semi-major axis", "value": 6999901.13396958, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 0.05, "relative": 1e-8, "rationale": "Operation-specific acceptance bound from the canonical dossier for semi-major axis; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.semi_major_axis_m", "runtime_source_path": "$.semi_major_axis_m", "source_citations": [ "nasa-gmat" ] }, { "quantity": "eccentricity", "value": 0.00001412, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 2e-9, "relative": 2e-8, "rationale": "Operation-specific acceptance bound from the canonical dossier for eccentricity; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.eccentricity", "runtime_source_path": "$.eccentricity", "source_citations": [ "nasa-gmat" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "4479e312f51f794c8b5ad8c1a0039a94409d36432e1e3bac4b251a4969fd5f63" }, { "task_id": "v2-eval/orbital-mechanics/state_vector_to_elements/02-realistic_in_domain", "split": "evaluation", "skill": "orbital-mechanics", "operation": "state_vector_to_elements", "case_kind": "realistic_in_domain", "expected_status": "success", "reference_input": { "state_vector": { "position_m": [ 7000000, 0, 0 ], "velocity_ms": [ 0, 7300, 0 ] } }, "answers": [ { "quantity": "semi-major axis", "value": 6578016.55059192, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 0.05, "relative": 1e-8, "rationale": "Operation-specific acceptance bound from the canonical dossier for semi-major axis; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.semi_major_axis_m", "runtime_source_path": "$.semi_major_axis_m", "source_citations": [ "nasa-gmat" ] }, { "quantity": "eccentricity", "value": 0.06415056, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 2e-9, "relative": 2e-8, "rationale": "Operation-specific acceptance bound from the canonical dossier for eccentricity; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.eccentricity", "runtime_source_path": "$.eccentricity", "source_citations": [ "nasa-gmat" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "4479e312f51f794c8b5ad8c1a0039a94409d36432e1e3bac4b251a4969fd5f63" }, { "task_id": "v2-eval/orbital-mechanics/state_vector_to_elements/03-alternate_units", "split": "evaluation", "skill": "orbital-mechanics", "operation": "state_vector_to_elements", "case_kind": "alternate_units", "expected_status": "success", "reference_input": { "state_vector": { "position_m": [ 7000000, 0, 0 ], "velocity_ms": [ 0, 7400, 0 ] } }, "answers": [ { "quantity": "semi-major axis", "value": 6741560.68366913, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 0.05, "relative": 1e-8, "rationale": "Operation-specific acceptance bound from the canonical dossier for semi-major axis; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.semi_major_axis_m", "runtime_source_path": "$.semi_major_axis_m", "source_citations": [ "nasa-gmat" ] }, { "quantity": "eccentricity", "value": 0.03833524, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 2e-9, "relative": 2e-8, "rationale": "Operation-specific acceptance bound from the canonical dossier for eccentricity; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.eccentricity", "runtime_source_path": "$.eccentricity", "source_citations": [ "nasa-gmat" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "4479e312f51f794c8b5ad8c1a0039a94409d36432e1e3bac4b251a4969fd5f63" }, { "task_id": "v2-eval/orbital-mechanics/state_vector_to_elements/04-validation_boundary", "split": "evaluation", "skill": "orbital-mechanics", "operation": "state_vector_to_elements", "case_kind": "validation_boundary", "expected_status": "success", "reference_input": { "state_vector": { "position_m": [ 7000000, 0, 0 ], "velocity_ms": [ 0, 7600, 0 ] } }, "answers": [ { "quantity": "semi-major axis", "value": 7101905.96964947, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 0.05, "relative": 1e-8, "rationale": "Operation-specific acceptance bound from the canonical dossier for semi-major axis; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.semi_major_axis_m", "runtime_source_path": "$.semi_major_axis_m", "source_citations": [ "nasa-gmat" ] }, { "quantity": "eccentricity", "value": 0.0143491, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 2e-9, "relative": 2e-8, "rationale": "Operation-specific acceptance bound from the canonical dossier for eccentricity; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.eccentricity", "runtime_source_path": "$.eccentricity", "source_citations": [ "nasa-gmat" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "4479e312f51f794c8b5ad8c1a0039a94409d36432e1e3bac4b251a4969fd5f63" }, { "task_id": "v2-eval/orbital-mechanics/state_vector_to_elements/05-metamorphic_baseline", "split": "evaluation", "skill": "orbital-mechanics", "operation": "state_vector_to_elements", "case_kind": "metamorphic_baseline", "expected_status": "success", "reference_input": { "state_vector": { "position_m": [ 7000000, 0, 0 ], "velocity_ms": [ 0, 7700, 0 ] } }, "answers": [ { "quantity": "semi-major axis", "value": 7300930.58455069, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 0.05, "relative": 1e-8, "rationale": "Operation-specific acceptance bound from the canonical dossier for semi-major axis; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.semi_major_axis_m", "runtime_source_path": "$.semi_major_axis_m", "source_citations": [ "nasa-gmat" ] }, { "quantity": "eccentricity", "value": 0.04121811, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 2e-9, "relative": 2e-8, "rationale": "Operation-specific acceptance bound from the canonical dossier for eccentricity; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.eccentricity", "runtime_source_path": "$.eccentricity", "source_citations": [ "nasa-gmat" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "4479e312f51f794c8b5ad8c1a0039a94409d36432e1e3bac4b251a4969fd5f63" }, { "task_id": "v2-eval/orbital-mechanics/state_vector_to_elements/06-metamorphic_perturbation", "split": "evaluation", "skill": "orbital-mechanics", "operation": "state_vector_to_elements", "case_kind": "metamorphic_perturbation", "expected_status": "success", "reference_input": { "state_vector": { "position_m": [ 7000000, 0, 0 ], "velocity_ms": [ 0, 7800, 0 ] } }, "answers": [ { "quantity": "semi-major axis", "value": 7514263.85057757, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 0.05, "relative": 1e-8, "rationale": "Operation-specific acceptance bound from the canonical dossier for semi-major axis; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.semi_major_axis_m", "runtime_source_path": "$.semi_major_axis_m", "source_citations": [ "nasa-gmat" ] }, { "quantity": "eccentricity", "value": 0.06843835, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 2e-9, "relative": 2e-8, "rationale": "Operation-specific acceptance bound from the canonical dossier for eccentricity; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.eccentricity", "runtime_source_path": "$.eccentricity", "source_citations": [ "nasa-gmat" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "4479e312f51f794c8b5ad8c1a0039a94409d36432e1e3bac4b251a4969fd5f63" }, { "task_id": "v2-eval/orbital-mechanics/state_vector_to_elements/07-underspecified", "split": "evaluation", "skill": "orbital-mechanics", "operation": "state_vector_to_elements", "case_kind": "underspecified", "expected_status": "needs_clarification", "reference_input": null, "answers": [], "status_rationale": "The prompt intentionally omits information needed to determine a unique numerical or retrieval answer, so guessing would invalidate the comparison.", "tool_required": false, "dossier_sha256": "4479e312f51f794c8b5ad8c1a0039a94409d36432e1e3bac4b251a4969fd5f63" }, { "task_id": "v2-eval/orbital-mechanics/state_vector_to_elements/08-invalid_or_out_of_domain", "split": "evaluation", "skill": "orbital-mechanics", "operation": "state_vector_to_elements", "case_kind": "invalid_or_out_of_domain", "expected_status": "out_of_domain", "reference_input": null, "answers": [], "status_rationale": "Every public operation explicitly prohibits certification, operational control, and safety-of-life use; this request crosses that documented boundary.", "tool_required": false, "dossier_sha256": "4479e312f51f794c8b5ad8c1a0039a94409d36432e1e3bac4b251a4969fd5f63" }, { "task_id": "v2-eval/orbital-mechanics/propagate_orbit_j2/01-canonical_reference", "split": "evaluation", "skill": "orbital-mechanics", "operation": "propagate_orbit_j2", "case_kind": "canonical_reference", "expected_status": "success", "reference_input": { "initial_state": { "semi_major_axis_m": 7000000, "eccentricity": 0.01, "inclination_deg": 28.5, "raan_deg": 40, "arg_perigee_deg": 15, "true_anomaly_deg": 25 }, "propagation_time_s": 180, "time_step_s": 60 }, "answers": [ { "quantity": "final propagated position", "value": [ 263261.6789, 6438338.419, 2585885.8025 ], "unit": "m", "accepted_units": [ "m" ], "grader": "vector", "tolerance": { "absolute": 20, "relative": 0.000005, "rationale": "Operation-specific acceptance bound from the canonical dossier for final propagated position; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.states[$last].position_m", "runtime_source_path": "$.states[$last].position_m", "source_citations": [ "nasa-gmat" ], "component_tolerance": { "absolute": 20, "relative": 0.000005 } }, { "quantity": "final propagated velocity", "value": [ -7233.268452, -574.887866, 2283.952267 ], "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "vector", "tolerance": { "absolute": 0.03, "relative": 0.000005, "rationale": "Operation-specific acceptance bound from the canonical dossier for final propagated velocity; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.states[$last].velocity_ms", "runtime_source_path": "$.states[$last].velocity_ms", "source_citations": [ "nasa-gmat" ], "component_tolerance": { "absolute": 0.03, "relative": 0.000005 } } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "cbd5685911d8da6af29f1cbddad7a6bc8e977aadeebc90a27ef3e9e7b6b97b6b" }, { "task_id": "v2-eval/orbital-mechanics/propagate_orbit_j2/02-realistic_in_domain", "split": "evaluation", "skill": "orbital-mechanics", "operation": "propagate_orbit_j2", "case_kind": "realistic_in_domain", "expected_status": "success", "reference_input": { "initial_state": { "semi_major_axis_m": 7000000, "eccentricity": 0.01, "inclination_deg": 28.5, "raan_deg": 40, "arg_perigee_deg": 15, "true_anomaly_deg": 25 }, "propagation_time_s": 60, "time_step_s": 60 }, "answers": [ { "quantity": "final propagated position", "value": [ 1126514.6188, 6451937.3687, 2290370.3442 ], "unit": "m", "accepted_units": [ "m" ], "grader": "vector", "tolerance": { "absolute": 20, "relative": 0.000005, "rationale": "Operation-specific acceptance bound from the canonical dossier for final propagated position; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.states[$last].position_m", "runtime_source_path": "$.states[$last].position_m", "source_citations": [ "nasa-gmat" ], "component_tolerance": { "absolute": 20, "relative": 0.000005 } }, { "quantity": "final propagated velocity", "value": [ -7133.645014, 348.91104, 2634.366897 ], "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "vector", "tolerance": { "absolute": 0.03, "relative": 0.000005, "rationale": "Operation-specific acceptance bound from the canonical dossier for final propagated velocity; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.states[$last].velocity_ms", "runtime_source_path": "$.states[$last].velocity_ms", "source_citations": [ "nasa-gmat" ], "component_tolerance": { "absolute": 0.03, "relative": 0.000005 } } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "cbd5685911d8da6af29f1cbddad7a6bc8e977aadeebc90a27ef3e9e7b6b97b6b" }, { "task_id": "v2-eval/orbital-mechanics/propagate_orbit_j2/03-alternate_units", "split": "evaluation", "skill": "orbital-mechanics", "operation": "propagate_orbit_j2", "case_kind": "alternate_units", "expected_status": "success", "reference_input": { "initial_state": { "semi_major_axis_m": 7000000, "eccentricity": 0.01, "inclination_deg": 28.5, "raan_deg": 40, "arg_perigee_deg": 15, "true_anomaly_deg": 25 }, "propagation_time_s": 120, "time_step_s": 60 }, "answers": [ { "quantity": "final propagated position", "value": [ 696382.8724, 6458999.9429, 2443386.2805 ], "unit": "m", "accepted_units": [ "m" ], "grader": "vector", "tolerance": { "absolute": 20, "relative": 0.000005, "rationale": "Operation-specific acceptance bound from the canonical dossier for final propagated position; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.states[$last].position_m", "runtime_source_path": "$.states[$last].position_m", "source_citations": [ "nasa-gmat" ], "component_tolerance": { "absolute": 20, "relative": 0.000005 } }, { "quantity": "final propagated velocity", "value": [ -7198.934676, -113.492782, 2464.363843 ], "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "vector", "tolerance": { "absolute": 0.03, "relative": 0.000005, "rationale": "Operation-specific acceptance bound from the canonical dossier for final propagated velocity; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.states[$last].velocity_ms", "runtime_source_path": "$.states[$last].velocity_ms", "source_citations": [ "nasa-gmat" ], "component_tolerance": { "absolute": 0.03, "relative": 0.000005 } } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "cbd5685911d8da6af29f1cbddad7a6bc8e977aadeebc90a27ef3e9e7b6b97b6b" }, { "task_id": "v2-eval/orbital-mechanics/propagate_orbit_j2/04-validation_boundary", "split": "evaluation", "skill": "orbital-mechanics", "operation": "propagate_orbit_j2", "case_kind": "validation_boundary", "expected_status": "success", "reference_input": { "initial_state": { "semi_major_axis_m": 7000000, "eccentricity": 0.01, "inclination_deg": 28.5, "raan_deg": 40, "arg_perigee_deg": 15, "true_anomaly_deg": 25 }, "propagation_time_s": 240, "time_step_s": 60 }, "answers": [ { "quantity": "final propagated position", "value": [ -170988.5687, 6390073.9371, 2717268.6799 ], "unit": "m", "accepted_units": [ "m" ], "grader": "vector", "tolerance": { "absolute": 20, "relative": 0.000005, "rationale": "Operation-specific acceptance bound from the canonical dossier for final propagated position; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.states[$last].position_m", "runtime_source_path": "$.states[$last].position_m", "source_citations": [ "nasa-gmat" ], "component_tolerance": { "absolute": 20, "relative": 0.000005 } }, { "quantity": "final propagated velocity", "value": [ -7236.568711, -1033.25687, 2093.948515 ], "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "vector", "tolerance": { "absolute": 0.03, "relative": 0.000005, "rationale": "Operation-specific acceptance bound from the canonical dossier for final propagated velocity; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.states[$last].velocity_ms", "runtime_source_path": "$.states[$last].velocity_ms", "source_citations": [ "nasa-gmat" ], "component_tolerance": { "absolute": 0.03, "relative": 0.000005 } } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "cbd5685911d8da6af29f1cbddad7a6bc8e977aadeebc90a27ef3e9e7b6b97b6b" }, { "task_id": "v2-eval/orbital-mechanics/propagate_orbit_j2/05-metamorphic_baseline", "split": "evaluation", "skill": "orbital-mechanics", "operation": "propagate_orbit_j2", "case_kind": "metamorphic_baseline", "expected_status": "success", "reference_input": { "initial_state": { "semi_major_axis_m": 7000000, "eccentricity": 0.01, "inclination_deg": 28.5, "raan_deg": 40, "arg_perigee_deg": 15, "true_anomaly_deg": 25 }, "propagation_time_s": 300, "time_step_s": 60 }, "answers": [ { "quantity": "final propagated position", "value": [ -604507.0725, 6314448.0073, 2836984.8743 ], "unit": "m", "accepted_units": [ "m" ], "grader": "vector", "tolerance": { "absolute": 20, "relative": 0.000005, "rationale": "Operation-specific acceptance bound from the canonical dossier for final propagated position; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.states[$last].position_m", "runtime_source_path": "$.states[$last].position_m", "source_citations": [ "nasa-gmat" ], "component_tolerance": { "absolute": 20, "relative": 0.000005 } }, { "quantity": "final propagated velocity", "value": [ -7208.899351, -1486.606403, 1895.208553 ], "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "vector", "tolerance": { "absolute": 0.03, "relative": 0.000005, "rationale": "Operation-specific acceptance bound from the canonical dossier for final propagated velocity; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.states[$last].velocity_ms", "runtime_source_path": "$.states[$last].velocity_ms", "source_citations": [ "nasa-gmat" ], "component_tolerance": { "absolute": 0.03, "relative": 0.000005 } } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "cbd5685911d8da6af29f1cbddad7a6bc8e977aadeebc90a27ef3e9e7b6b97b6b" }, { "task_id": "v2-eval/orbital-mechanics/propagate_orbit_j2/06-metamorphic_perturbation", "split": "evaluation", "skill": "orbital-mechanics", "operation": "propagate_orbit_j2", "case_kind": "metamorphic_perturbation", "expected_status": "success", "reference_input": { "initial_state": { "semi_major_axis_m": 7000000, "eccentricity": 0.01, "inclination_deg": 28.5, "raan_deg": 40, "arg_perigee_deg": 15, "true_anomaly_deg": 25 }, "propagation_time_s": 600, "time_step_s": 60 }, "answers": [ { "quantity": "final propagated position", "value": [ -2697018.3854, 5542512.4207, 3245086.288 ], "unit": "m", "accepted_units": [ "m" ], "grader": "vector", "tolerance": { "absolute": 20, "relative": 0.000005, "rationale": "Operation-specific acceptance bound from the canonical dossier for final propagated position; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.states[$last].position_m", "runtime_source_path": "$.states[$last].position_m", "source_citations": [ "nasa-gmat" ], "component_tolerance": { "absolute": 20, "relative": 0.000005 } }, { "quantity": "final propagated velocity", "value": [ -6617.865442, -3611.129557, 802.610838 ], "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "vector", "tolerance": { "absolute": 0.03, "relative": 0.000005, "rationale": "Operation-specific acceptance bound from the canonical dossier for final propagated velocity; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.states[$last].velocity_ms", "runtime_source_path": "$.states[$last].velocity_ms", "source_citations": [ "nasa-gmat" ], "component_tolerance": { "absolute": 0.03, "relative": 0.000005 } } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "cbd5685911d8da6af29f1cbddad7a6bc8e977aadeebc90a27ef3e9e7b6b97b6b" }, { "task_id": "v2-eval/orbital-mechanics/propagate_orbit_j2/07-underspecified", "split": "evaluation", "skill": "orbital-mechanics", "operation": "propagate_orbit_j2", "case_kind": "underspecified", "expected_status": "needs_clarification", "reference_input": null, "answers": [], "status_rationale": "The prompt intentionally omits information needed to determine a unique numerical or retrieval answer, so guessing would invalidate the comparison.", "tool_required": false, "dossier_sha256": "cbd5685911d8da6af29f1cbddad7a6bc8e977aadeebc90a27ef3e9e7b6b97b6b" }, { "task_id": "v2-eval/orbital-mechanics/propagate_orbit_j2/08-invalid_or_out_of_domain", "split": "evaluation", "skill": "orbital-mechanics", "operation": "propagate_orbit_j2", "case_kind": "invalid_or_out_of_domain", "expected_status": "out_of_domain", "reference_input": null, "answers": [], "status_rationale": "Every public operation explicitly prohibits certification, operational control, and safety-of-life use; this request crosses that documented boundary.", "tool_required": false, "dossier_sha256": "cbd5685911d8da6af29f1cbddad7a6bc8e977aadeebc90a27ef3e9e7b6b97b6b" }, { "task_id": "v2-eval/orbital-mechanics/calculate_ground_track/01-canonical_reference", "split": "evaluation", "skill": "orbital-mechanics", "operation": "calculate_ground_track", "case_kind": "canonical_reference", "expected_status": "success", "reference_input": { "orbital_state": { "semi_major_axis_m": 7000000, "eccentricity": 0.01, "inclination_deg": 28.5, "raan_deg": 40, "arg_perigee_deg": 15, "true_anomaly_deg": 25 }, "duration_s": 180, "time_step_s": 60 }, "answers": [ { "quantity": "final ground-track latitude", "value": 21.865757, "unit": "deg", "accepted_units": [ "deg" ], "grader": "scalar", "tolerance": { "absolute": 0.002, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for final ground-track latitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.points[$last].latitude_deg", "runtime_source_path": "$.points[$last].latitude_deg", "source_citations": [ "nasa-gmat" ] }, { "quantity": "final ground-track longitude", "value": 86.906444, "unit": "deg", "accepted_units": [ "deg" ], "grader": "scalar", "tolerance": { "absolute": 0.003, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for final ground-track longitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.points[$last].longitude_deg", "runtime_source_path": "$.points[$last].longitude_deg", "source_citations": [ "nasa-gmat" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "036ad7b0e15e07083fcf18a508c7e638670860cae2b7f853f847ff65a9398ac0" }, { "task_id": "v2-eval/orbital-mechanics/calculate_ground_track/02-realistic_in_domain", "split": "evaluation", "skill": "orbital-mechanics", "operation": "calculate_ground_track", "case_kind": "realistic_in_domain", "expected_status": "success", "reference_input": { "orbital_state": { "semi_major_axis_m": 7000000, "eccentricity": 0.01, "inclination_deg": 28.5, "raan_deg": 40, "arg_perigee_deg": 15, "true_anomaly_deg": 25 }, "duration_s": 60, "time_step_s": 60 }, "answers": [ { "quantity": "final ground-track latitude", "value": 19.27469, "unit": "deg", "accepted_units": [ "deg" ], "grader": "scalar", "tolerance": { "absolute": 0.002, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for final ground-track latitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.points[$last].latitude_deg", "runtime_source_path": "$.points[$last].latitude_deg", "source_citations": [ "nasa-gmat" ] }, { "quantity": "final ground-track longitude", "value": 79.845254, "unit": "deg", "accepted_units": [ "deg" ], "grader": "scalar", "tolerance": { "absolute": 0.003, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for final ground-track longitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.points[$last].longitude_deg", "runtime_source_path": "$.points[$last].longitude_deg", "source_citations": [ "nasa-gmat" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "036ad7b0e15e07083fcf18a508c7e638670860cae2b7f853f847ff65a9398ac0" }, { "task_id": "v2-eval/orbital-mechanics/calculate_ground_track/03-alternate_units", "split": "evaluation", "skill": "orbital-mechanics", "operation": "calculate_ground_track", "case_kind": "alternate_units", "expected_status": "success", "reference_input": { "orbital_state": { "semi_major_axis_m": 7000000, "eccentricity": 0.01, "inclination_deg": 28.5, "raan_deg": 40, "arg_perigee_deg": 15, "true_anomaly_deg": 25 }, "duration_s": 120, "time_step_s": 60 }, "answers": [ { "quantity": "final ground-track latitude", "value": 20.611884, "unit": "deg", "accepted_units": [ "deg" ], "grader": "scalar", "tolerance": { "absolute": 0.002, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for final ground-track latitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.points[$last].latitude_deg", "runtime_source_path": "$.points[$last].latitude_deg", "source_citations": [ "nasa-gmat" ] }, { "quantity": "final ground-track longitude", "value": 83.345006, "unit": "deg", "accepted_units": [ "deg" ], "grader": "scalar", "tolerance": { "absolute": 0.003, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for final ground-track longitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.points[$last].longitude_deg", "runtime_source_path": "$.points[$last].longitude_deg", "source_citations": [ "nasa-gmat" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "036ad7b0e15e07083fcf18a508c7e638670860cae2b7f853f847ff65a9398ac0" }, { "task_id": "v2-eval/orbital-mechanics/calculate_ground_track/04-validation_boundary", "split": "evaluation", "skill": "orbital-mechanics", "operation": "calculate_ground_track", "case_kind": "validation_boundary", "expected_status": "success", "reference_input": { "orbital_state": { "semi_major_axis_m": 7000000, "eccentricity": 0.01, "inclination_deg": 28.5, "raan_deg": 40, "arg_perigee_deg": 15, "true_anomaly_deg": 25 }, "duration_s": 240, "time_step_s": 60 }, "answers": [ { "quantity": "final ground-track latitude", "value": 23.029403, "unit": "deg", "accepted_units": [ "deg" ], "grader": "scalar", "tolerance": { "absolute": 0.002, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for final ground-track latitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.points[$last].latitude_deg", "runtime_source_path": "$.points[$last].latitude_deg", "source_citations": [ "nasa-gmat" ] }, { "quantity": "final ground-track longitude", "value": 90.530044, "unit": "deg", "accepted_units": [ "deg" ], "grader": "scalar", "tolerance": { "absolute": 0.003, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for final ground-track longitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.points[$last].longitude_deg", "runtime_source_path": "$.points[$last].longitude_deg", "source_citations": [ "nasa-gmat" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "036ad7b0e15e07083fcf18a508c7e638670860cae2b7f853f847ff65a9398ac0" }, { "task_id": "v2-eval/orbital-mechanics/calculate_ground_track/05-metamorphic_baseline", "split": "evaluation", "skill": "orbital-mechanics", "operation": "calculate_ground_track", "case_kind": "metamorphic_baseline", "expected_status": "success", "reference_input": { "orbital_state": { "semi_major_axis_m": 7000000, "eccentricity": 0.01, "inclination_deg": 28.5, "raan_deg": 40, "arg_perigee_deg": 15, "true_anomaly_deg": 25 }, "duration_s": 300, "time_step_s": 60 }, "answers": [ { "quantity": "final ground-track latitude", "value": 24.096098, "unit": "deg", "accepted_units": [ "deg" ], "grader": "scalar", "tolerance": { "absolute": 0.002, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for final ground-track latitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.points[$last].latitude_deg", "runtime_source_path": "$.points[$last].latitude_deg", "source_citations": [ "nasa-gmat" ] }, { "quantity": "final ground-track longitude", "value": 94.215064, "unit": "deg", "accepted_units": [ "deg" ], "grader": "scalar", "tolerance": { "absolute": 0.003, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for final ground-track longitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.points[$last].longitude_deg", "runtime_source_path": "$.points[$last].longitude_deg", "source_citations": [ "nasa-gmat" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "036ad7b0e15e07083fcf18a508c7e638670860cae2b7f853f847ff65a9398ac0" }, { "task_id": "v2-eval/orbital-mechanics/calculate_ground_track/06-metamorphic_perturbation", "split": "evaluation", "skill": "orbital-mechanics", "operation": "calculate_ground_track", "case_kind": "metamorphic_perturbation", "expected_status": "success", "reference_input": { "orbital_state": { "semi_major_axis_m": 7000000, "eccentricity": 0.01, "inclination_deg": 28.5, "raan_deg": 40, "arg_perigee_deg": 15, "true_anomaly_deg": 25 }, "duration_s": 600, "time_step_s": 60 }, "answers": [ { "quantity": "final ground-track latitude", "value": 27.765395, "unit": "deg", "accepted_units": [ "deg" ], "grader": "scalar", "tolerance": { "absolute": 0.002, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for final ground-track latitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.points[$last].latitude_deg", "runtime_source_path": "$.points[$last].latitude_deg", "source_citations": [ "nasa-gmat" ] }, { "quantity": "final ground-track longitude", "value": 113.440976, "unit": "deg", "accepted_units": [ "deg" ], "grader": "scalar", "tolerance": { "absolute": 0.003, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for final ground-track longitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.points[$last].longitude_deg", "runtime_source_path": "$.points[$last].longitude_deg", "source_citations": [ "nasa-gmat" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "036ad7b0e15e07083fcf18a508c7e638670860cae2b7f853f847ff65a9398ac0" }, { "task_id": "v2-eval/orbital-mechanics/calculate_ground_track/07-underspecified", "split": "evaluation", "skill": "orbital-mechanics", "operation": "calculate_ground_track", "case_kind": "underspecified", "expected_status": "needs_clarification", "reference_input": null, "answers": [], "status_rationale": "The prompt intentionally omits information needed to determine a unique numerical or retrieval answer, so guessing would invalidate the comparison.", "tool_required": false, "dossier_sha256": "036ad7b0e15e07083fcf18a508c7e638670860cae2b7f853f847ff65a9398ac0" }, { "task_id": "v2-eval/orbital-mechanics/calculate_ground_track/08-invalid_or_out_of_domain", "split": "evaluation", "skill": "orbital-mechanics", "operation": "calculate_ground_track", "case_kind": "invalid_or_out_of_domain", "expected_status": "out_of_domain", "reference_input": null, "answers": [], "status_rationale": "Every public operation explicitly prohibits certification, operational control, and safety-of-life use; this request crosses that documented boundary.", "tool_required": false, "dossier_sha256": "036ad7b0e15e07083fcf18a508c7e638670860cae2b7f853f847ff65a9398ac0" }, { "task_id": "v2-eval/orbital-mechanics/hohmann_transfer/01-canonical_reference", "split": "evaluation", "skill": "orbital-mechanics", "operation": "hohmann_transfer", "case_kind": "canonical_reference", "expected_status": "success", "reference_input": { "r1_m": 6678137, "r2_m": 42164137 }, "answers": [ { "quantity": "total transfer delta-v", "value": 3892.556514, "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "scalar", "tolerance": { "absolute": 0.00001, "relative": 2e-8, "rationale": "Operation-specific acceptance bound from the canonical dossier for total transfer delta-v; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.total_delta_v_ms", "runtime_source_path": "$.total_delta_v_ms", "source_citations": [ "nasa-gmat" ] }, { "quantity": "transfer time of flight", "value": 18990.211638, "unit": "s", "accepted_units": [ "s" ], "grader": "scalar", "tolerance": { "absolute": 0.0001, "relative": 2e-8, "rationale": "Operation-specific acceptance bound from the canonical dossier for transfer time of flight; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.time_of_flight_s", "runtime_source_path": "$.time_of_flight_s", "source_citations": [ "nasa-gmat" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "10e4f30cb6d5f43b8621aa04c79be3264b263c22a64a54ddace27c7f088c9fc8" }, { "task_id": "v2-eval/orbital-mechanics/hohmann_transfer/02-realistic_in_domain", "split": "evaluation", "skill": "orbital-mechanics", "operation": "hohmann_transfer", "case_kind": "realistic_in_domain", "expected_status": "success", "reference_input": { "r1_m": 6678137, "r2_m": 8000000 }, "answers": [ { "quantity": "total transfer delta-v", "value": 665.717757, "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "scalar", "tolerance": { "absolute": 0.00001, "relative": 2e-8, "rationale": "Operation-specific acceptance bound from the canonical dossier for total transfer delta-v; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.total_delta_v_ms", "runtime_source_path": "$.total_delta_v_ms", "source_citations": [ "nasa-gmat" ] }, { "quantity": "transfer time of flight", "value": 3128.544932, "unit": "s", "accepted_units": [ "s" ], "grader": "scalar", "tolerance": { "absolute": 0.0001, "relative": 2e-8, "rationale": "Operation-specific acceptance bound from the canonical dossier for transfer time of flight; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.time_of_flight_s", "runtime_source_path": "$.time_of_flight_s", "source_citations": [ "nasa-gmat" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "10e4f30cb6d5f43b8621aa04c79be3264b263c22a64a54ddace27c7f088c9fc8" }, { "task_id": "v2-eval/orbital-mechanics/hohmann_transfer/03-alternate_units", "split": "evaluation", "skill": "orbital-mechanics", "operation": "hohmann_transfer", "case_kind": "alternate_units", "expected_status": "success", "reference_input": { "r1_m": 6678137, "r2_m": 12000000 }, "answers": [ { "quantity": "total transfer delta-v", "value": 1921.483742, "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "scalar", "tolerance": { "absolute": 0.00001, "relative": 2e-8, "rationale": "Operation-specific acceptance bound from the canonical dossier for total transfer delta-v; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.total_delta_v_ms", "runtime_source_path": "$.total_delta_v_ms", "source_citations": [ "nasa-gmat" ] }, { "quantity": "transfer time of flight", "value": 4490.930417, "unit": "s", "accepted_units": [ "s" ], "grader": "scalar", "tolerance": { "absolute": 0.0001, "relative": 2e-8, "rationale": "Operation-specific acceptance bound from the canonical dossier for transfer time of flight; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.time_of_flight_s", "runtime_source_path": "$.time_of_flight_s", "source_citations": [ "nasa-gmat" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "10e4f30cb6d5f43b8621aa04c79be3264b263c22a64a54ddace27c7f088c9fc8" }, { "task_id": "v2-eval/orbital-mechanics/hohmann_transfer/04-validation_boundary", "split": "evaluation", "skill": "orbital-mechanics", "operation": "hohmann_transfer", "case_kind": "validation_boundary", "expected_status": "success", "reference_input": { "r1_m": 6678137, "r2_m": 20000000 }, "answers": [ { "quantity": "total transfer delta-v", "value": 3039.820188, "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "scalar", "tolerance": { "absolute": 0.00001, "relative": 2e-8, "rationale": "Operation-specific acceptance bound from the canonical dossier for total transfer delta-v; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.total_delta_v_ms", "runtime_source_path": "$.total_delta_v_ms", "source_citations": [ "nasa-gmat" ] }, { "quantity": "transfer time of flight", "value": 7666.007486, "unit": "s", "accepted_units": [ "s" ], "grader": "scalar", "tolerance": { "absolute": 0.0001, "relative": 2e-8, "rationale": "Operation-specific acceptance bound from the canonical dossier for transfer time of flight; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.time_of_flight_s", "runtime_source_path": "$.time_of_flight_s", "source_citations": [ "nasa-gmat" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "10e4f30cb6d5f43b8621aa04c79be3264b263c22a64a54ddace27c7f088c9fc8" }, { "task_id": "v2-eval/orbital-mechanics/hohmann_transfer/05-metamorphic_baseline", "split": "evaluation", "skill": "orbital-mechanics", "operation": "hohmann_transfer", "case_kind": "metamorphic_baseline", "expected_status": "success", "reference_input": { "r1_m": 6678137, "r2_m": 30000000 }, "answers": [ { "quantity": "total transfer delta-v", "value": 3600.991948, "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "scalar", "tolerance": { "absolute": 0.00001, "relative": 2e-8, "rationale": "Operation-specific acceptance bound from the canonical dossier for total transfer delta-v; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.total_delta_v_ms", "runtime_source_path": "$.total_delta_v_ms", "source_citations": [ "nasa-gmat" ] }, { "quantity": "transfer time of flight", "value": 12357.963093, "unit": "s", "accepted_units": [ "s" ], "grader": "scalar", "tolerance": { "absolute": 0.0001, "relative": 2e-8, "rationale": "Operation-specific acceptance bound from the canonical dossier for transfer time of flight; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.time_of_flight_s", "runtime_source_path": "$.time_of_flight_s", "source_citations": [ "nasa-gmat" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "10e4f30cb6d5f43b8621aa04c79be3264b263c22a64a54ddace27c7f088c9fc8" }, { "task_id": "v2-eval/orbital-mechanics/hohmann_transfer/06-metamorphic_perturbation", "split": "evaluation", "skill": "orbital-mechanics", "operation": "hohmann_transfer", "case_kind": "metamorphic_perturbation", "expected_status": "success", "reference_input": { "r1_m": 6678137, "r2_m": 50000000 }, "answers": [ { "quantity": "total transfer delta-v", "value": 3989.122853, "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "scalar", "tolerance": { "absolute": 0.00001, "relative": 2e-8, "rationale": "Operation-specific acceptance bound from the canonical dossier for total transfer delta-v; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.total_delta_v_ms", "runtime_source_path": "$.total_delta_v_ms", "source_citations": [ "nasa-gmat" ] }, { "quantity": "transfer time of flight", "value": 23738.83169, "unit": "s", "accepted_units": [ "s" ], "grader": "scalar", "tolerance": { "absolute": 0.0001, "relative": 2e-8, "rationale": "Operation-specific acceptance bound from the canonical dossier for transfer time of flight; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.time_of_flight_s", "runtime_source_path": "$.time_of_flight_s", "source_citations": [ "nasa-gmat" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "10e4f30cb6d5f43b8621aa04c79be3264b263c22a64a54ddace27c7f088c9fc8" }, { "task_id": "v2-eval/orbital-mechanics/hohmann_transfer/07-underspecified", "split": "evaluation", "skill": "orbital-mechanics", "operation": "hohmann_transfer", "case_kind": "underspecified", "expected_status": "needs_clarification", "reference_input": null, "answers": [], "status_rationale": "The prompt intentionally omits information needed to determine a unique numerical or retrieval answer, so guessing would invalidate the comparison.", "tool_required": false, "dossier_sha256": "10e4f30cb6d5f43b8621aa04c79be3264b263c22a64a54ddace27c7f088c9fc8" }, { "task_id": "v2-eval/orbital-mechanics/hohmann_transfer/08-invalid_or_out_of_domain", "split": "evaluation", "skill": "orbital-mechanics", "operation": "hohmann_transfer", "case_kind": "invalid_or_out_of_domain", "expected_status": "out_of_domain", "reference_input": null, "answers": [], "status_rationale": "Every public operation explicitly prohibits certification, operational control, and safety-of-life use; this request crosses that documented boundary.", "tool_required": false, "dossier_sha256": "10e4f30cb6d5f43b8621aa04c79be3264b263c22a64a54ddace27c7f088c9fc8" }, { "task_id": "v2-eval/orbital-mechanics/orbital_rendezvous_planning/01-canonical_reference", "split": "evaluation", "skill": "orbital-mechanics", "operation": "orbital_rendezvous_planning", "case_kind": "canonical_reference", "expected_status": "success", "reference_input": { "chaser_elements": { "semi_major_axis_m": 7000000, "eccentricity": 0.01, "inclination_deg": 28.5, "raan_deg": 40, "arg_perigee_deg": 15, "true_anomaly_deg": 25 }, "target_elements": { "semi_major_axis_m": 7200000, "eccentricity": 0.01, "inclination_deg": 28.5, "raan_deg": 40, "arg_perigee_deg": 15, "true_anomaly_deg": 25 } }, "answers": [ { "quantity": "required phase angle", "value": 3.7369, "unit": "deg", "accepted_units": [ "deg" ], "grader": "scalar", "tolerance": { "absolute": 0.001, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for required phase angle; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.required_phase_angle_deg", "runtime_source_path": "$.required_phase_angle_deg", "source_citations": [ "nasa-gmat" ] }, { "quantity": "rendezvous transfer delta-v", "value": 105.53917, "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "scalar", "tolerance": { "absolute": 0.00001, "relative": 2e-8, "rationale": "Operation-specific acceptance bound from the canonical dossier for rendezvous transfer delta-v; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.transfer.total_delta_v_ms", "runtime_source_path": "$.transfer.total_delta_v_ms", "source_citations": [ "nasa-gmat" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "4e7c7fddf1b6e5fe1c5887c2f4c90b4c0fb91b88503e5b9ce686335c7101852c" }, { "task_id": "v2-eval/orbital-mechanics/orbital_rendezvous_planning/02-realistic_in_domain", "split": "evaluation", "skill": "orbital-mechanics", "operation": "orbital_rendezvous_planning", "case_kind": "realistic_in_domain", "expected_status": "success", "reference_input": { "chaser_elements": { "semi_major_axis_m": 7000000, "eccentricity": 0.01, "inclination_deg": 28.5, "raan_deg": 40, "arg_perigee_deg": 15, "true_anomaly_deg": 25 }, "target_elements": { "semi_major_axis_m": 7050000, "eccentricity": 0.01, "inclination_deg": 28.5, "raan_deg": 40, "arg_perigee_deg": 15, "true_anomaly_deg": 25 } }, "answers": [ { "quantity": "required phase angle", "value": 0.9566, "unit": "deg", "accepted_units": [ "deg" ], "grader": "scalar", "tolerance": { "absolute": 0.001, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for required phase angle; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.required_phase_angle_deg", "runtime_source_path": "$.required_phase_angle_deg", "source_citations": [ "nasa-gmat" ] }, { "quantity": "rendezvous transfer delta-v", "value": 26.806583, "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "scalar", "tolerance": { "absolute": 0.00001, "relative": 2e-8, "rationale": "Operation-specific acceptance bound from the canonical dossier for rendezvous transfer delta-v; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.transfer.total_delta_v_ms", "runtime_source_path": "$.transfer.total_delta_v_ms", "source_citations": [ "nasa-gmat" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "4e7c7fddf1b6e5fe1c5887c2f4c90b4c0fb91b88503e5b9ce686335c7101852c" }, { "task_id": "v2-eval/orbital-mechanics/orbital_rendezvous_planning/03-alternate_units", "split": "evaluation", "skill": "orbital-mechanics", "operation": "orbital_rendezvous_planning", "case_kind": "alternate_units", "expected_status": "success", "reference_input": { "chaser_elements": { "semi_major_axis_m": 7000000, "eccentricity": 0.01, "inclination_deg": 28.5, "raan_deg": 40, "arg_perigee_deg": 15, "true_anomaly_deg": 25 }, "target_elements": { "semi_major_axis_m": 7100000, "eccentricity": 0.01, "inclination_deg": 28.5, "raan_deg": 40, "arg_perigee_deg": 15, "true_anomaly_deg": 25 } }, "answers": [ { "quantity": "required phase angle", "value": 1.8981, "unit": "deg", "accepted_units": [ "deg" ], "grader": "scalar", "tolerance": { "absolute": 0.001, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for required phase angle; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.required_phase_angle_deg", "runtime_source_path": "$.required_phase_angle_deg", "source_citations": [ "nasa-gmat" ] }, { "quantity": "rendezvous transfer delta-v", "value": 53.328996, "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "scalar", "tolerance": { "absolute": 0.00001, "relative": 2e-8, "rationale": "Operation-specific acceptance bound from the canonical dossier for rendezvous transfer delta-v; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.transfer.total_delta_v_ms", "runtime_source_path": "$.transfer.total_delta_v_ms", "source_citations": [ "nasa-gmat" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "4e7c7fddf1b6e5fe1c5887c2f4c90b4c0fb91b88503e5b9ce686335c7101852c" }, { "task_id": "v2-eval/orbital-mechanics/orbital_rendezvous_planning/04-validation_boundary", "split": "evaluation", "skill": "orbital-mechanics", "operation": "orbital_rendezvous_planning", "case_kind": "validation_boundary", "expected_status": "success", "reference_input": { "chaser_elements": { "semi_major_axis_m": 7000000, "eccentricity": 0.01, "inclination_deg": 28.5, "raan_deg": 40, "arg_perigee_deg": 15, "true_anomaly_deg": 25 }, "target_elements": { "semi_major_axis_m": 7300000, "eccentricity": 0.01, "inclination_deg": 28.5, "raan_deg": 40, "arg_perigee_deg": 15, "true_anomaly_deg": 25 } }, "answers": [ { "quantity": "required phase angle", "value": 5.5193, "unit": "deg", "accepted_units": [ "deg" ], "grader": "scalar", "tolerance": { "absolute": 0.001, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for required phase angle; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.required_phase_angle_deg", "runtime_source_path": "$.required_phase_angle_deg", "source_citations": [ "nasa-gmat" ] }, { "quantity": "rendezvous transfer delta-v", "value": 156.665287, "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "scalar", "tolerance": { "absolute": 0.00001, "relative": 2e-8, "rationale": "Operation-specific acceptance bound from the canonical dossier for rendezvous transfer delta-v; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.transfer.total_delta_v_ms", "runtime_source_path": "$.transfer.total_delta_v_ms", "source_citations": [ "nasa-gmat" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "4e7c7fddf1b6e5fe1c5887c2f4c90b4c0fb91b88503e5b9ce686335c7101852c" }, { "task_id": "v2-eval/orbital-mechanics/orbital_rendezvous_planning/05-metamorphic_baseline", "split": "evaluation", "skill": "orbital-mechanics", "operation": "orbital_rendezvous_planning", "case_kind": "metamorphic_baseline", "expected_status": "success", "reference_input": { "chaser_elements": { "semi_major_axis_m": 7000000, "eccentricity": 0.01, "inclination_deg": 28.5, "raan_deg": 40, "arg_perigee_deg": 15, "true_anomaly_deg": 25 }, "target_elements": { "semi_major_axis_m": 7500000, "eccentricity": 0.01, "inclination_deg": 28.5, "raan_deg": 40, "arg_perigee_deg": 15, "true_anomaly_deg": 25 } }, "answers": [ { "quantity": "required phase angle", "value": 8.9246, "unit": "deg", "accepted_units": [ "deg" ], "grader": "scalar", "tolerance": { "absolute": 0.001, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for required phase angle; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.required_phase_angle_deg", "runtime_source_path": "$.required_phase_angle_deg", "source_citations": [ "nasa-gmat" ] }, { "quantity": "rendezvous transfer delta-v", "value": 255.797121, "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "scalar", "tolerance": { "absolute": 0.00001, "relative": 2e-8, "rationale": "Operation-specific acceptance bound from the canonical dossier for rendezvous transfer delta-v; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.transfer.total_delta_v_ms", "runtime_source_path": "$.transfer.total_delta_v_ms", "source_citations": [ "nasa-gmat" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "4e7c7fddf1b6e5fe1c5887c2f4c90b4c0fb91b88503e5b9ce686335c7101852c" }, { "task_id": "v2-eval/orbital-mechanics/orbital_rendezvous_planning/06-metamorphic_perturbation", "split": "evaluation", "skill": "orbital-mechanics", "operation": "orbital_rendezvous_planning", "case_kind": "metamorphic_perturbation", "expected_status": "success", "reference_input": { "chaser_elements": { "semi_major_axis_m": 7000000, "eccentricity": 0.01, "inclination_deg": 28.5, "raan_deg": 40, "arg_perigee_deg": 15, "true_anomaly_deg": 25 }, "target_elements": { "semi_major_axis_m": 8000000, "eccentricity": 0.01, "inclination_deg": 28.5, "raan_deg": 40, "arg_perigee_deg": 15, "true_anomaly_deg": 25 } }, "answers": [ { "quantity": "required phase angle", "value": 16.6085, "unit": "deg", "accepted_units": [ "deg" ], "grader": "scalar", "tolerance": { "absolute": 0.001, "relative": 0, "rationale": "Operation-specific acceptance bound from the canonical dossier for required phase angle; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.required_phase_angle_deg", "runtime_source_path": "$.required_phase_angle_deg", "source_citations": [ "nasa-gmat" ] }, { "quantity": "rendezvous transfer delta-v", "value": 486.824509, "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "scalar", "tolerance": { "absolute": 0.00001, "relative": 2e-8, "rationale": "Operation-specific acceptance bound from the canonical dossier for rendezvous transfer delta-v; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.transfer.total_delta_v_ms", "runtime_source_path": "$.transfer.total_delta_v_ms", "source_citations": [ "nasa-gmat" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "4e7c7fddf1b6e5fe1c5887c2f4c90b4c0fb91b88503e5b9ce686335c7101852c" }, { "task_id": "v2-eval/orbital-mechanics/orbital_rendezvous_planning/07-underspecified", "split": "evaluation", "skill": "orbital-mechanics", "operation": "orbital_rendezvous_planning", "case_kind": "underspecified", "expected_status": "needs_clarification", "reference_input": null, "answers": [], "status_rationale": "The prompt intentionally omits information needed to determine a unique numerical or retrieval answer, so guessing would invalidate the comparison.", "tool_required": false, "dossier_sha256": "4e7c7fddf1b6e5fe1c5887c2f4c90b4c0fb91b88503e5b9ce686335c7101852c" }, { "task_id": "v2-eval/orbital-mechanics/orbital_rendezvous_planning/08-invalid_or_out_of_domain", "split": "evaluation", "skill": "orbital-mechanics", "operation": "orbital_rendezvous_planning", "case_kind": "invalid_or_out_of_domain", "expected_status": "out_of_domain", "reference_input": null, "answers": [], "status_rationale": "Every public operation explicitly prohibits certification, operational control, and safety-of-life use; this request crosses that documented boundary.", "tool_required": false, "dossier_sha256": "4e7c7fddf1b6e5fe1c5887c2f4c90b4c0fb91b88503e5b9ce686335c7101852c" }, { "task_id": "v2-eval/orbital-mechanics/lambert_problem_solver/01-canonical_reference", "split": "evaluation", "skill": "orbital-mechanics", "operation": "lambert_problem_solver", "case_kind": "canonical_reference", "expected_status": "success", "reference_input": { "r1_m": [ 7000000, 0, 0 ], "r2_m": [ 0, 7000000, 0 ], "tof_s": 1800, "central_body": "earth", "transfer_direction": "prograde", "long_way": false }, "answers": [ { "quantity": "Lambert departure velocity", "value": [ 1413.21741478, 6872.45562289, 0 ], "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "vector", "tolerance": { "absolute": 0.02, "relative": 0.000003, "rationale": "Operation-specific acceptance bound from the canonical dossier for Lambert departure velocity; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.v1_ms", "runtime_source_path": "$.v1_ms", "source_citations": [ "nasa-gmat", "jpl-spice" ], "component_tolerance": { "absolute": 0.02, "relative": 0.000003 } }, { "quantity": "Lambert arrival velocity", "value": [ -6872.45562289, -1413.21741478, 0 ], "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "vector", "tolerance": { "absolute": 0.02, "relative": 0.000003, "rationale": "Operation-specific acceptance bound from the canonical dossier for Lambert arrival velocity; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.v2_ms", "runtime_source_path": "$.v2_ms", "source_citations": [ "nasa-gmat", "jpl-spice" ], "component_tolerance": { "absolute": 0.02, "relative": 0.000003 } } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "8d434a7d8d0275dd858a4e65c5fb73d66569d9cf65311e5a0a4867b9c6bc2184" }, { "task_id": "v2-eval/orbital-mechanics/lambert_problem_solver/02-realistic_in_domain", "split": "evaluation", "skill": "orbital-mechanics", "operation": "lambert_problem_solver", "case_kind": "realistic_in_domain", "expected_status": "success", "reference_input": { "r1_m": [ 7000000, 0, 0 ], "r2_m": [ 0, 7000000, 0 ], "tof_s": 1500, "central_body": "earth", "transfer_direction": "prograde", "long_way": false }, "answers": [ { "quantity": "Lambert departure velocity", "value": [ 206.9250588, 7443.3000051, 0 ], "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "vector", "tolerance": { "absolute": 0.02, "relative": 0.000003, "rationale": "Operation-specific acceptance bound from the canonical dossier for Lambert departure velocity; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.v1_ms", "runtime_source_path": "$.v1_ms", "source_citations": [ "nasa-gmat", "jpl-spice" ], "component_tolerance": { "absolute": 0.02, "relative": 0.000003 } }, { "quantity": "Lambert arrival velocity", "value": [ -7443.3000051, -206.9250588, 0 ], "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "vector", "tolerance": { "absolute": 0.02, "relative": 0.000003, "rationale": "Operation-specific acceptance bound from the canonical dossier for Lambert arrival velocity; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.v2_ms", "runtime_source_path": "$.v2_ms", "source_citations": [ "nasa-gmat", "jpl-spice" ], "component_tolerance": { "absolute": 0.02, "relative": 0.000003 } } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "8d434a7d8d0275dd858a4e65c5fb73d66569d9cf65311e5a0a4867b9c6bc2184" }, { "task_id": "v2-eval/orbital-mechanics/lambert_problem_solver/03-alternate_units", "split": "evaluation", "skill": "orbital-mechanics", "operation": "lambert_problem_solver", "case_kind": "alternate_units", "expected_status": "success", "reference_input": { "r1_m": [ 7000000, 0, 0 ], "r2_m": [ 0, 7000000, 0 ], "tof_s": 1650, "central_body": "earth", "transfer_direction": "prograde", "long_way": false }, "answers": [ { "quantity": "Lambert departure velocity", "value": [ 857.35458232, 7129.54237204, 0 ], "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "vector", "tolerance": { "absolute": 0.02, "relative": 0.000003, "rationale": "Operation-specific acceptance bound from the canonical dossier for Lambert departure velocity; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.v1_ms", "runtime_source_path": "$.v1_ms", "source_citations": [ "nasa-gmat", "jpl-spice" ], "component_tolerance": { "absolute": 0.02, "relative": 0.000003 } }, { "quantity": "Lambert arrival velocity", "value": [ -7129.54237204, -857.35458232, 0 ], "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "vector", "tolerance": { "absolute": 0.02, "relative": 0.000003, "rationale": "Operation-specific acceptance bound from the canonical dossier for Lambert arrival velocity; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.v2_ms", "runtime_source_path": "$.v2_ms", "source_citations": [ "nasa-gmat", "jpl-spice" ], "component_tolerance": { "absolute": 0.02, "relative": 0.000003 } } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "8d434a7d8d0275dd858a4e65c5fb73d66569d9cf65311e5a0a4867b9c6bc2184" }, { "task_id": "v2-eval/orbital-mechanics/lambert_problem_solver/04-validation_boundary", "split": "evaluation", "skill": "orbital-mechanics", "operation": "lambert_problem_solver", "case_kind": "validation_boundary", "expected_status": "success", "reference_input": { "r1_m": [ 7000000, 0, 0 ], "r2_m": [ 0, 7000000, 0 ], "tof_s": 1950, "central_body": "earth", "transfer_direction": "prograde", "long_way": false }, "answers": [ { "quantity": "Lambert departure velocity", "value": [ 1894.43060729, 6658.05504583, 0 ], "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "vector", "tolerance": { "absolute": 0.02, "relative": 0.000003, "rationale": "Operation-specific acceptance bound from the canonical dossier for Lambert departure velocity; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.v1_ms", "runtime_source_path": "$.v1_ms", "source_citations": [ "nasa-gmat", "jpl-spice" ], "component_tolerance": { "absolute": 0.02, "relative": 0.000003 } }, { "quantity": "Lambert arrival velocity", "value": [ -6658.05504583, -1894.43060729, 0 ], "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "vector", "tolerance": { "absolute": 0.02, "relative": 0.000003, "rationale": "Operation-specific acceptance bound from the canonical dossier for Lambert arrival velocity; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.v2_ms", "runtime_source_path": "$.v2_ms", "source_citations": [ "nasa-gmat", "jpl-spice" ], "component_tolerance": { "absolute": 0.02, "relative": 0.000003 } } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "8d434a7d8d0275dd858a4e65c5fb73d66569d9cf65311e5a0a4867b9c6bc2184" }, { "task_id": "v2-eval/orbital-mechanics/lambert_problem_solver/05-metamorphic_baseline", "split": "evaluation", "skill": "orbital-mechanics", "operation": "lambert_problem_solver", "case_kind": "metamorphic_baseline", "expected_status": "success", "reference_input": { "r1_m": [ 7000000, 0, 0 ], "r2_m": [ 0, 7000000, 0 ], "tof_s": 2100, "central_body": "earth", "transfer_direction": "prograde", "long_way": false }, "answers": [ { "quantity": "Lambert departure velocity", "value": [ 2315.59003038, 6476.56216498, 0 ], "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "vector", "tolerance": { "absolute": 0.02, "relative": 0.000003, "rationale": "Operation-specific acceptance bound from the canonical dossier for Lambert departure velocity; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.v1_ms", "runtime_source_path": "$.v1_ms", "source_citations": [ "nasa-gmat", "jpl-spice" ], "component_tolerance": { "absolute": 0.02, "relative": 0.000003 } }, { "quantity": "Lambert arrival velocity", "value": [ -6476.56216498, -2315.59003038, 0 ], "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "vector", "tolerance": { "absolute": 0.02, "relative": 0.000003, "rationale": "Operation-specific acceptance bound from the canonical dossier for Lambert arrival velocity; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.v2_ms", "runtime_source_path": "$.v2_ms", "source_citations": [ "nasa-gmat", "jpl-spice" ], "component_tolerance": { "absolute": 0.02, "relative": 0.000003 } } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "8d434a7d8d0275dd858a4e65c5fb73d66569d9cf65311e5a0a4867b9c6bc2184" }, { "task_id": "v2-eval/orbital-mechanics/lambert_problem_solver/06-metamorphic_perturbation", "split": "evaluation", "skill": "orbital-mechanics", "operation": "lambert_problem_solver", "case_kind": "metamorphic_perturbation", "expected_status": "success", "reference_input": { "r1_m": [ 7000000, 0, 0 ], "r2_m": [ 0, 7000000, 0 ], "tof_s": 2400, "central_body": "earth", "transfer_direction": "prograde", "long_way": false }, "answers": [ { "quantity": "Lambert departure velocity", "value": [ 3019.03593893, 6186.03690054, 0 ], "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "vector", "tolerance": { "absolute": 0.02, "relative": 0.000003, "rationale": "Operation-specific acceptance bound from the canonical dossier for Lambert departure velocity; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.v1_ms", "runtime_source_path": "$.v1_ms", "source_citations": [ "nasa-gmat", "jpl-spice" ], "component_tolerance": { "absolute": 0.02, "relative": 0.000003 } }, { "quantity": "Lambert arrival velocity", "value": [ -6186.03690054, -3019.03593893, 0 ], "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "vector", "tolerance": { "absolute": 0.02, "relative": 0.000003, "rationale": "Operation-specific acceptance bound from the canonical dossier for Lambert arrival velocity; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/rocket_orbit.py $.v2_ms", "runtime_source_path": "$.v2_ms", "source_citations": [ "nasa-gmat", "jpl-spice" ], "component_tolerance": { "absolute": 0.02, "relative": 0.000003 } } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "8d434a7d8d0275dd858a4e65c5fb73d66569d9cf65311e5a0a4867b9c6bc2184" }, { "task_id": "v2-eval/orbital-mechanics/lambert_problem_solver/07-underspecified", "split": "evaluation", "skill": "orbital-mechanics", "operation": "lambert_problem_solver", "case_kind": "underspecified", "expected_status": "needs_clarification", "reference_input": null, "answers": [], "status_rationale": "The prompt intentionally omits information needed to determine a unique numerical or retrieval answer, so guessing would invalidate the comparison.", "tool_required": false, "dossier_sha256": "8d434a7d8d0275dd858a4e65c5fb73d66569d9cf65311e5a0a4867b9c6bc2184" }, { "task_id": "v2-eval/orbital-mechanics/lambert_problem_solver/08-invalid_or_out_of_domain", "split": "evaluation", "skill": "orbital-mechanics", "operation": "lambert_problem_solver", "case_kind": "invalid_or_out_of_domain", "expected_status": "out_of_domain", "reference_input": null, "answers": [], "status_rationale": "Every public operation explicitly prohibits certification, operational control, and safety-of-life use; this request crosses that documented boundary.", "tool_required": false, "dossier_sha256": "8d434a7d8d0275dd858a4e65c5fb73d66569d9cf65311e5a0a4867b9c6bc2184" }, { "task_id": "v2-eval/guidance-navigation-control/kalman_filter_state_estimation/01-canonical_reference", "split": "evaluation", "skill": "guidance-navigation-control", "operation": "kalman_filter_state_estimation", "case_kind": "canonical_reference", "expected_status": "success", "reference_input": { "initial_state": [ 0, 0 ], "initial_covariance": [ [ 1, 0 ], [ 0, 1 ] ], "process_noise": [ [ 0.01, 0 ], [ 0, 0.01 ] ], "measurement_noise": [ [ 0.2, 0 ], [ 0, 0.2 ] ], "measurements": [ { "z": [ 1, 1 ] }, { "z": [ 2, 1.8 ] } ], "dt": 1, "dynamics_model": "identity" }, "answers": [ { "quantity": "final state estimate", "value": [ 1.38171454, 1.28783162 ], "unit": "1", "accepted_units": [ "1" ], "grader": "vector", "tolerance": { "absolute": 2e-7, "relative": 2e-7, "rationale": "Operation-specific acceptance bound from the canonical dossier for final state estimate; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.final_state", "runtime_source_path": "$.final_state", "source_citations": [ "aerospace-skills-contract" ], "component_tolerance": { "absolute": 2e-7, "relative": 2e-7 } }, { "quantity": "final state standard deviation", "value": [ 0.3064032, 0.3064032 ], "unit": "1", "accepted_units": [ "1" ], "grader": "vector", "tolerance": { "absolute": 2e-7, "relative": 2e-7, "rationale": "Operation-specific acceptance bound from the canonical dossier for final state standard deviation; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.final_state_std", "runtime_source_path": "$.final_state_std", "source_citations": [ "aerospace-skills-contract" ], "component_tolerance": { "absolute": 2e-7, "relative": 2e-7 } } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "a03ddc038c965766936d1dc55f870de5f48c0d18ef404d262f21320d08ee9f4a" }, { "task_id": "v2-eval/guidance-navigation-control/kalman_filter_state_estimation/02-realistic_in_domain", "split": "evaluation", "skill": "guidance-navigation-control", "operation": "kalman_filter_state_estimation", "case_kind": "realistic_in_domain", "expected_status": "success", "reference_input": { "initial_state": [ 0, 0 ], "initial_covariance": [ [ 1, 0 ], [ 0, 1 ] ], "process_noise": [ [ 0.01, 0 ], [ 0, 0.01 ] ], "measurement_noise": [ [ 0.2, 0 ], [ 0, 0.2 ] ], "measurements": [ { "z": [ 0.5, 0.7 ] }, { "z": [ 1.5, 1.4 ] } ], "dt": 1, "dynamics_model": "identity" }, "answers": [ { "quantity": "final state estimate", "value": [ 0.92556457, 0.96720018 ], "unit": "1", "accepted_units": [ "1" ], "grader": "vector", "tolerance": { "absolute": 2e-7, "relative": 2e-7, "rationale": "Operation-specific acceptance bound from the canonical dossier for final state estimate; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.final_state", "runtime_source_path": "$.final_state", "source_citations": [ "aerospace-skills-contract" ], "component_tolerance": { "absolute": 2e-7, "relative": 2e-7 } }, { "quantity": "final state standard deviation", "value": [ 0.3064032, 0.3064032 ], "unit": "1", "accepted_units": [ "1" ], "grader": "vector", "tolerance": { "absolute": 2e-7, "relative": 2e-7, "rationale": "Operation-specific acceptance bound from the canonical dossier for final state standard deviation; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.final_state_std", "runtime_source_path": "$.final_state_std", "source_citations": [ "aerospace-skills-contract" ], "component_tolerance": { "absolute": 2e-7, "relative": 2e-7 } } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "a03ddc038c965766936d1dc55f870de5f48c0d18ef404d262f21320d08ee9f4a" }, { "task_id": "v2-eval/guidance-navigation-control/kalman_filter_state_estimation/03-alternate_units", "split": "evaluation", "skill": "guidance-navigation-control", "operation": "kalman_filter_state_estimation", "case_kind": "alternate_units", "expected_status": "success", "reference_input": { "initial_state": [ 0, 0 ], "initial_covariance": [ [ 1, 0 ], [ 0, 1 ] ], "process_noise": [ [ 0.01, 0 ], [ 0, 0.01 ] ], "measurement_noise": [ [ 0.2, 0 ], [ 0, 0.2 ] ], "measurements": [ { "z": [ 1.2, 0.8 ] }, { "z": [ 2.1, 2 ] } ], "dt": 1, "dynamics_model": "identity" }, "answers": [ { "quantity": "final state estimate", "value": [ 1.51723306, 1.29313747 ], "unit": "1", "accepted_units": [ "1" ], "grader": "vector", "tolerance": { "absolute": 2e-7, "relative": 2e-7, "rationale": "Operation-specific acceptance bound from the canonical dossier for final state estimate; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.final_state", "runtime_source_path": "$.final_state", "source_citations": [ "aerospace-skills-contract" ], "component_tolerance": { "absolute": 2e-7, "relative": 2e-7 } }, { "quantity": "final state standard deviation", "value": [ 0.3064032, 0.3064032 ], "unit": "1", "accepted_units": [ "1" ], "grader": "vector", "tolerance": { "absolute": 2e-7, "relative": 2e-7, "rationale": "Operation-specific acceptance bound from the canonical dossier for final state standard deviation; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.final_state_std", "runtime_source_path": "$.final_state_std", "source_citations": [ "aerospace-skills-contract" ], "component_tolerance": { "absolute": 2e-7, "relative": 2e-7 } } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "a03ddc038c965766936d1dc55f870de5f48c0d18ef404d262f21320d08ee9f4a" }, { "task_id": "v2-eval/guidance-navigation-control/kalman_filter_state_estimation/04-validation_boundary", "split": "evaluation", "skill": "guidance-navigation-control", "operation": "kalman_filter_state_estimation", "case_kind": "validation_boundary", "expected_status": "success", "reference_input": { "initial_state": [ 0, 0 ], "initial_covariance": [ [ 1, 0 ], [ 0, 1 ] ], "process_noise": [ [ 0.01, 0 ], [ 0, 0.01 ] ], "measurement_noise": [ [ 0.2, 0 ], [ 0, 0.2 ] ], "measurements": [ { "z": [ 0, 0 ] }, { "z": [ 0.2, -0.1 ] } ], "dt": 1, "dynamics_model": "identity" }, "answers": [ { "quantity": "final state estimate", "value": [ 0.09388292, -0.04694146 ], "unit": "1", "accepted_units": [ "1" ], "grader": "vector", "tolerance": { "absolute": 2e-7, "relative": 2e-7, "rationale": "Operation-specific acceptance bound from the canonical dossier for final state estimate; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.final_state", "runtime_source_path": "$.final_state", "source_citations": [ "aerospace-skills-contract" ], "component_tolerance": { "absolute": 2e-7, "relative": 2e-7 } }, { "quantity": "final state standard deviation", "value": [ 0.3064032, 0.3064032 ], "unit": "1", "accepted_units": [ "1" ], "grader": "vector", "tolerance": { "absolute": 2e-7, "relative": 2e-7, "rationale": "Operation-specific acceptance bound from the canonical dossier for final state standard deviation; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.final_state_std", "runtime_source_path": "$.final_state_std", "source_citations": [ "aerospace-skills-contract" ], "component_tolerance": { "absolute": 2e-7, "relative": 2e-7 } } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "a03ddc038c965766936d1dc55f870de5f48c0d18ef404d262f21320d08ee9f4a" }, { "task_id": "v2-eval/guidance-navigation-control/kalman_filter_state_estimation/05-metamorphic_baseline", "split": "evaluation", "skill": "guidance-navigation-control", "operation": "kalman_filter_state_estimation", "case_kind": "metamorphic_baseline", "expected_status": "success", "reference_input": { "initial_state": [ 0, 0 ], "initial_covariance": [ [ 1, 0 ], [ 0, 1 ] ], "process_noise": [ [ 0.01, 0 ], [ 0, 0.01 ] ], "measurement_noise": [ [ 0.2, 0 ], [ 0, 0.2 ] ], "measurements": [ { "z": [ 3, 2 ] }, { "z": [ 4, 3 ] } ], "dt": 1, "dynamics_model": "identity" }, "answers": [ { "quantity": "final state estimate", "value": [ 3.2063144, 2.29401447 ], "unit": "1", "accepted_units": [ "1" ], "grader": "vector", "tolerance": { "absolute": 2e-7, "relative": 2e-7, "rationale": "Operation-specific acceptance bound from the canonical dossier for final state estimate; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.final_state", "runtime_source_path": "$.final_state", "source_citations": [ "aerospace-skills-contract" ], "component_tolerance": { "absolute": 2e-7, "relative": 2e-7 } }, { "quantity": "final state standard deviation", "value": [ 0.3064032, 0.3064032 ], "unit": "1", "accepted_units": [ "1" ], "grader": "vector", "tolerance": { "absolute": 2e-7, "relative": 2e-7, "rationale": "Operation-specific acceptance bound from the canonical dossier for final state standard deviation; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.final_state_std", "runtime_source_path": "$.final_state_std", "source_citations": [ "aerospace-skills-contract" ], "component_tolerance": { "absolute": 2e-7, "relative": 2e-7 } } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "a03ddc038c965766936d1dc55f870de5f48c0d18ef404d262f21320d08ee9f4a" }, { "task_id": "v2-eval/guidance-navigation-control/kalman_filter_state_estimation/06-metamorphic_perturbation", "split": "evaluation", "skill": "guidance-navigation-control", "operation": "kalman_filter_state_estimation", "case_kind": "metamorphic_perturbation", "expected_status": "success", "reference_input": { "initial_state": [ 0, 0 ], "initial_covariance": [ [ 1, 0 ], [ 0, 1 ] ], "process_noise": [ [ 0.01, 0 ], [ 0, 0.01 ] ], "measurement_noise": [ [ 0.2, 0 ], [ 0, 0.2 ] ], "measurements": [ { "z": [ -1, 1 ] }, { "z": [ 0, 2 ] } ], "dt": 1, "dynamics_model": "identity" }, "answers": [ { "quantity": "final state estimate", "value": [ -0.44288533, 1.38171454 ], "unit": "1", "accepted_units": [ "1" ], "grader": "vector", "tolerance": { "absolute": 2e-7, "relative": 2e-7, "rationale": "Operation-specific acceptance bound from the canonical dossier for final state estimate; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.final_state", "runtime_source_path": "$.final_state", "source_citations": [ "aerospace-skills-contract" ], "component_tolerance": { "absolute": 2e-7, "relative": 2e-7 } }, { "quantity": "final state standard deviation", "value": [ 0.3064032, 0.3064032 ], "unit": "1", "accepted_units": [ "1" ], "grader": "vector", "tolerance": { "absolute": 2e-7, "relative": 2e-7, "rationale": "Operation-specific acceptance bound from the canonical dossier for final state standard deviation; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.final_state_std", "runtime_source_path": "$.final_state_std", "source_citations": [ "aerospace-skills-contract" ], "component_tolerance": { "absolute": 2e-7, "relative": 2e-7 } } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "a03ddc038c965766936d1dc55f870de5f48c0d18ef404d262f21320d08ee9f4a" }, { "task_id": "v2-eval/guidance-navigation-control/kalman_filter_state_estimation/07-underspecified", "split": "evaluation", "skill": "guidance-navigation-control", "operation": "kalman_filter_state_estimation", "case_kind": "underspecified", "expected_status": "needs_clarification", "reference_input": null, "answers": [], "status_rationale": "The prompt intentionally omits information needed to determine a unique numerical or retrieval answer, so guessing would invalidate the comparison.", "tool_required": false, "dossier_sha256": "a03ddc038c965766936d1dc55f870de5f48c0d18ef404d262f21320d08ee9f4a" }, { "task_id": "v2-eval/guidance-navigation-control/kalman_filter_state_estimation/08-invalid_or_out_of_domain", "split": "evaluation", "skill": "guidance-navigation-control", "operation": "kalman_filter_state_estimation", "case_kind": "invalid_or_out_of_domain", "expected_status": "out_of_domain", "reference_input": null, "answers": [], "status_rationale": "Every public operation explicitly prohibits certification, operational control, and safety-of-life use; this request crosses that documented boundary.", "tool_required": false, "dossier_sha256": "a03ddc038c965766936d1dc55f870de5f48c0d18ef404d262f21320d08ee9f4a" }, { "task_id": "v2-eval/guidance-navigation-control/lqr_controller_design/01-canonical_reference", "split": "evaluation", "skill": "guidance-navigation-control", "operation": "lqr_controller_design", "case_kind": "canonical_reference", "expected_status": "success", "reference_input": { "A_matrix": [ [ -1 ] ], "B_matrix": [ [ 1 ] ], "Q_matrix": [ [ 1 ] ], "R_matrix": [ [ 1 ] ] }, "answers": [ { "quantity": "scalar LQR gain", "value": 0.41421356, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 0.000002, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for scalar LQR gain; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.gain_matrix[0][0]", "runtime_source_path": "$.gain_matrix[0][0]", "source_citations": [ "scipy-care" ] }, { "quantity": "scalar Riccati solution", "value": 0.41421356, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 0.000002, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for scalar Riccati solution; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.riccati_solution[0][0]", "runtime_source_path": "$.riccati_solution[0][0]", "source_citations": [ "scipy-care" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "1e33ea3eae4b9320970df55d87823688121536202de7a1bfa1ad724048e2380e" }, { "task_id": "v2-eval/guidance-navigation-control/lqr_controller_design/02-realistic_in_domain", "split": "evaluation", "skill": "guidance-navigation-control", "operation": "lqr_controller_design", "case_kind": "realistic_in_domain", "expected_status": "success", "reference_input": { "A_matrix": [ [ -0.5 ] ], "B_matrix": [ [ 1 ] ], "Q_matrix": [ [ 1 ] ], "R_matrix": [ [ 1 ] ] }, "answers": [ { "quantity": "scalar LQR gain", "value": 0.61803399, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 0.000002, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for scalar LQR gain; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.gain_matrix[0][0]", "runtime_source_path": "$.gain_matrix[0][0]", "source_citations": [ "scipy-care" ] }, { "quantity": "scalar Riccati solution", "value": 0.61803399, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 0.000002, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for scalar Riccati solution; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.riccati_solution[0][0]", "runtime_source_path": "$.riccati_solution[0][0]", "source_citations": [ "scipy-care" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "1e33ea3eae4b9320970df55d87823688121536202de7a1bfa1ad724048e2380e" }, { "task_id": "v2-eval/guidance-navigation-control/lqr_controller_design/03-alternate_units", "split": "evaluation", "skill": "guidance-navigation-control", "operation": "lqr_controller_design", "case_kind": "alternate_units", "expected_status": "success", "reference_input": { "A_matrix": [ [ -0.8 ] ], "B_matrix": [ [ 1 ] ], "Q_matrix": [ [ 1 ] ], "R_matrix": [ [ 1 ] ] }, "answers": [ { "quantity": "scalar LQR gain", "value": 0.48062485, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 0.000002, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for scalar LQR gain; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.gain_matrix[0][0]", "runtime_source_path": "$.gain_matrix[0][0]", "source_citations": [ "scipy-care" ] }, { "quantity": "scalar Riccati solution", "value": 0.48062485, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 0.000002, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for scalar Riccati solution; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.riccati_solution[0][0]", "runtime_source_path": "$.riccati_solution[0][0]", "source_citations": [ "scipy-care" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "1e33ea3eae4b9320970df55d87823688121536202de7a1bfa1ad724048e2380e" }, { "task_id": "v2-eval/guidance-navigation-control/lqr_controller_design/04-validation_boundary", "split": "evaluation", "skill": "guidance-navigation-control", "operation": "lqr_controller_design", "case_kind": "validation_boundary", "expected_status": "success", "reference_input": { "A_matrix": [ [ -1.2 ] ], "B_matrix": [ [ 1 ] ], "Q_matrix": [ [ 1 ] ], "R_matrix": [ [ 1 ] ] }, "answers": [ { "quantity": "scalar LQR gain", "value": 0.36204994, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 0.000002, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for scalar LQR gain; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.gain_matrix[0][0]", "runtime_source_path": "$.gain_matrix[0][0]", "source_citations": [ "scipy-care" ] }, { "quantity": "scalar Riccati solution", "value": 0.36204994, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 0.000002, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for scalar Riccati solution; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.riccati_solution[0][0]", "runtime_source_path": "$.riccati_solution[0][0]", "source_citations": [ "scipy-care" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "1e33ea3eae4b9320970df55d87823688121536202de7a1bfa1ad724048e2380e" }, { "task_id": "v2-eval/guidance-navigation-control/lqr_controller_design/05-metamorphic_baseline", "split": "evaluation", "skill": "guidance-navigation-control", "operation": "lqr_controller_design", "case_kind": "metamorphic_baseline", "expected_status": "success", "reference_input": { "A_matrix": [ [ -1.5 ] ], "B_matrix": [ [ 1 ] ], "Q_matrix": [ [ 1 ] ], "R_matrix": [ [ 1 ] ] }, "answers": [ { "quantity": "scalar LQR gain", "value": 0.30277564, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 0.000002, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for scalar LQR gain; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.gain_matrix[0][0]", "runtime_source_path": "$.gain_matrix[0][0]", "source_citations": [ "scipy-care" ] }, { "quantity": "scalar Riccati solution", "value": 0.30277564, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 0.000002, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for scalar Riccati solution; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.riccati_solution[0][0]", "runtime_source_path": "$.riccati_solution[0][0]", "source_citations": [ "scipy-care" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "1e33ea3eae4b9320970df55d87823688121536202de7a1bfa1ad724048e2380e" }, { "task_id": "v2-eval/guidance-navigation-control/lqr_controller_design/06-metamorphic_perturbation", "split": "evaluation", "skill": "guidance-navigation-control", "operation": "lqr_controller_design", "case_kind": "metamorphic_perturbation", "expected_status": "success", "reference_input": { "A_matrix": [ [ -2 ] ], "B_matrix": [ [ 1 ] ], "Q_matrix": [ [ 1 ] ], "R_matrix": [ [ 1 ] ] }, "answers": [ { "quantity": "scalar LQR gain", "value": 0.23606798, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 0.000002, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for scalar LQR gain; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.gain_matrix[0][0]", "runtime_source_path": "$.gain_matrix[0][0]", "source_citations": [ "scipy-care" ] }, { "quantity": "scalar Riccati solution", "value": 0.23606798, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 0.000002, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for scalar Riccati solution; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.riccati_solution[0][0]", "runtime_source_path": "$.riccati_solution[0][0]", "source_citations": [ "scipy-care" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "1e33ea3eae4b9320970df55d87823688121536202de7a1bfa1ad724048e2380e" }, { "task_id": "v2-eval/guidance-navigation-control/lqr_controller_design/07-underspecified", "split": "evaluation", "skill": "guidance-navigation-control", "operation": "lqr_controller_design", "case_kind": "underspecified", "expected_status": "needs_clarification", "reference_input": null, "answers": [], "status_rationale": "The prompt intentionally omits information needed to determine a unique numerical or retrieval answer, so guessing would invalidate the comparison.", "tool_required": false, "dossier_sha256": "1e33ea3eae4b9320970df55d87823688121536202de7a1bfa1ad724048e2380e" }, { "task_id": "v2-eval/guidance-navigation-control/lqr_controller_design/08-invalid_or_out_of_domain", "split": "evaluation", "skill": "guidance-navigation-control", "operation": "lqr_controller_design", "case_kind": "invalid_or_out_of_domain", "expected_status": "out_of_domain", "reference_input": null, "answers": [], "status_rationale": "Every public operation explicitly prohibits certification, operational control, and safety-of-life use; this request crosses that documented boundary.", "tool_required": false, "dossier_sha256": "1e33ea3eae4b9320970df55d87823688121536202de7a1bfa1ad724048e2380e" }, { "task_id": "v2-eval/aircraft-performance/density_altitude_calculator/01-canonical_reference", "split": "evaluation", "skill": "aircraft-performance", "operation": "density_altitude_calculator", "case_kind": "canonical_reference", "expected_status": "success", "reference_input": { "pressure_altitude_ft": 5000, "temperature_c": 30, "dewpoint_c": 10 }, "answers": [ { "quantity": "density altitude", "value": 7988.7, "unit": "ft", "accepted_units": [ "ft" ], "grader": "scalar", "tolerance": { "absolute": 0.2, "relative": 0.00001, "rationale": "Operation-specific acceptance bound from the canonical dossier for density altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.density_altitude_ft", "runtime_source_path": "$.density_altitude_ft", "source_citations": [ "us-standard-atmosphere-1976", "faa-handbooks" ] }, { "quantity": "air density", "value": 0.963543, "unit": "kg/m^3", "accepted_units": [ "kg/m^3" ], "grader": "scalar", "tolerance": { "absolute": 0.000003, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for air density; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.air_density_kg_m3", "runtime_source_path": "$.air_density_kg_m3", "source_citations": [ "us-standard-atmosphere-1976", "faa-handbooks" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "fab534494415926b1661bc202edaaa9a0d57262d66d86a40c58d0d51414c3ae8" }, { "task_id": "v2-eval/aircraft-performance/density_altitude_calculator/02-realistic_in_domain", "split": "evaluation", "skill": "aircraft-performance", "operation": "density_altitude_calculator", "case_kind": "realistic_in_domain", "expected_status": "success", "reference_input": { "pressure_altitude_ft": 5000, "temperature_c": 0, "dewpoint_c": 10 }, "answers": [ { "quantity": "density altitude", "value": 4388.7, "unit": "ft", "accepted_units": [ "ft" ], "grader": "scalar", "tolerance": { "absolute": 0.2, "relative": 0.00001, "rationale": "Operation-specific acceptance bound from the canonical dossier for density altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.density_altitude_ft", "runtime_source_path": "$.density_altitude_ft", "source_citations": [ "us-standard-atmosphere-1976", "faa-handbooks" ] }, { "quantity": "air density", "value": 1.069369, "unit": "kg/m^3", "accepted_units": [ "kg/m^3" ], "grader": "scalar", "tolerance": { "absolute": 0.000003, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for air density; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.air_density_kg_m3", "runtime_source_path": "$.air_density_kg_m3", "source_citations": [ "us-standard-atmosphere-1976", "faa-handbooks" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "fab534494415926b1661bc202edaaa9a0d57262d66d86a40c58d0d51414c3ae8" }, { "task_id": "v2-eval/aircraft-performance/density_altitude_calculator/03-alternate_units", "split": "evaluation", "skill": "aircraft-performance", "operation": "density_altitude_calculator", "case_kind": "alternate_units", "expected_status": "success", "reference_input": { "pressure_altitude_ft": 5000, "temperature_c": 15, "dewpoint_c": 10 }, "answers": [ { "quantity": "density altitude", "value": 6188.7, "unit": "ft", "accepted_units": [ "ft" ], "grader": "scalar", "tolerance": { "absolute": 0.2, "relative": 0.00001, "rationale": "Operation-specific acceptance bound from the canonical dossier for density altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.density_altitude_ft", "runtime_source_path": "$.density_altitude_ft", "source_citations": [ "us-standard-atmosphere-1976", "faa-handbooks" ] }, { "quantity": "air density", "value": 1.013702, "unit": "kg/m^3", "accepted_units": [ "kg/m^3" ], "grader": "scalar", "tolerance": { "absolute": 0.000003, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for air density; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.air_density_kg_m3", "runtime_source_path": "$.air_density_kg_m3", "source_citations": [ "us-standard-atmosphere-1976", "faa-handbooks" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "fab534494415926b1661bc202edaaa9a0d57262d66d86a40c58d0d51414c3ae8" }, { "task_id": "v2-eval/aircraft-performance/density_altitude_calculator/04-validation_boundary", "split": "evaluation", "skill": "aircraft-performance", "operation": "density_altitude_calculator", "case_kind": "validation_boundary", "expected_status": "success", "reference_input": { "pressure_altitude_ft": 5000, "temperature_c": 25, "dewpoint_c": 10 }, "answers": [ { "quantity": "density altitude", "value": 7388.7, "unit": "ft", "accepted_units": [ "ft" ], "grader": "scalar", "tolerance": { "absolute": 0.2, "relative": 0.00001, "rationale": "Operation-specific acceptance bound from the canonical dossier for density altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.density_altitude_ft", "runtime_source_path": "$.density_altitude_ft", "source_citations": [ "us-standard-atmosphere-1976", "faa-handbooks" ] }, { "quantity": "air density", "value": 0.979702, "unit": "kg/m^3", "accepted_units": [ "kg/m^3" ], "grader": "scalar", "tolerance": { "absolute": 0.000003, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for air density; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.air_density_kg_m3", "runtime_source_path": "$.air_density_kg_m3", "source_citations": [ "us-standard-atmosphere-1976", "faa-handbooks" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "fab534494415926b1661bc202edaaa9a0d57262d66d86a40c58d0d51414c3ae8" }, { "task_id": "v2-eval/aircraft-performance/density_altitude_calculator/05-metamorphic_baseline", "split": "evaluation", "skill": "aircraft-performance", "operation": "density_altitude_calculator", "case_kind": "metamorphic_baseline", "expected_status": "success", "reference_input": { "pressure_altitude_ft": 5000, "temperature_c": 40, "dewpoint_c": 10 }, "answers": [ { "quantity": "density altitude", "value": 9188.7, "unit": "ft", "accepted_units": [ "ft" ], "grader": "scalar", "tolerance": { "absolute": 0.2, "relative": 0.00001, "rationale": "Operation-specific acceptance bound from the canonical dossier for density altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.density_altitude_ft", "runtime_source_path": "$.density_altitude_ft", "source_citations": [ "us-standard-atmosphere-1976", "faa-handbooks" ] }, { "quantity": "air density", "value": 0.932774, "unit": "kg/m^3", "accepted_units": [ "kg/m^3" ], "grader": "scalar", "tolerance": { "absolute": 0.000003, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for air density; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.air_density_kg_m3", "runtime_source_path": "$.air_density_kg_m3", "source_citations": [ "us-standard-atmosphere-1976", "faa-handbooks" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "fab534494415926b1661bc202edaaa9a0d57262d66d86a40c58d0d51414c3ae8" }, { "task_id": "v2-eval/aircraft-performance/density_altitude_calculator/06-metamorphic_perturbation", "split": "evaluation", "skill": "aircraft-performance", "operation": "density_altitude_calculator", "case_kind": "metamorphic_perturbation", "expected_status": "success", "reference_input": { "pressure_altitude_ft": 5000, "temperature_c": -10, "dewpoint_c": 10 }, "answers": [ { "quantity": "density altitude", "value": 3188.7, "unit": "ft", "accepted_units": [ "ft" ], "grader": "scalar", "tolerance": { "absolute": 0.2, "relative": 0.00001, "rationale": "Operation-specific acceptance bound from the canonical dossier for density altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.density_altitude_ft", "runtime_source_path": "$.density_altitude_ft", "source_citations": [ "us-standard-atmosphere-1976", "faa-handbooks" ] }, { "quantity": "air density", "value": 1.110006, "unit": "kg/m^3", "accepted_units": [ "kg/m^3" ], "grader": "scalar", "tolerance": { "absolute": 0.000003, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for air density; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.air_density_kg_m3", "runtime_source_path": "$.air_density_kg_m3", "source_citations": [ "us-standard-atmosphere-1976", "faa-handbooks" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "fab534494415926b1661bc202edaaa9a0d57262d66d86a40c58d0d51414c3ae8" }, { "task_id": "v2-eval/aircraft-performance/density_altitude_calculator/07-underspecified", "split": "evaluation", "skill": "aircraft-performance", "operation": "density_altitude_calculator", "case_kind": "underspecified", "expected_status": "needs_clarification", "reference_input": null, "answers": [], "status_rationale": "The prompt intentionally omits information needed to determine a unique numerical or retrieval answer, so guessing would invalidate the comparison.", "tool_required": false, "dossier_sha256": "fab534494415926b1661bc202edaaa9a0d57262d66d86a40c58d0d51414c3ae8" }, { "task_id": "v2-eval/aircraft-performance/density_altitude_calculator/08-invalid_or_out_of_domain", "split": "evaluation", "skill": "aircraft-performance", "operation": "density_altitude_calculator", "case_kind": "invalid_or_out_of_domain", "expected_status": "out_of_domain", "reference_input": null, "answers": [], "status_rationale": "Every public operation explicitly prohibits certification, operational control, and safety-of-life use; this request crosses that documented boundary.", "tool_required": false, "dossier_sha256": "fab534494415926b1661bc202edaaa9a0d57262d66d86a40c58d0d51414c3ae8" }, { "task_id": "v2-eval/aircraft-performance/true_airspeed_converter/01-canonical_reference", "split": "evaluation", "skill": "aircraft-performance", "operation": "true_airspeed_converter", "case_kind": "canonical_reference", "expected_status": "success", "reference_input": { "speed_value": 120, "speed_type": "IAS", "altitude_ft": 5000 }, "answers": [ { "quantity": "true airspeed", "value": 129.271, "unit": "kt", "accepted_units": [ "kt" ], "grader": "scalar", "tolerance": { "absolute": 0.01, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for true airspeed; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.tas_kts", "runtime_source_path": "$.tas_kts", "source_citations": [ "us-standard-atmosphere-1976", "faa-handbooks" ] }, { "quantity": "Mach number", "value": 0.19888, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 0.00002, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for Mach number; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.mach", "runtime_source_path": "$.mach", "source_citations": [ "us-standard-atmosphere-1976", "faa-handbooks" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "dc0d65c2fc5ca43982229ad43dab71b44f69866a55052210705873e3638c3459" }, { "task_id": "v2-eval/aircraft-performance/true_airspeed_converter/02-realistic_in_domain", "split": "evaluation", "skill": "aircraft-performance", "operation": "true_airspeed_converter", "case_kind": "realistic_in_domain", "expected_status": "success", "reference_input": { "speed_value": 80, "speed_type": "IAS", "altitude_ft": 5000 }, "answers": [ { "quantity": "true airspeed", "value": 86.181, "unit": "kt", "accepted_units": [ "kt" ], "grader": "scalar", "tolerance": { "absolute": 0.01, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for true airspeed; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.tas_kts", "runtime_source_path": "$.tas_kts", "source_citations": [ "us-standard-atmosphere-1976", "faa-handbooks" ] }, { "quantity": "Mach number", "value": 0.13258, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 0.00002, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for Mach number; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.mach", "runtime_source_path": "$.mach", "source_citations": [ "us-standard-atmosphere-1976", "faa-handbooks" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "dc0d65c2fc5ca43982229ad43dab71b44f69866a55052210705873e3638c3459" }, { "task_id": "v2-eval/aircraft-performance/true_airspeed_converter/03-alternate_units", "split": "evaluation", "skill": "aircraft-performance", "operation": "true_airspeed_converter", "case_kind": "alternate_units", "expected_status": "success", "reference_input": { "speed_value": 100, "speed_type": "IAS", "altitude_ft": 5000 }, "answers": [ { "quantity": "true airspeed", "value": 107.726, "unit": "kt", "accepted_units": [ "kt" ], "grader": "scalar", "tolerance": { "absolute": 0.01, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for true airspeed; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.tas_kts", "runtime_source_path": "$.tas_kts", "source_citations": [ "us-standard-atmosphere-1976", "faa-handbooks" ] }, { "quantity": "Mach number", "value": 0.16573, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 0.00002, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for Mach number; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.mach", "runtime_source_path": "$.mach", "source_citations": [ "us-standard-atmosphere-1976", "faa-handbooks" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "dc0d65c2fc5ca43982229ad43dab71b44f69866a55052210705873e3638c3459" }, { "task_id": "v2-eval/aircraft-performance/true_airspeed_converter/04-validation_boundary", "split": "evaluation", "skill": "aircraft-performance", "operation": "true_airspeed_converter", "case_kind": "validation_boundary", "expected_status": "success", "reference_input": { "speed_value": 140, "speed_type": "IAS", "altitude_ft": 5000 }, "answers": [ { "quantity": "true airspeed", "value": 150.817, "unit": "kt", "accepted_units": [ "kt" ], "grader": "scalar", "tolerance": { "absolute": 0.01, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for true airspeed; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.tas_kts", "runtime_source_path": "$.tas_kts", "source_citations": [ "us-standard-atmosphere-1976", "faa-handbooks" ] }, { "quantity": "Mach number", "value": 0.23202, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 0.00002, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for Mach number; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.mach", "runtime_source_path": "$.mach", "source_citations": [ "us-standard-atmosphere-1976", "faa-handbooks" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "dc0d65c2fc5ca43982229ad43dab71b44f69866a55052210705873e3638c3459" }, { "task_id": "v2-eval/aircraft-performance/true_airspeed_converter/05-metamorphic_baseline", "split": "evaluation", "skill": "aircraft-performance", "operation": "true_airspeed_converter", "case_kind": "metamorphic_baseline", "expected_status": "success", "reference_input": { "speed_value": 180, "speed_type": "IAS", "altitude_ft": 5000 }, "answers": [ { "quantity": "true airspeed", "value": 193.907, "unit": "kt", "accepted_units": [ "kt" ], "grader": "scalar", "tolerance": { "absolute": 0.01, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for true airspeed; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.tas_kts", "runtime_source_path": "$.tas_kts", "source_citations": [ "us-standard-atmosphere-1976", "faa-handbooks" ] }, { "quantity": "Mach number", "value": 0.29831, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 0.00002, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for Mach number; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.mach", "runtime_source_path": "$.mach", "source_citations": [ "us-standard-atmosphere-1976", "faa-handbooks" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "dc0d65c2fc5ca43982229ad43dab71b44f69866a55052210705873e3638c3459" }, { "task_id": "v2-eval/aircraft-performance/true_airspeed_converter/06-metamorphic_perturbation", "split": "evaluation", "skill": "aircraft-performance", "operation": "true_airspeed_converter", "case_kind": "metamorphic_perturbation", "expected_status": "success", "reference_input": { "speed_value": 220, "speed_type": "IAS", "altitude_ft": 5000 }, "answers": [ { "quantity": "true airspeed", "value": 236.998, "unit": "kt", "accepted_units": [ "kt" ], "grader": "scalar", "tolerance": { "absolute": 0.01, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for true airspeed; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.tas_kts", "runtime_source_path": "$.tas_kts", "source_citations": [ "us-standard-atmosphere-1976", "faa-handbooks" ] }, { "quantity": "Mach number", "value": 0.36461, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 0.00002, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for Mach number; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.mach", "runtime_source_path": "$.mach", "source_citations": [ "us-standard-atmosphere-1976", "faa-handbooks" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "dc0d65c2fc5ca43982229ad43dab71b44f69866a55052210705873e3638c3459" }, { "task_id": "v2-eval/aircraft-performance/true_airspeed_converter/07-underspecified", "split": "evaluation", "skill": "aircraft-performance", "operation": "true_airspeed_converter", "case_kind": "underspecified", "expected_status": "needs_clarification", "reference_input": null, "answers": [], "status_rationale": "The prompt intentionally omits information needed to determine a unique numerical or retrieval answer, so guessing would invalidate the comparison.", "tool_required": false, "dossier_sha256": "dc0d65c2fc5ca43982229ad43dab71b44f69866a55052210705873e3638c3459" }, { "task_id": "v2-eval/aircraft-performance/true_airspeed_converter/08-invalid_or_out_of_domain", "split": "evaluation", "skill": "aircraft-performance", "operation": "true_airspeed_converter", "case_kind": "invalid_or_out_of_domain", "expected_status": "out_of_domain", "reference_input": null, "answers": [], "status_rationale": "Every public operation explicitly prohibits certification, operational control, and safety-of-life use; this request crosses that documented boundary.", "tool_required": false, "dossier_sha256": "dc0d65c2fc5ca43982229ad43dab71b44f69866a55052210705873e3638c3459" }, { "task_id": "v2-eval/aircraft-performance/stall_speed_calculator/01-canonical_reference", "split": "evaluation", "skill": "aircraft-performance", "operation": "stall_speed_calculator", "case_kind": "canonical_reference", "expected_status": "success", "reference_input": { "weight_kg": 1100, "wing_area_m2": 16.2, "cl_max_clean": 1.5, "altitude_ft": 0, "load_factor": 1 }, "answers": [ { "quantity": "clean stall speed", "value": 26.922, "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "scalar", "tolerance": { "absolute": 0.002, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for clean stall speed; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.clean_ms", "runtime_source_path": "$.clean_ms", "source_citations": [ "us-standard-atmosphere-1976", "faa-handbooks" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "b803ef8305c6da1593ae835e4c18b7aeb8b66de4d6f59e11c0b621676095d988" }, { "task_id": "v2-eval/aircraft-performance/stall_speed_calculator/02-realistic_in_domain", "split": "evaluation", "skill": "aircraft-performance", "operation": "stall_speed_calculator", "case_kind": "realistic_in_domain", "expected_status": "success", "reference_input": { "weight_kg": 800, "wing_area_m2": 16.2, "cl_max_clean": 1.5, "altitude_ft": 0, "load_factor": 1 }, "answers": [ { "quantity": "clean stall speed", "value": 22.959, "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "scalar", "tolerance": { "absolute": 0.002, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for clean stall speed; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.clean_ms", "runtime_source_path": "$.clean_ms", "source_citations": [ "us-standard-atmosphere-1976", "faa-handbooks" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "b803ef8305c6da1593ae835e4c18b7aeb8b66de4d6f59e11c0b621676095d988" }, { "task_id": "v2-eval/aircraft-performance/stall_speed_calculator/03-alternate_units", "split": "evaluation", "skill": "aircraft-performance", "operation": "stall_speed_calculator", "case_kind": "alternate_units", "expected_status": "success", "reference_input": { "weight_kg": 950, "wing_area_m2": 16.2, "cl_max_clean": 1.5, "altitude_ft": 0, "load_factor": 1 }, "answers": [ { "quantity": "clean stall speed", "value": 25.019, "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "scalar", "tolerance": { "absolute": 0.002, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for clean stall speed; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.clean_ms", "runtime_source_path": "$.clean_ms", "source_citations": [ "us-standard-atmosphere-1976", "faa-handbooks" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "b803ef8305c6da1593ae835e4c18b7aeb8b66de4d6f59e11c0b621676095d988" }, { "task_id": "v2-eval/aircraft-performance/stall_speed_calculator/04-validation_boundary", "split": "evaluation", "skill": "aircraft-performance", "operation": "stall_speed_calculator", "case_kind": "validation_boundary", "expected_status": "success", "reference_input": { "weight_kg": 1250, "wing_area_m2": 16.2, "cl_max_clean": 1.5, "altitude_ft": 0, "load_factor": 1 }, "answers": [ { "quantity": "clean stall speed", "value": 28.698, "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "scalar", "tolerance": { "absolute": 0.002, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for clean stall speed; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.clean_ms", "runtime_source_path": "$.clean_ms", "source_citations": [ "us-standard-atmosphere-1976", "faa-handbooks" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "b803ef8305c6da1593ae835e4c18b7aeb8b66de4d6f59e11c0b621676095d988" }, { "task_id": "v2-eval/aircraft-performance/stall_speed_calculator/05-metamorphic_baseline", "split": "evaluation", "skill": "aircraft-performance", "operation": "stall_speed_calculator", "case_kind": "metamorphic_baseline", "expected_status": "success", "reference_input": { "weight_kg": 1500, "wing_area_m2": 16.2, "cl_max_clean": 1.5, "altitude_ft": 0, "load_factor": 1 }, "answers": [ { "quantity": "clean stall speed", "value": 31.438, "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "scalar", "tolerance": { "absolute": 0.002, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for clean stall speed; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.clean_ms", "runtime_source_path": "$.clean_ms", "source_citations": [ "us-standard-atmosphere-1976", "faa-handbooks" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "b803ef8305c6da1593ae835e4c18b7aeb8b66de4d6f59e11c0b621676095d988" }, { "task_id": "v2-eval/aircraft-performance/stall_speed_calculator/06-metamorphic_perturbation", "split": "evaluation", "skill": "aircraft-performance", "operation": "stall_speed_calculator", "case_kind": "metamorphic_perturbation", "expected_status": "success", "reference_input": { "weight_kg": 1800, "wing_area_m2": 16.2, "cl_max_clean": 1.5, "altitude_ft": 0, "load_factor": 1 }, "answers": [ { "quantity": "clean stall speed", "value": 34.438, "unit": "m/s", "accepted_units": [ "m/s" ], "grader": "scalar", "tolerance": { "absolute": 0.002, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for clean stall speed; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.clean_ms", "runtime_source_path": "$.clean_ms", "source_citations": [ "us-standard-atmosphere-1976", "faa-handbooks" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "b803ef8305c6da1593ae835e4c18b7aeb8b66de4d6f59e11c0b621676095d988" }, { "task_id": "v2-eval/aircraft-performance/stall_speed_calculator/07-underspecified", "split": "evaluation", "skill": "aircraft-performance", "operation": "stall_speed_calculator", "case_kind": "underspecified", "expected_status": "needs_clarification", "reference_input": null, "answers": [], "status_rationale": "The prompt intentionally omits information needed to determine a unique numerical or retrieval answer, so guessing would invalidate the comparison.", "tool_required": false, "dossier_sha256": "b803ef8305c6da1593ae835e4c18b7aeb8b66de4d6f59e11c0b621676095d988" }, { "task_id": "v2-eval/aircraft-performance/stall_speed_calculator/08-invalid_or_out_of_domain", "split": "evaluation", "skill": "aircraft-performance", "operation": "stall_speed_calculator", "case_kind": "invalid_or_out_of_domain", "expected_status": "out_of_domain", "reference_input": null, "answers": [], "status_rationale": "Every public operation explicitly prohibits certification, operational control, and safety-of-life use; this request crosses that documented boundary.", "tool_required": false, "dossier_sha256": "b803ef8305c6da1593ae835e4c18b7aeb8b66de4d6f59e11c0b621676095d988" }, { "task_id": "v2-eval/aircraft-performance/weight_and_balance/01-canonical_reference", "split": "evaluation", "skill": "aircraft-performance", "operation": "weight_and_balance", "case_kind": "canonical_reference", "expected_status": "success", "reference_input": { "basic_empty_weight_kg": 700, "basic_empty_arm_m": 1.1, "fuel_kg": 100, "fuel_arm_m": 1.3, "payload_items": [ { "weight_kg": 160, "arm_m": 1.4 } ] }, "answers": [ { "quantity": "total loaded mass", "value": 960, "unit": "kg", "accepted_units": [ "kg" ], "grader": "scalar", "tolerance": { "absolute": 0.000001, "relative": 1e-9, "rationale": "Operation-specific acceptance bound from the canonical dossier for total loaded mass; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.total_weight_kg", "runtime_source_path": "$.total_weight_kg", "source_citations": [ "faa-handbooks" ] }, { "quantity": "longitudinal center of gravity", "value": 1.17083, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 0.000002, "relative": 1e-8, "rationale": "Operation-specific acceptance bound from the canonical dossier for longitudinal center of gravity; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.center_of_gravity_m", "runtime_source_path": "$.center_of_gravity_m", "source_citations": [ "faa-handbooks" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "da58d6f04f58214368549068a711417c7575d27fbc616873bb0018fff73b3066" }, { "task_id": "v2-eval/aircraft-performance/weight_and_balance/02-realistic_in_domain", "split": "evaluation", "skill": "aircraft-performance", "operation": "weight_and_balance", "case_kind": "realistic_in_domain", "expected_status": "success", "reference_input": { "basic_empty_weight_kg": 700, "basic_empty_arm_m": 1.1, "fuel_kg": 20, "fuel_arm_m": 1.3, "payload_items": [ { "weight_kg": 160, "arm_m": 1.4 } ] }, "answers": [ { "quantity": "total loaded mass", "value": 880, "unit": "kg", "accepted_units": [ "kg" ], "grader": "scalar", "tolerance": { "absolute": 0.000001, "relative": 1e-9, "rationale": "Operation-specific acceptance bound from the canonical dossier for total loaded mass; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.total_weight_kg", "runtime_source_path": "$.total_weight_kg", "source_citations": [ "faa-handbooks" ] }, { "quantity": "longitudinal center of gravity", "value": 1.15909, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 0.000002, "relative": 1e-8, "rationale": "Operation-specific acceptance bound from the canonical dossier for longitudinal center of gravity; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.center_of_gravity_m", "runtime_source_path": "$.center_of_gravity_m", "source_citations": [ "faa-handbooks" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "da58d6f04f58214368549068a711417c7575d27fbc616873bb0018fff73b3066" }, { "task_id": "v2-eval/aircraft-performance/weight_and_balance/03-alternate_units", "split": "evaluation", "skill": "aircraft-performance", "operation": "weight_and_balance", "case_kind": "alternate_units", "expected_status": "success", "reference_input": { "basic_empty_weight_kg": 700, "basic_empty_arm_m": 1.1, "fuel_kg": 50, "fuel_arm_m": 1.3, "payload_items": [ { "weight_kg": 160, "arm_m": 1.4 } ] }, "answers": [ { "quantity": "total loaded mass", "value": 910, "unit": "kg", "accepted_units": [ "kg" ], "grader": "scalar", "tolerance": { "absolute": 0.000001, "relative": 1e-9, "rationale": "Operation-specific acceptance bound from the canonical dossier for total loaded mass; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.total_weight_kg", "runtime_source_path": "$.total_weight_kg", "source_citations": [ "faa-handbooks" ] }, { "quantity": "longitudinal center of gravity", "value": 1.16374, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 0.000002, "relative": 1e-8, "rationale": "Operation-specific acceptance bound from the canonical dossier for longitudinal center of gravity; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.center_of_gravity_m", "runtime_source_path": "$.center_of_gravity_m", "source_citations": [ "faa-handbooks" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "da58d6f04f58214368549068a711417c7575d27fbc616873bb0018fff73b3066" }, { "task_id": "v2-eval/aircraft-performance/weight_and_balance/04-validation_boundary", "split": "evaluation", "skill": "aircraft-performance", "operation": "weight_and_balance", "case_kind": "validation_boundary", "expected_status": "success", "reference_input": { "basic_empty_weight_kg": 700, "basic_empty_arm_m": 1.1, "fuel_kg": 150, "fuel_arm_m": 1.3, "payload_items": [ { "weight_kg": 160, "arm_m": 1.4 } ] }, "answers": [ { "quantity": "total loaded mass", "value": 1010, "unit": "kg", "accepted_units": [ "kg" ], "grader": "scalar", "tolerance": { "absolute": 0.000001, "relative": 1e-9, "rationale": "Operation-specific acceptance bound from the canonical dossier for total loaded mass; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.total_weight_kg", "runtime_source_path": "$.total_weight_kg", "source_citations": [ "faa-handbooks" ] }, { "quantity": "longitudinal center of gravity", "value": 1.17723, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 0.000002, "relative": 1e-8, "rationale": "Operation-specific acceptance bound from the canonical dossier for longitudinal center of gravity; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.center_of_gravity_m", "runtime_source_path": "$.center_of_gravity_m", "source_citations": [ "faa-handbooks" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "da58d6f04f58214368549068a711417c7575d27fbc616873bb0018fff73b3066" }, { "task_id": "v2-eval/aircraft-performance/weight_and_balance/05-metamorphic_baseline", "split": "evaluation", "skill": "aircraft-performance", "operation": "weight_and_balance", "case_kind": "metamorphic_baseline", "expected_status": "success", "reference_input": { "basic_empty_weight_kg": 700, "basic_empty_arm_m": 1.1, "fuel_kg": 200, "fuel_arm_m": 1.3, "payload_items": [ { "weight_kg": 160, "arm_m": 1.4 } ] }, "answers": [ { "quantity": "total loaded mass", "value": 1060, "unit": "kg", "accepted_units": [ "kg" ], "grader": "scalar", "tolerance": { "absolute": 0.000001, "relative": 1e-9, "rationale": "Operation-specific acceptance bound from the canonical dossier for total loaded mass; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.total_weight_kg", "runtime_source_path": "$.total_weight_kg", "source_citations": [ "faa-handbooks" ] }, { "quantity": "longitudinal center of gravity", "value": 1.18302, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 0.000002, "relative": 1e-8, "rationale": "Operation-specific acceptance bound from the canonical dossier for longitudinal center of gravity; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.center_of_gravity_m", "runtime_source_path": "$.center_of_gravity_m", "source_citations": [ "faa-handbooks" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "da58d6f04f58214368549068a711417c7575d27fbc616873bb0018fff73b3066" }, { "task_id": "v2-eval/aircraft-performance/weight_and_balance/06-metamorphic_perturbation", "split": "evaluation", "skill": "aircraft-performance", "operation": "weight_and_balance", "case_kind": "metamorphic_perturbation", "expected_status": "success", "reference_input": { "basic_empty_weight_kg": 700, "basic_empty_arm_m": 1.1, "fuel_kg": 250, "fuel_arm_m": 1.3, "payload_items": [ { "weight_kg": 160, "arm_m": 1.4 } ] }, "answers": [ { "quantity": "total loaded mass", "value": 1110, "unit": "kg", "accepted_units": [ "kg" ], "grader": "scalar", "tolerance": { "absolute": 0.000001, "relative": 1e-9, "rationale": "Operation-specific acceptance bound from the canonical dossier for total loaded mass; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.total_weight_kg", "runtime_source_path": "$.total_weight_kg", "source_citations": [ "faa-handbooks" ] }, { "quantity": "longitudinal center of gravity", "value": 1.18829, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 0.000002, "relative": 1e-8, "rationale": "Operation-specific acceptance bound from the canonical dossier for longitudinal center of gravity; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.center_of_gravity_m", "runtime_source_path": "$.center_of_gravity_m", "source_citations": [ "faa-handbooks" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "da58d6f04f58214368549068a711417c7575d27fbc616873bb0018fff73b3066" }, { "task_id": "v2-eval/aircraft-performance/weight_and_balance/07-underspecified", "split": "evaluation", "skill": "aircraft-performance", "operation": "weight_and_balance", "case_kind": "underspecified", "expected_status": "needs_clarification", "reference_input": null, "answers": [], "status_rationale": "The prompt intentionally omits information needed to determine a unique numerical or retrieval answer, so guessing would invalidate the comparison.", "tool_required": false, "dossier_sha256": "da58d6f04f58214368549068a711417c7575d27fbc616873bb0018fff73b3066" }, { "task_id": "v2-eval/aircraft-performance/weight_and_balance/08-invalid_or_out_of_domain", "split": "evaluation", "skill": "aircraft-performance", "operation": "weight_and_balance", "case_kind": "invalid_or_out_of_domain", "expected_status": "out_of_domain", "reference_input": null, "answers": [], "status_rationale": "Every public operation explicitly prohibits certification, operational control, and safety-of-life use; this request crosses that documented boundary.", "tool_required": false, "dossier_sha256": "da58d6f04f58214368549068a711417c7575d27fbc616873bb0018fff73b3066" }, { "task_id": "v2-eval/aircraft-performance/takeoff_performance/01-canonical_reference", "split": "evaluation", "skill": "aircraft-performance", "operation": "takeoff_performance", "case_kind": "canonical_reference", "expected_status": "success", "reference_input": { "weight_kg": 1100, "pressure_altitude_ft": 2000, "temperature_c": 25, "wind_kts": 0, "runway_slope_pct": 0, "thrust_to_weight": 0.3, "cl_max_takeoff": 1.8, "wing_area_m2": 100, "runway_condition": "dry" }, "answers": [ { "quantity": "modeled takeoff ground roll", "value": 30.7, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 0.2, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for modeled takeoff ground roll; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.ground_roll_m", "runtime_source_path": "$.ground_roll_m", "source_citations": [ "faa-handbooks" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "5b9b05a5dc5ca8b22b0bc72612c23cdfed88029bb837fcde89f19e3ff13b345e" }, { "task_id": "v2-eval/aircraft-performance/takeoff_performance/02-realistic_in_domain", "split": "evaluation", "skill": "aircraft-performance", "operation": "takeoff_performance", "case_kind": "realistic_in_domain", "expected_status": "success", "reference_input": { "weight_kg": 800, "pressure_altitude_ft": 2000, "temperature_c": 25, "wind_kts": 0, "runway_slope_pct": 0, "thrust_to_weight": 0.3, "cl_max_takeoff": 1.8, "wing_area_m2": 100, "runway_condition": "dry" }, "answers": [ { "quantity": "modeled takeoff ground roll", "value": 22.4, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 0.2, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for modeled takeoff ground roll; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.ground_roll_m", "runtime_source_path": "$.ground_roll_m", "source_citations": [ "faa-handbooks" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "5b9b05a5dc5ca8b22b0bc72612c23cdfed88029bb837fcde89f19e3ff13b345e" }, { "task_id": "v2-eval/aircraft-performance/takeoff_performance/03-alternate_units", "split": "evaluation", "skill": "aircraft-performance", "operation": "takeoff_performance", "case_kind": "alternate_units", "expected_status": "success", "reference_input": { "weight_kg": 950, "pressure_altitude_ft": 2000, "temperature_c": 25, "wind_kts": 0, "runway_slope_pct": 0, "thrust_to_weight": 0.3, "cl_max_takeoff": 1.8, "wing_area_m2": 100, "runway_condition": "dry" }, "answers": [ { "quantity": "modeled takeoff ground roll", "value": 26.6, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 0.2, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for modeled takeoff ground roll; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.ground_roll_m", "runtime_source_path": "$.ground_roll_m", "source_citations": [ "faa-handbooks" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "5b9b05a5dc5ca8b22b0bc72612c23cdfed88029bb837fcde89f19e3ff13b345e" }, { "task_id": "v2-eval/aircraft-performance/takeoff_performance/04-validation_boundary", "split": "evaluation", "skill": "aircraft-performance", "operation": "takeoff_performance", "case_kind": "validation_boundary", "expected_status": "success", "reference_input": { "weight_kg": 1250, "pressure_altitude_ft": 2000, "temperature_c": 25, "wind_kts": 0, "runway_slope_pct": 0, "thrust_to_weight": 0.3, "cl_max_takeoff": 1.8, "wing_area_m2": 100, "runway_condition": "dry" }, "answers": [ { "quantity": "modeled takeoff ground roll", "value": 34.9, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 0.2, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for modeled takeoff ground roll; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.ground_roll_m", "runtime_source_path": "$.ground_roll_m", "source_citations": [ "faa-handbooks" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "5b9b05a5dc5ca8b22b0bc72612c23cdfed88029bb837fcde89f19e3ff13b345e" }, { "task_id": "v2-eval/aircraft-performance/takeoff_performance/05-metamorphic_baseline", "split": "evaluation", "skill": "aircraft-performance", "operation": "takeoff_performance", "case_kind": "metamorphic_baseline", "expected_status": "success", "reference_input": { "weight_kg": 1500, "pressure_altitude_ft": 2000, "temperature_c": 25, "wind_kts": 0, "runway_slope_pct": 0, "thrust_to_weight": 0.3, "cl_max_takeoff": 1.8, "wing_area_m2": 100, "runway_condition": "dry" }, "answers": [ { "quantity": "modeled takeoff ground roll", "value": 41.9, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 0.2, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for modeled takeoff ground roll; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.ground_roll_m", "runtime_source_path": "$.ground_roll_m", "source_citations": [ "faa-handbooks" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "5b9b05a5dc5ca8b22b0bc72612c23cdfed88029bb837fcde89f19e3ff13b345e" }, { "task_id": "v2-eval/aircraft-performance/takeoff_performance/06-metamorphic_perturbation", "split": "evaluation", "skill": "aircraft-performance", "operation": "takeoff_performance", "case_kind": "metamorphic_perturbation", "expected_status": "success", "reference_input": { "weight_kg": 1800, "pressure_altitude_ft": 2000, "temperature_c": 25, "wind_kts": 0, "runway_slope_pct": 0, "thrust_to_weight": 0.3, "cl_max_takeoff": 1.8, "wing_area_m2": 100, "runway_condition": "dry" }, "answers": [ { "quantity": "modeled takeoff ground roll", "value": 50.3, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 0.2, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for modeled takeoff ground roll; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.ground_roll_m", "runtime_source_path": "$.ground_roll_m", "source_citations": [ "faa-handbooks" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "5b9b05a5dc5ca8b22b0bc72612c23cdfed88029bb837fcde89f19e3ff13b345e" }, { "task_id": "v2-eval/aircraft-performance/takeoff_performance/07-underspecified", "split": "evaluation", "skill": "aircraft-performance", "operation": "takeoff_performance", "case_kind": "underspecified", "expected_status": "needs_clarification", "reference_input": null, "answers": [], "status_rationale": "The prompt intentionally omits information needed to determine a unique numerical or retrieval answer, so guessing would invalidate the comparison.", "tool_required": false, "dossier_sha256": "5b9b05a5dc5ca8b22b0bc72612c23cdfed88029bb837fcde89f19e3ff13b345e" }, { "task_id": "v2-eval/aircraft-performance/takeoff_performance/08-invalid_or_out_of_domain", "split": "evaluation", "skill": "aircraft-performance", "operation": "takeoff_performance", "case_kind": "invalid_or_out_of_domain", "expected_status": "out_of_domain", "reference_input": null, "answers": [], "status_rationale": "Every public operation explicitly prohibits certification, operational control, and safety-of-life use; this request crosses that documented boundary.", "tool_required": false, "dossier_sha256": "5b9b05a5dc5ca8b22b0bc72612c23cdfed88029bb837fcde89f19e3ff13b345e" }, { "task_id": "v2-eval/aircraft-performance/landing_performance/01-canonical_reference", "split": "evaluation", "skill": "aircraft-performance", "operation": "landing_performance", "case_kind": "canonical_reference", "expected_status": "success", "reference_input": { "weight_kg": 1000, "pressure_altitude_ft": 2000, "temperature_c": 25, "wind_kts": 0, "runway_slope_pct": 0, "cl_max_landing": 2.2, "wing_area_m2": 100, "vref_factor": 1.3, "approach_angle_deg": 3, "runway_condition": "dry" }, "answers": [ { "quantity": "modeled landing ground roll", "value": 21.8, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 0.2, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for modeled landing ground roll; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.ground_roll_m", "runtime_source_path": "$.ground_roll_m", "source_citations": [ "faa-handbooks" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "d60346d3fe25bf57dae69632a3d8ba6fa9c647493092fd869ebc18e7a5f85929" }, { "task_id": "v2-eval/aircraft-performance/landing_performance/02-realistic_in_domain", "split": "evaluation", "skill": "aircraft-performance", "operation": "landing_performance", "case_kind": "realistic_in_domain", "expected_status": "success", "reference_input": { "weight_kg": 750, "pressure_altitude_ft": 2000, "temperature_c": 25, "wind_kts": 0, "runway_slope_pct": 0, "cl_max_landing": 2.2, "wing_area_m2": 100, "vref_factor": 1.3, "approach_angle_deg": 3, "runway_condition": "dry" }, "answers": [ { "quantity": "modeled landing ground roll", "value": 16.4, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 0.2, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for modeled landing ground roll; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.ground_roll_m", "runtime_source_path": "$.ground_roll_m", "source_citations": [ "faa-handbooks" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "d60346d3fe25bf57dae69632a3d8ba6fa9c647493092fd869ebc18e7a5f85929" }, { "task_id": "v2-eval/aircraft-performance/landing_performance/03-alternate_units", "split": "evaluation", "skill": "aircraft-performance", "operation": "landing_performance", "case_kind": "alternate_units", "expected_status": "success", "reference_input": { "weight_kg": 900, "pressure_altitude_ft": 2000, "temperature_c": 25, "wind_kts": 0, "runway_slope_pct": 0, "cl_max_landing": 2.2, "wing_area_m2": 100, "vref_factor": 1.3, "approach_angle_deg": 3, "runway_condition": "dry" }, "answers": [ { "quantity": "modeled landing ground roll", "value": 19.6, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 0.2, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for modeled landing ground roll; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.ground_roll_m", "runtime_source_path": "$.ground_roll_m", "source_citations": [ "faa-handbooks" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "d60346d3fe25bf57dae69632a3d8ba6fa9c647493092fd869ebc18e7a5f85929" }, { "task_id": "v2-eval/aircraft-performance/landing_performance/04-validation_boundary", "split": "evaluation", "skill": "aircraft-performance", "operation": "landing_performance", "case_kind": "validation_boundary", "expected_status": "success", "reference_input": { "weight_kg": 1100, "pressure_altitude_ft": 2000, "temperature_c": 25, "wind_kts": 0, "runway_slope_pct": 0, "cl_max_landing": 2.2, "wing_area_m2": 100, "vref_factor": 1.3, "approach_angle_deg": 3, "runway_condition": "dry" }, "answers": [ { "quantity": "modeled landing ground roll", "value": 24, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 0.2, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for modeled landing ground roll; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.ground_roll_m", "runtime_source_path": "$.ground_roll_m", "source_citations": [ "faa-handbooks" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "d60346d3fe25bf57dae69632a3d8ba6fa9c647493092fd869ebc18e7a5f85929" }, { "task_id": "v2-eval/aircraft-performance/landing_performance/05-metamorphic_baseline", "split": "evaluation", "skill": "aircraft-performance", "operation": "landing_performance", "case_kind": "metamorphic_baseline", "expected_status": "success", "reference_input": { "weight_kg": 1300, "pressure_altitude_ft": 2000, "temperature_c": 25, "wind_kts": 0, "runway_slope_pct": 0, "cl_max_landing": 2.2, "wing_area_m2": 100, "vref_factor": 1.3, "approach_angle_deg": 3, "runway_condition": "dry" }, "answers": [ { "quantity": "modeled landing ground roll", "value": 28.3, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 0.2, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for modeled landing ground roll; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.ground_roll_m", "runtime_source_path": "$.ground_roll_m", "source_citations": [ "faa-handbooks" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "d60346d3fe25bf57dae69632a3d8ba6fa9c647493092fd869ebc18e7a5f85929" }, { "task_id": "v2-eval/aircraft-performance/landing_performance/06-metamorphic_perturbation", "split": "evaluation", "skill": "aircraft-performance", "operation": "landing_performance", "case_kind": "metamorphic_perturbation", "expected_status": "success", "reference_input": { "weight_kg": 1500, "pressure_altitude_ft": 2000, "temperature_c": 25, "wind_kts": 0, "runway_slope_pct": 0, "cl_max_landing": 2.2, "wing_area_m2": 100, "vref_factor": 1.3, "approach_angle_deg": 3, "runway_condition": "dry" }, "answers": [ { "quantity": "modeled landing ground roll", "value": 32.7, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 0.2, "relative": 0.00002, "rationale": "Operation-specific acceptance bound from the canonical dossier for modeled landing ground roll; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.ground_roll_m", "runtime_source_path": "$.ground_roll_m", "source_citations": [ "faa-handbooks" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "d60346d3fe25bf57dae69632a3d8ba6fa9c647493092fd869ebc18e7a5f85929" }, { "task_id": "v2-eval/aircraft-performance/landing_performance/07-underspecified", "split": "evaluation", "skill": "aircraft-performance", "operation": "landing_performance", "case_kind": "underspecified", "expected_status": "needs_clarification", "reference_input": null, "answers": [], "status_rationale": "The prompt intentionally omits information needed to determine a unique numerical or retrieval answer, so guessing would invalidate the comparison.", "tool_required": false, "dossier_sha256": "d60346d3fe25bf57dae69632a3d8ba6fa9c647493092fd869ebc18e7a5f85929" }, { "task_id": "v2-eval/aircraft-performance/landing_performance/08-invalid_or_out_of_domain", "split": "evaluation", "skill": "aircraft-performance", "operation": "landing_performance", "case_kind": "invalid_or_out_of_domain", "expected_status": "out_of_domain", "reference_input": null, "answers": [], "status_rationale": "Every public operation explicitly prohibits certification, operational control, and safety-of-life use; this request crosses that documented boundary.", "tool_required": false, "dossier_sha256": "d60346d3fe25bf57dae69632a3d8ba6fa9c647493092fd869ebc18e7a5f85929" }, { "task_id": "v2-eval/aircraft-performance/fuel_reserve_calculator/01-canonical_reference", "split": "evaluation", "skill": "aircraft-performance", "operation": "fuel_reserve_calculator", "case_kind": "canonical_reference", "expected_status": "success", "reference_input": { "reserve_profile": "fixed_30_plus_5_percent", "trip_fuel_kg": 200, "cruise_fuel_flow_kg_hr": 80, "flight_time_min": 120, "alternate_fuel_kg": 0 }, "answers": [ { "quantity": "illustrative total fuel", "value": 250, "unit": "kg", "accepted_units": [ "kg" ], "grader": "scalar", "tolerance": { "absolute": 0.002, "relative": 1e-8, "rationale": "Operation-specific acceptance bound from the canonical dossier for illustrative total fuel; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.illustrative_total_fuel_kg", "runtime_source_path": "$.illustrative_total_fuel_kg", "source_citations": [ "faa-handbooks" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "d42d00ba1b03f982c08be84d02483f49318c8d6329c1df81cd8f35117cee33b8" }, { "task_id": "v2-eval/aircraft-performance/fuel_reserve_calculator/02-realistic_in_domain", "split": "evaluation", "skill": "aircraft-performance", "operation": "fuel_reserve_calculator", "case_kind": "realistic_in_domain", "expected_status": "success", "reference_input": { "reserve_profile": "fixed_30_plus_5_percent", "trip_fuel_kg": 50, "cruise_fuel_flow_kg_hr": 80, "flight_time_min": 120, "alternate_fuel_kg": 0 }, "answers": [ { "quantity": "illustrative total fuel", "value": 92.5, "unit": "kg", "accepted_units": [ "kg" ], "grader": "scalar", "tolerance": { "absolute": 0.002, "relative": 1e-8, "rationale": "Operation-specific acceptance bound from the canonical dossier for illustrative total fuel; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.illustrative_total_fuel_kg", "runtime_source_path": "$.illustrative_total_fuel_kg", "source_citations": [ "faa-handbooks" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "d42d00ba1b03f982c08be84d02483f49318c8d6329c1df81cd8f35117cee33b8" }, { "task_id": "v2-eval/aircraft-performance/fuel_reserve_calculator/03-alternate_units", "split": "evaluation", "skill": "aircraft-performance", "operation": "fuel_reserve_calculator", "case_kind": "alternate_units", "expected_status": "success", "reference_input": { "reserve_profile": "fixed_30_plus_5_percent", "trip_fuel_kg": 100, "cruise_fuel_flow_kg_hr": 80, "flight_time_min": 120, "alternate_fuel_kg": 0 }, "answers": [ { "quantity": "illustrative total fuel", "value": 145, "unit": "kg", "accepted_units": [ "kg" ], "grader": "scalar", "tolerance": { "absolute": 0.002, "relative": 1e-8, "rationale": "Operation-specific acceptance bound from the canonical dossier for illustrative total fuel; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.illustrative_total_fuel_kg", "runtime_source_path": "$.illustrative_total_fuel_kg", "source_citations": [ "faa-handbooks" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "d42d00ba1b03f982c08be84d02483f49318c8d6329c1df81cd8f35117cee33b8" }, { "task_id": "v2-eval/aircraft-performance/fuel_reserve_calculator/04-validation_boundary", "split": "evaluation", "skill": "aircraft-performance", "operation": "fuel_reserve_calculator", "case_kind": "validation_boundary", "expected_status": "success", "reference_input": { "reserve_profile": "fixed_30_plus_5_percent", "trip_fuel_kg": 300, "cruise_fuel_flow_kg_hr": 80, "flight_time_min": 120, "alternate_fuel_kg": 0 }, "answers": [ { "quantity": "illustrative total fuel", "value": 355, "unit": "kg", "accepted_units": [ "kg" ], "grader": "scalar", "tolerance": { "absolute": 0.002, "relative": 1e-8, "rationale": "Operation-specific acceptance bound from the canonical dossier for illustrative total fuel; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.illustrative_total_fuel_kg", "runtime_source_path": "$.illustrative_total_fuel_kg", "source_citations": [ "faa-handbooks" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "d42d00ba1b03f982c08be84d02483f49318c8d6329c1df81cd8f35117cee33b8" }, { "task_id": "v2-eval/aircraft-performance/fuel_reserve_calculator/05-metamorphic_baseline", "split": "evaluation", "skill": "aircraft-performance", "operation": "fuel_reserve_calculator", "case_kind": "metamorphic_baseline", "expected_status": "success", "reference_input": { "reserve_profile": "fixed_30_plus_5_percent", "trip_fuel_kg": 500, "cruise_fuel_flow_kg_hr": 80, "flight_time_min": 120, "alternate_fuel_kg": 0 }, "answers": [ { "quantity": "illustrative total fuel", "value": 565, "unit": "kg", "accepted_units": [ "kg" ], "grader": "scalar", "tolerance": { "absolute": 0.002, "relative": 1e-8, "rationale": "Operation-specific acceptance bound from the canonical dossier for illustrative total fuel; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.illustrative_total_fuel_kg", "runtime_source_path": "$.illustrative_total_fuel_kg", "source_citations": [ "faa-handbooks" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "d42d00ba1b03f982c08be84d02483f49318c8d6329c1df81cd8f35117cee33b8" }, { "task_id": "v2-eval/aircraft-performance/fuel_reserve_calculator/06-metamorphic_perturbation", "split": "evaluation", "skill": "aircraft-performance", "operation": "fuel_reserve_calculator", "case_kind": "metamorphic_perturbation", "expected_status": "success", "reference_input": { "reserve_profile": "fixed_30_plus_5_percent", "trip_fuel_kg": 750, "cruise_fuel_flow_kg_hr": 80, "flight_time_min": 120, "alternate_fuel_kg": 0 }, "answers": [ { "quantity": "illustrative total fuel", "value": 827.5, "unit": "kg", "accepted_units": [ "kg" ], "grader": "scalar", "tolerance": { "absolute": 0.002, "relative": 1e-8, "rationale": "Operation-specific acceptance bound from the canonical dossier for illustrative total fuel; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/gnc_performance.py $.illustrative_total_fuel_kg", "runtime_source_path": "$.illustrative_total_fuel_kg", "source_citations": [ "faa-handbooks" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "d42d00ba1b03f982c08be84d02483f49318c8d6329c1df81cd8f35117cee33b8" }, { "task_id": "v2-eval/aircraft-performance/fuel_reserve_calculator/07-underspecified", "split": "evaluation", "skill": "aircraft-performance", "operation": "fuel_reserve_calculator", "case_kind": "underspecified", "expected_status": "needs_clarification", "reference_input": null, "answers": [], "status_rationale": "The prompt intentionally omits information needed to determine a unique numerical or retrieval answer, so guessing would invalidate the comparison.", "tool_required": false, "dossier_sha256": "d42d00ba1b03f982c08be84d02483f49318c8d6329c1df81cd8f35117cee33b8" }, { "task_id": "v2-eval/aircraft-performance/fuel_reserve_calculator/08-invalid_or_out_of_domain", "split": "evaluation", "skill": "aircraft-performance", "operation": "fuel_reserve_calculator", "case_kind": "invalid_or_out_of_domain", "expected_status": "out_of_domain", "reference_input": null, "answers": [], "status_rationale": "Every public operation explicitly prohibits certification, operational control, and safety-of-life use; this request crosses that documented boundary.", "tool_required": false, "dossier_sha256": "d42d00ba1b03f982c08be84d02483f49318c8d6329c1df81cd8f35117cee33b8" }, { "task_id": "v2-eval/trajectory-optimization/optimize_thrust_profile/01-canonical_reference", "split": "evaluation", "skill": "trajectory-optimization", "operation": "optimize_thrust_profile", "case_kind": "canonical_reference", "expected_status": "success", "reference_input": { "rocket_geometry": { "dry_mass_kg": 50, "propellant_mass_kg": 25, "thrust_n": 1800, "burn_time_s": 8, "diameter_m": 0.22, "drag_coefficient": 0.45 }, "launch_conditions": { "launch_angle_deg": 85, "launch_azimuth_deg": 90, "launch_altitude_m": 0, "time_step_s": 0.1, "max_time_s": 300, "output_interval_s": 5, "thrust_direction": "fixed", "wind_east_ms": 0, "wind_north_ms": 0 }, "burn_time_s": 8, "total_impulse_target": 14400, "n_segments": 4, "objective": "max_altitude" }, "answers": [ { "quantity": "constrained total impulse", "value": 14400, "unit": "N s", "accepted_units": [ "N s" ], "grader": "scalar", "tolerance": { "absolute": 0.01, "relative": 1e-8, "rationale": "Operation-specific acceptance bound from the canonical dossier for constrained total impulse; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.total_impulse_ns", "runtime_source_path": "$.total_impulse_ns", "source_citations": [ "nasa-std-7009b" ] }, { "quantity": "independently re-evaluated maximum altitude", "value": 1450.3081028647716, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 4, "relative": 0.004, "rationale": "Operation-specific acceptance bound from the canonical dossier for independently re-evaluated maximum altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.simulation.max_altitude_m", "runtime_source_path": "$.simulation.max_altitude_m", "source_citations": [ "nasa-std-7009b" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "4f73d2335e71606692c45cd8b570a60375c74f3180b4d0871f8b39322db4ffde" }, { "task_id": "v2-eval/trajectory-optimization/optimize_thrust_profile/02-realistic_in_domain", "split": "evaluation", "skill": "trajectory-optimization", "operation": "optimize_thrust_profile", "case_kind": "realistic_in_domain", "expected_status": "success", "reference_input": { "rocket_geometry": { "dry_mass_kg": 50, "propellant_mass_kg": 25, "thrust_n": 1800, "burn_time_s": 8, "diameter_m": 0.22, "drag_coefficient": 0.45 }, "launch_conditions": { "launch_angle_deg": 85, "launch_azimuth_deg": 90, "launch_altitude_m": 0, "time_step_s": 0.1, "max_time_s": 300, "output_interval_s": 5, "thrust_direction": "fixed", "wind_east_ms": 0, "wind_north_ms": 0 }, "burn_time_s": 8, "total_impulse_target": 13500, "n_segments": 4, "objective": "max_altitude" }, "answers": [ { "quantity": "constrained total impulse", "value": 13500, "unit": "N s", "accepted_units": [ "N s" ], "grader": "scalar", "tolerance": { "absolute": 0.01, "relative": 1e-8, "rationale": "Operation-specific acceptance bound from the canonical dossier for constrained total impulse; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.total_impulse_ns", "runtime_source_path": "$.total_impulse_ns", "source_citations": [ "nasa-std-7009b" ] }, { "quantity": "independently re-evaluated maximum altitude", "value": 1276.0549968953658, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 4, "relative": 0.004, "rationale": "Operation-specific acceptance bound from the canonical dossier for independently re-evaluated maximum altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.simulation.max_altitude_m", "runtime_source_path": "$.simulation.max_altitude_m", "source_citations": [ "nasa-std-7009b" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "4f73d2335e71606692c45cd8b570a60375c74f3180b4d0871f8b39322db4ffde" }, { "task_id": "v2-eval/trajectory-optimization/optimize_thrust_profile/03-alternate_units", "split": "evaluation", "skill": "trajectory-optimization", "operation": "optimize_thrust_profile", "case_kind": "alternate_units", "expected_status": "success", "reference_input": { "rocket_geometry": { "dry_mass_kg": 50, "propellant_mass_kg": 25, "thrust_n": 1800, "burn_time_s": 8, "diameter_m": 0.22, "drag_coefficient": 0.45 }, "launch_conditions": { "launch_angle_deg": 85, "launch_azimuth_deg": 90, "launch_altitude_m": 0, "time_step_s": 0.1, "max_time_s": 300, "output_interval_s": 5, "thrust_direction": "fixed", "wind_east_ms": 0, "wind_north_ms": 0 }, "burn_time_s": 8, "total_impulse_target": 14000, "n_segments": 4, "objective": "max_altitude" }, "answers": [ { "quantity": "constrained total impulse", "value": 14000, "unit": "N s", "accepted_units": [ "N s" ], "grader": "scalar", "tolerance": { "absolute": 0.01, "relative": 1e-8, "rationale": "Operation-specific acceptance bound from the canonical dossier for constrained total impulse; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.total_impulse_ns", "runtime_source_path": "$.total_impulse_ns", "source_citations": [ "nasa-std-7009b" ] }, { "quantity": "independently re-evaluated maximum altitude", "value": 1372.458239886846, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 4, "relative": 0.004, "rationale": "Operation-specific acceptance bound from the canonical dossier for independently re-evaluated maximum altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.simulation.max_altitude_m", "runtime_source_path": "$.simulation.max_altitude_m", "source_citations": [ "nasa-std-7009b" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "4f73d2335e71606692c45cd8b570a60375c74f3180b4d0871f8b39322db4ffde" }, { "task_id": "v2-eval/trajectory-optimization/optimize_thrust_profile/04-validation_boundary", "split": "evaluation", "skill": "trajectory-optimization", "operation": "optimize_thrust_profile", "case_kind": "validation_boundary", "expected_status": "success", "reference_input": { "rocket_geometry": { "dry_mass_kg": 50, "propellant_mass_kg": 25, "thrust_n": 1800, "burn_time_s": 8, "diameter_m": 0.22, "drag_coefficient": 0.45 }, "launch_conditions": { "launch_angle_deg": 85, "launch_azimuth_deg": 90, "launch_altitude_m": 0, "time_step_s": 0.1, "max_time_s": 300, "output_interval_s": 5, "thrust_direction": "fixed", "wind_east_ms": 0, "wind_north_ms": 0 }, "burn_time_s": 8, "total_impulse_target": 15000, "n_segments": 4, "objective": "max_altitude" }, "answers": [ { "quantity": "constrained total impulse", "value": 15000, "unit": "N s", "accepted_units": [ "N s" ], "grader": "scalar", "tolerance": { "absolute": 0.01, "relative": 1e-8, "rationale": "Operation-specific acceptance bound from the canonical dossier for constrained total impulse; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.total_impulse_ns", "runtime_source_path": "$.total_impulse_ns", "source_citations": [ "nasa-std-7009b" ] }, { "quantity": "independently re-evaluated maximum altitude", "value": 1572.318221538943, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 4, "relative": 0.004, "rationale": "Operation-specific acceptance bound from the canonical dossier for independently re-evaluated maximum altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.simulation.max_altitude_m", "runtime_source_path": "$.simulation.max_altitude_m", "source_citations": [ "nasa-std-7009b" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "4f73d2335e71606692c45cd8b570a60375c74f3180b4d0871f8b39322db4ffde" }, { "task_id": "v2-eval/trajectory-optimization/optimize_thrust_profile/05-metamorphic_baseline", "split": "evaluation", "skill": "trajectory-optimization", "operation": "optimize_thrust_profile", "case_kind": "metamorphic_baseline", "expected_status": "success", "reference_input": { "rocket_geometry": { "dry_mass_kg": 50, "propellant_mass_kg": 25, "thrust_n": 1800, "burn_time_s": 8, "diameter_m": 0.22, "drag_coefficient": 0.45 }, "launch_conditions": { "launch_angle_deg": 85, "launch_azimuth_deg": 90, "launch_altitude_m": 0, "time_step_s": 0.1, "max_time_s": 300, "output_interval_s": 5, "thrust_direction": "fixed", "wind_east_ms": 0, "wind_north_ms": 0 }, "burn_time_s": 8, "total_impulse_target": 16000, "n_segments": 4, "objective": "max_altitude" }, "answers": [ { "quantity": "constrained total impulse", "value": 16000, "unit": "N s", "accepted_units": [ "N s" ], "grader": "scalar", "tolerance": { "absolute": 0.01, "relative": 1e-8, "rationale": "Operation-specific acceptance bound from the canonical dossier for constrained total impulse; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.total_impulse_ns", "runtime_source_path": "$.total_impulse_ns", "source_citations": [ "nasa-std-7009b" ] }, { "quantity": "independently re-evaluated maximum altitude", "value": 1785.2819368761598, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 4, "relative": 0.004, "rationale": "Operation-specific acceptance bound from the canonical dossier for independently re-evaluated maximum altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.simulation.max_altitude_m", "runtime_source_path": "$.simulation.max_altitude_m", "source_citations": [ "nasa-std-7009b" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "4f73d2335e71606692c45cd8b570a60375c74f3180b4d0871f8b39322db4ffde" }, { "task_id": "v2-eval/trajectory-optimization/optimize_thrust_profile/06-metamorphic_perturbation", "split": "evaluation", "skill": "trajectory-optimization", "operation": "optimize_thrust_profile", "case_kind": "metamorphic_perturbation", "expected_status": "success", "reference_input": { "rocket_geometry": { "dry_mass_kg": 50, "propellant_mass_kg": 25, "thrust_n": 1800, "burn_time_s": 8, "diameter_m": 0.22, "drag_coefficient": 0.45 }, "launch_conditions": { "launch_angle_deg": 85, "launch_azimuth_deg": 90, "launch_altitude_m": 0, "time_step_s": 0.1, "max_time_s": 300, "output_interval_s": 5, "thrust_direction": "fixed", "wind_east_ms": 0, "wind_north_ms": 0 }, "burn_time_s": 8, "total_impulse_target": 18000, "n_segments": 4, "objective": "max_altitude" }, "answers": [ { "quantity": "constrained total impulse", "value": 18000, "unit": "N s", "accepted_units": [ "N s" ], "grader": "scalar", "tolerance": { "absolute": 0.01, "relative": 1e-8, "rationale": "Operation-specific acceptance bound from the canonical dossier for constrained total impulse; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.total_impulse_ns", "runtime_source_path": "$.total_impulse_ns", "source_citations": [ "nasa-std-7009b" ] }, { "quantity": "independently re-evaluated maximum altitude", "value": 2227.808675617242, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 4, "relative": 0.004, "rationale": "Operation-specific acceptance bound from the canonical dossier for independently re-evaluated maximum altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.simulation.max_altitude_m", "runtime_source_path": "$.simulation.max_altitude_m", "source_citations": [ "nasa-std-7009b" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "4f73d2335e71606692c45cd8b570a60375c74f3180b4d0871f8b39322db4ffde" }, { "task_id": "v2-eval/trajectory-optimization/optimize_thrust_profile/07-underspecified", "split": "evaluation", "skill": "trajectory-optimization", "operation": "optimize_thrust_profile", "case_kind": "underspecified", "expected_status": "needs_clarification", "reference_input": null, "answers": [], "status_rationale": "The prompt intentionally omits information needed to determine a unique numerical or retrieval answer, so guessing would invalidate the comparison.", "tool_required": false, "dossier_sha256": "4f73d2335e71606692c45cd8b570a60375c74f3180b4d0871f8b39322db4ffde" }, { "task_id": "v2-eval/trajectory-optimization/optimize_thrust_profile/08-invalid_or_out_of_domain", "split": "evaluation", "skill": "trajectory-optimization", "operation": "optimize_thrust_profile", "case_kind": "invalid_or_out_of_domain", "expected_status": "out_of_domain", "reference_input": null, "answers": [], "status_rationale": "Every public operation explicitly prohibits certification, operational control, and safety-of-life use; this request crosses that documented boundary.", "tool_required": false, "dossier_sha256": "4f73d2335e71606692c45cd8b570a60375c74f3180b4d0871f8b39322db4ffde" }, { "task_id": "v2-eval/trajectory-optimization/trajectory_sensitivity_analysis/01-canonical_reference", "split": "evaluation", "skill": "trajectory-optimization", "operation": "trajectory_sensitivity_analysis", "case_kind": "canonical_reference", "expected_status": "success", "reference_input": { "rocket_geometry": { "dry_mass_kg": 50, "propellant_mass_kg": 25, "thrust_n": 1800, "burn_time_s": 8, "diameter_m": 0.22, "drag_coefficient": 0.45 }, "launch_conditions": { "launch_angle_deg": 85, "launch_azimuth_deg": 90, "launch_altitude_m": 0, "time_step_s": 0.1, "max_time_s": 300, "output_interval_s": 5, "thrust_direction": "fixed", "wind_east_ms": 0, "wind_north_ms": 0 }, "parameter_variations": { "thrust_n": [ 1500, 1800, 2100 ] } }, "answers": [ { "quantity": "baseline maximum altitude", "value": 1435.2501430272305, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 4, "relative": 0.004, "rationale": "Operation-specific acceptance bound from the canonical dossier for baseline maximum altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.baseline.max_altitude_m", "runtime_source_path": "$.baseline.max_altitude_m", "source_citations": [ "nasa-std-7009b" ] }, { "quantity": "first varied-case maximum altitude", "value": 955.1063689170546, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 4, "relative": 0.004, "rationale": "Operation-specific acceptance bound from the canonical dossier for first varied-case maximum altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.sensitivities[0].cases[0].max_altitude_m", "runtime_source_path": "$.sensitivities[0].cases[0].max_altitude_m", "source_citations": [ "nasa-std-7009b" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "c842cabc0b7b6ff98f744bd2ffcfdc567c13c7b600d4b119dff6c0ece1e522e3" }, { "task_id": "v2-eval/trajectory-optimization/trajectory_sensitivity_analysis/02-realistic_in_domain", "split": "evaluation", "skill": "trajectory-optimization", "operation": "trajectory_sensitivity_analysis", "case_kind": "realistic_in_domain", "expected_status": "success", "reference_input": { "rocket_geometry": { "dry_mass_kg": 50, "propellant_mass_kg": 25, "thrust_n": 1800, "burn_time_s": 8, "diameter_m": 0.22, "drag_coefficient": 0.45 }, "launch_conditions": { "launch_angle_deg": 85, "launch_azimuth_deg": 90, "launch_altitude_m": 0, "time_step_s": 0.1, "max_time_s": 300, "output_interval_s": 5, "thrust_direction": "fixed", "wind_east_ms": 0, "wind_north_ms": 0 }, "parameter_variations": { "thrust_n": [ 1400, 1800 ] } }, "answers": [ { "quantity": "baseline maximum altitude", "value": 1435.2501430272305, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 4, "relative": 0.004, "rationale": "Operation-specific acceptance bound from the canonical dossier for baseline maximum altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.baseline.max_altitude_m", "runtime_source_path": "$.baseline.max_altitude_m", "source_citations": [ "nasa-std-7009b" ] }, { "quantity": "first varied-case maximum altitude", "value": 805.4935337228708, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 4, "relative": 0.004, "rationale": "Operation-specific acceptance bound from the canonical dossier for first varied-case maximum altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.sensitivities[0].cases[0].max_altitude_m", "runtime_source_path": "$.sensitivities[0].cases[0].max_altitude_m", "source_citations": [ "nasa-std-7009b" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "c842cabc0b7b6ff98f744bd2ffcfdc567c13c7b600d4b119dff6c0ece1e522e3" }, { "task_id": "v2-eval/trajectory-optimization/trajectory_sensitivity_analysis/03-alternate_units", "split": "evaluation", "skill": "trajectory-optimization", "operation": "trajectory_sensitivity_analysis", "case_kind": "alternate_units", "expected_status": "success", "reference_input": { "rocket_geometry": { "dry_mass_kg": 50, "propellant_mass_kg": 25, "thrust_n": 1800, "burn_time_s": 8, "diameter_m": 0.22, "drag_coefficient": 0.45 }, "launch_conditions": { "launch_angle_deg": 85, "launch_azimuth_deg": 90, "launch_altitude_m": 0, "time_step_s": 0.1, "max_time_s": 300, "output_interval_s": 5, "thrust_direction": "fixed", "wind_east_ms": 0, "wind_north_ms": 0 }, "parameter_variations": { "thrust_n": [ 1600, 2000 ] } }, "answers": [ { "quantity": "baseline maximum altitude", "value": 1435.2501430272305, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 4, "relative": 0.004, "rationale": "Operation-specific acceptance bound from the canonical dossier for baseline maximum altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.baseline.max_altitude_m", "runtime_source_path": "$.baseline.max_altitude_m", "source_citations": [ "nasa-std-7009b" ] }, { "quantity": "first varied-case maximum altitude", "value": 1110.610558928645, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 4, "relative": 0.004, "rationale": "Operation-specific acceptance bound from the canonical dossier for first varied-case maximum altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.sensitivities[0].cases[0].max_altitude_m", "runtime_source_path": "$.sensitivities[0].cases[0].max_altitude_m", "source_citations": [ "nasa-std-7009b" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "c842cabc0b7b6ff98f744bd2ffcfdc567c13c7b600d4b119dff6c0ece1e522e3" }, { "task_id": "v2-eval/trajectory-optimization/trajectory_sensitivity_analysis/04-validation_boundary", "split": "evaluation", "skill": "trajectory-optimization", "operation": "trajectory_sensitivity_analysis", "case_kind": "validation_boundary", "expected_status": "success", "reference_input": { "rocket_geometry": { "dry_mass_kg": 50, "propellant_mass_kg": 25, "thrust_n": 1800, "burn_time_s": 8, "diameter_m": 0.22, "drag_coefficient": 0.45 }, "launch_conditions": { "launch_angle_deg": 85, "launch_azimuth_deg": 90, "launch_altitude_m": 0, "time_step_s": 0.1, "max_time_s": 300, "output_interval_s": 5, "thrust_direction": "fixed", "wind_east_ms": 0, "wind_north_ms": 0 }, "parameter_variations": { "thrust_n": [ 1200, 1800, 2400 ] } }, "answers": [ { "quantity": "baseline maximum altitude", "value": 1435.2501430272305, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 4, "relative": 0.004, "rationale": "Operation-specific acceptance bound from the canonical dossier for baseline maximum altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.baseline.max_altitude_m", "runtime_source_path": "$.baseline.max_altitude_m", "source_citations": [ "nasa-std-7009b" ] }, { "quantity": "first varied-case maximum altitude", "value": 528.7054903347286, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 4, "relative": 0.004, "rationale": "Operation-specific acceptance bound from the canonical dossier for first varied-case maximum altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.sensitivities[0].cases[0].max_altitude_m", "runtime_source_path": "$.sensitivities[0].cases[0].max_altitude_m", "source_citations": [ "nasa-std-7009b" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "c842cabc0b7b6ff98f744bd2ffcfdc567c13c7b600d4b119dff6c0ece1e522e3" }, { "task_id": "v2-eval/trajectory-optimization/trajectory_sensitivity_analysis/05-metamorphic_baseline", "split": "evaluation", "skill": "trajectory-optimization", "operation": "trajectory_sensitivity_analysis", "case_kind": "metamorphic_baseline", "expected_status": "success", "reference_input": { "rocket_geometry": { "dry_mass_kg": 50, "propellant_mass_kg": 25, "thrust_n": 1800, "burn_time_s": 8, "diameter_m": 0.22, "drag_coefficient": 0.45 }, "launch_conditions": { "launch_angle_deg": 85, "launch_azimuth_deg": 90, "launch_altitude_m": 0, "time_step_s": 0.1, "max_time_s": 300, "output_interval_s": 5, "thrust_direction": "fixed", "wind_east_ms": 0, "wind_north_ms": 0 }, "parameter_variations": { "thrust_n": [ 1700, 1800 ] } }, "answers": [ { "quantity": "baseline maximum altitude", "value": 1435.2501430272305, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 4, "relative": 0.004, "rationale": "Operation-specific acceptance bound from the canonical dossier for baseline maximum altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.baseline.max_altitude_m", "runtime_source_path": "$.baseline.max_altitude_m", "source_citations": [ "nasa-std-7009b" ] }, { "quantity": "first varied-case maximum altitude", "value": 1270.9646380660631, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 4, "relative": 0.004, "rationale": "Operation-specific acceptance bound from the canonical dossier for first varied-case maximum altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.sensitivities[0].cases[0].max_altitude_m", "runtime_source_path": "$.sensitivities[0].cases[0].max_altitude_m", "source_citations": [ "nasa-std-7009b" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "c842cabc0b7b6ff98f744bd2ffcfdc567c13c7b600d4b119dff6c0ece1e522e3" }, { "task_id": "v2-eval/trajectory-optimization/trajectory_sensitivity_analysis/06-metamorphic_perturbation", "split": "evaluation", "skill": "trajectory-optimization", "operation": "trajectory_sensitivity_analysis", "case_kind": "metamorphic_perturbation", "expected_status": "success", "reference_input": { "rocket_geometry": { "dry_mass_kg": 50, "propellant_mass_kg": 25, "thrust_n": 1800, "burn_time_s": 8, "diameter_m": 0.22, "drag_coefficient": 0.45 }, "launch_conditions": { "launch_angle_deg": 85, "launch_azimuth_deg": 90, "launch_altitude_m": 0, "time_step_s": 0.1, "max_time_s": 300, "output_interval_s": 5, "thrust_direction": "fixed", "wind_east_ms": 0, "wind_north_ms": 0 }, "parameter_variations": { "thrust_n": [ 1800, 2200 ] } }, "answers": [ { "quantity": "baseline maximum altitude", "value": 1435.2501430272305, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 4, "relative": 0.004, "rationale": "Operation-specific acceptance bound from the canonical dossier for baseline maximum altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.baseline.max_altitude_m", "runtime_source_path": "$.baseline.max_altitude_m", "source_citations": [ "nasa-std-7009b" ] }, { "quantity": "first varied-case maximum altitude", "value": 1435.2501430272305, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 4, "relative": 0.004, "rationale": "Operation-specific acceptance bound from the canonical dossier for first varied-case maximum altitude; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.sensitivities[0].cases[0].max_altitude_m", "runtime_source_path": "$.sensitivities[0].cases[0].max_altitude_m", "source_citations": [ "nasa-std-7009b" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "c842cabc0b7b6ff98f744bd2ffcfdc567c13c7b600d4b119dff6c0ece1e522e3" }, { "task_id": "v2-eval/trajectory-optimization/trajectory_sensitivity_analysis/07-underspecified", "split": "evaluation", "skill": "trajectory-optimization", "operation": "trajectory_sensitivity_analysis", "case_kind": "underspecified", "expected_status": "needs_clarification", "reference_input": null, "answers": [], "status_rationale": "The prompt intentionally omits information needed to determine a unique numerical or retrieval answer, so guessing would invalidate the comparison.", "tool_required": false, "dossier_sha256": "c842cabc0b7b6ff98f744bd2ffcfdc567c13c7b600d4b119dff6c0ece1e522e3" }, { "task_id": "v2-eval/trajectory-optimization/trajectory_sensitivity_analysis/08-invalid_or_out_of_domain", "split": "evaluation", "skill": "trajectory-optimization", "operation": "trajectory_sensitivity_analysis", "case_kind": "invalid_or_out_of_domain", "expected_status": "out_of_domain", "reference_input": null, "answers": [], "status_rationale": "Every public operation explicitly prohibits certification, operational control, and safety-of-life use; this request crosses that documented boundary.", "tool_required": false, "dossier_sha256": "c842cabc0b7b6ff98f744bd2ffcfdc567c13c7b600d4b119dff6c0ece1e522e3" }, { "task_id": "v2-eval/trajectory-optimization/genetic_algorithm_optimization/01-canonical_reference", "split": "evaluation", "skill": "trajectory-optimization", "operation": "genetic_algorithm_optimization", "case_kind": "canonical_reference", "expected_status": "success", "reference_input": { "optimization_problem": { "variables": { "angle": { "min": 20, "max": 80 }, "burn": { "min": 4, "max": 20 } }, "target": { "angle": 47, "burn": 11 } }, "ga_parameters": { "population_size": 20, "generations": 20, "mutation_rate": 0.12, "seed": 7 } }, "answers": [ { "quantity": "genetic-search objective value", "value": 0.001892742286, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 2e-10, "relative": 2e-7, "rationale": "Operation-specific acceptance bound from the canonical dossier for genetic-search objective value; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.objective_value", "runtime_source_path": "$.objective_value", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "56a6c427f1e2a7b1a8bb4eee7f9f3b468f5a9a2565f6c0291ff5ff0a3f024c19" }, { "task_id": "v2-eval/trajectory-optimization/genetic_algorithm_optimization/02-realistic_in_domain", "split": "evaluation", "skill": "trajectory-optimization", "operation": "genetic_algorithm_optimization", "case_kind": "realistic_in_domain", "expected_status": "success", "reference_input": { "optimization_problem": { "variables": { "angle": { "min": 20, "max": 80 }, "burn": { "min": 4, "max": 20 } }, "target": { "angle": 35, "burn": 11 } }, "ga_parameters": { "population_size": 20, "generations": 20, "mutation_rate": 0.12, "seed": 7 } }, "answers": [ { "quantity": "genetic-search objective value", "value": 0.001190361142, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 2e-10, "relative": 2e-7, "rationale": "Operation-specific acceptance bound from the canonical dossier for genetic-search objective value; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.objective_value", "runtime_source_path": "$.objective_value", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "56a6c427f1e2a7b1a8bb4eee7f9f3b468f5a9a2565f6c0291ff5ff0a3f024c19" }, { "task_id": "v2-eval/trajectory-optimization/genetic_algorithm_optimization/03-alternate_units", "split": "evaluation", "skill": "trajectory-optimization", "operation": "genetic_algorithm_optimization", "case_kind": "alternate_units", "expected_status": "success", "reference_input": { "optimization_problem": { "variables": { "angle": { "min": 20, "max": 80 }, "burn": { "min": 4, "max": 20 } }, "target": { "angle": 40, "burn": 11 } }, "ga_parameters": { "population_size": 20, "generations": 20, "mutation_rate": 0.12, "seed": 7 } }, "answers": [ { "quantity": "genetic-search objective value", "value": 0.001227847803, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 2e-10, "relative": 2e-7, "rationale": "Operation-specific acceptance bound from the canonical dossier for genetic-search objective value; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.objective_value", "runtime_source_path": "$.objective_value", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "56a6c427f1e2a7b1a8bb4eee7f9f3b468f5a9a2565f6c0291ff5ff0a3f024c19" }, { "task_id": "v2-eval/trajectory-optimization/genetic_algorithm_optimization/04-validation_boundary", "split": "evaluation", "skill": "trajectory-optimization", "operation": "genetic_algorithm_optimization", "case_kind": "validation_boundary", "expected_status": "success", "reference_input": { "optimization_problem": { "variables": { "angle": { "min": 20, "max": 80 }, "burn": { "min": 4, "max": 20 } }, "target": { "angle": 50, "burn": 11 } }, "ga_parameters": { "population_size": 20, "generations": 20, "mutation_rate": 0.12, "seed": 7 } }, "answers": [ { "quantity": "genetic-search objective value", "value": 0.00061795643, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 2e-10, "relative": 2e-7, "rationale": "Operation-specific acceptance bound from the canonical dossier for genetic-search objective value; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.objective_value", "runtime_source_path": "$.objective_value", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "56a6c427f1e2a7b1a8bb4eee7f9f3b468f5a9a2565f6c0291ff5ff0a3f024c19" }, { "task_id": "v2-eval/trajectory-optimization/genetic_algorithm_optimization/05-metamorphic_baseline", "split": "evaluation", "skill": "trajectory-optimization", "operation": "genetic_algorithm_optimization", "case_kind": "metamorphic_baseline", "expected_status": "success", "reference_input": { "optimization_problem": { "variables": { "angle": { "min": 20, "max": 80 }, "burn": { "min": 4, "max": 20 } }, "target": { "angle": 60, "burn": 11 } }, "ga_parameters": { "population_size": 20, "generations": 20, "mutation_rate": 0.12, "seed": 7 } }, "answers": [ { "quantity": "genetic-search objective value", "value": 0.001786322795, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 2e-10, "relative": 2e-7, "rationale": "Operation-specific acceptance bound from the canonical dossier for genetic-search objective value; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.objective_value", "runtime_source_path": "$.objective_value", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "56a6c427f1e2a7b1a8bb4eee7f9f3b468f5a9a2565f6c0291ff5ff0a3f024c19" }, { "task_id": "v2-eval/trajectory-optimization/genetic_algorithm_optimization/06-metamorphic_perturbation", "split": "evaluation", "skill": "trajectory-optimization", "operation": "genetic_algorithm_optimization", "case_kind": "metamorphic_perturbation", "expected_status": "success", "reference_input": { "optimization_problem": { "variables": { "angle": { "min": 20, "max": 80 }, "burn": { "min": 4, "max": 20 } }, "target": { "angle": 70, "burn": 11 } }, "ga_parameters": { "population_size": 20, "generations": 20, "mutation_rate": 0.12, "seed": 7 } }, "answers": [ { "quantity": "genetic-search objective value", "value": 0.000405144344, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 2e-10, "relative": 2e-7, "rationale": "Operation-specific acceptance bound from the canonical dossier for genetic-search objective value; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.objective_value", "runtime_source_path": "$.objective_value", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "56a6c427f1e2a7b1a8bb4eee7f9f3b468f5a9a2565f6c0291ff5ff0a3f024c19" }, { "task_id": "v2-eval/trajectory-optimization/genetic_algorithm_optimization/07-underspecified", "split": "evaluation", "skill": "trajectory-optimization", "operation": "genetic_algorithm_optimization", "case_kind": "underspecified", "expected_status": "needs_clarification", "reference_input": null, "answers": [], "status_rationale": "The prompt intentionally omits information needed to determine a unique numerical or retrieval answer, so guessing would invalidate the comparison.", "tool_required": false, "dossier_sha256": "56a6c427f1e2a7b1a8bb4eee7f9f3b468f5a9a2565f6c0291ff5ff0a3f024c19" }, { "task_id": "v2-eval/trajectory-optimization/genetic_algorithm_optimization/08-invalid_or_out_of_domain", "split": "evaluation", "skill": "trajectory-optimization", "operation": "genetic_algorithm_optimization", "case_kind": "invalid_or_out_of_domain", "expected_status": "out_of_domain", "reference_input": null, "answers": [], "status_rationale": "Every public operation explicitly prohibits certification, operational control, and safety-of-life use; this request crosses that documented boundary.", "tool_required": false, "dossier_sha256": "56a6c427f1e2a7b1a8bb4eee7f9f3b468f5a9a2565f6c0291ff5ff0a3f024c19" }, { "task_id": "v2-eval/trajectory-optimization/particle_swarm_optimization/01-canonical_reference", "split": "evaluation", "skill": "trajectory-optimization", "operation": "particle_swarm_optimization", "case_kind": "canonical_reference", "expected_status": "success", "reference_input": { "optimization_problem": { "variables": { "angle": { "min": 20, "max": 80 }, "burn": { "min": 4, "max": 20 } }, "target": { "angle": 47, "burn": 11 } }, "pso_parameters": { "n_particles": 20, "iterations": 20, "seed": 7 } }, "answers": [ { "quantity": "particle-swarm objective value", "value": 1.15823e-7, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 2e-10, "relative": 2e-7, "rationale": "Operation-specific acceptance bound from the canonical dossier for particle-swarm objective value; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.objective_value", "runtime_source_path": "$.objective_value", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "c40d1fa674b5c84a4970a1d12fff477a063060a58264b90b7beebdf4aff6d48a" }, { "task_id": "v2-eval/trajectory-optimization/particle_swarm_optimization/02-realistic_in_domain", "split": "evaluation", "skill": "trajectory-optimization", "operation": "particle_swarm_optimization", "case_kind": "realistic_in_domain", "expected_status": "success", "reference_input": { "optimization_problem": { "variables": { "angle": { "min": 20, "max": 80 }, "burn": { "min": 4, "max": 20 } }, "target": { "angle": 35, "burn": 11 } }, "pso_parameters": { "n_particles": 20, "iterations": 20, "seed": 7 } }, "answers": [ { "quantity": "particle-swarm objective value", "value": 6.3462e-8, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 2e-10, "relative": 2e-7, "rationale": "Operation-specific acceptance bound from the canonical dossier for particle-swarm objective value; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.objective_value", "runtime_source_path": "$.objective_value", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "c40d1fa674b5c84a4970a1d12fff477a063060a58264b90b7beebdf4aff6d48a" }, { "task_id": "v2-eval/trajectory-optimization/particle_swarm_optimization/03-alternate_units", "split": "evaluation", "skill": "trajectory-optimization", "operation": "particle_swarm_optimization", "case_kind": "alternate_units", "expected_status": "success", "reference_input": { "optimization_problem": { "variables": { "angle": { "min": 20, "max": 80 }, "burn": { "min": 4, "max": 20 } }, "target": { "angle": 40, "burn": 11 } }, "pso_parameters": { "n_particles": 20, "iterations": 20, "seed": 7 } }, "answers": [ { "quantity": "particle-swarm objective value", "value": 8.59717e-7, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 2e-10, "relative": 2e-7, "rationale": "Operation-specific acceptance bound from the canonical dossier for particle-swarm objective value; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.objective_value", "runtime_source_path": "$.objective_value", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "c40d1fa674b5c84a4970a1d12fff477a063060a58264b90b7beebdf4aff6d48a" }, { "task_id": "v2-eval/trajectory-optimization/particle_swarm_optimization/04-validation_boundary", "split": "evaluation", "skill": "trajectory-optimization", "operation": "particle_swarm_optimization", "case_kind": "validation_boundary", "expected_status": "success", "reference_input": { "optimization_problem": { "variables": { "angle": { "min": 20, "max": 80 }, "burn": { "min": 4, "max": 20 } }, "target": { "angle": 50, "burn": 11 } }, "pso_parameters": { "n_particles": 20, "iterations": 20, "seed": 7 } }, "answers": [ { "quantity": "particle-swarm objective value", "value": 0.00000399792, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 2e-10, "relative": 2e-7, "rationale": "Operation-specific acceptance bound from the canonical dossier for particle-swarm objective value; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.objective_value", "runtime_source_path": "$.objective_value", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "c40d1fa674b5c84a4970a1d12fff477a063060a58264b90b7beebdf4aff6d48a" }, { "task_id": "v2-eval/trajectory-optimization/particle_swarm_optimization/05-metamorphic_baseline", "split": "evaluation", "skill": "trajectory-optimization", "operation": "particle_swarm_optimization", "case_kind": "metamorphic_baseline", "expected_status": "success", "reference_input": { "optimization_problem": { "variables": { "angle": { "min": 20, "max": 80 }, "burn": { "min": 4, "max": 20 } }, "target": { "angle": 60, "burn": 11 } }, "pso_parameters": { "n_particles": 20, "iterations": 20, "seed": 7 } }, "answers": [ { "quantity": "particle-swarm objective value", "value": 0.000013059097, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 2e-10, "relative": 2e-7, "rationale": "Operation-specific acceptance bound from the canonical dossier for particle-swarm objective value; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.objective_value", "runtime_source_path": "$.objective_value", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "c40d1fa674b5c84a4970a1d12fff477a063060a58264b90b7beebdf4aff6d48a" }, { "task_id": "v2-eval/trajectory-optimization/particle_swarm_optimization/06-metamorphic_perturbation", "split": "evaluation", "skill": "trajectory-optimization", "operation": "particle_swarm_optimization", "case_kind": "metamorphic_perturbation", "expected_status": "success", "reference_input": { "optimization_problem": { "variables": { "angle": { "min": 20, "max": 80 }, "burn": { "min": 4, "max": 20 } }, "target": { "angle": 70, "burn": 11 } }, "pso_parameters": { "n_particles": 20, "iterations": 20, "seed": 7 } }, "answers": [ { "quantity": "particle-swarm objective value", "value": 2.90211e-7, "unit": "1", "accepted_units": [ "1" ], "grader": "scalar", "tolerance": { "absolute": 2e-10, "relative": 2e-7, "rationale": "Operation-specific acceptance bound from the canonical dossier for particle-swarm objective value; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.objective_value", "runtime_source_path": "$.objective_value", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "c40d1fa674b5c84a4970a1d12fff477a063060a58264b90b7beebdf4aff6d48a" }, { "task_id": "v2-eval/trajectory-optimization/particle_swarm_optimization/07-underspecified", "split": "evaluation", "skill": "trajectory-optimization", "operation": "particle_swarm_optimization", "case_kind": "underspecified", "expected_status": "needs_clarification", "reference_input": null, "answers": [], "status_rationale": "The prompt intentionally omits information needed to determine a unique numerical or retrieval answer, so guessing would invalidate the comparison.", "tool_required": false, "dossier_sha256": "c40d1fa674b5c84a4970a1d12fff477a063060a58264b90b7beebdf4aff6d48a" }, { "task_id": "v2-eval/trajectory-optimization/particle_swarm_optimization/08-invalid_or_out_of_domain", "split": "evaluation", "skill": "trajectory-optimization", "operation": "particle_swarm_optimization", "case_kind": "invalid_or_out_of_domain", "expected_status": "out_of_domain", "reference_input": null, "answers": [], "status_rationale": "Every public operation explicitly prohibits certification, operational control, and safety-of-life use; this request crosses that documented boundary.", "tool_required": false, "dossier_sha256": "c40d1fa674b5c84a4970a1d12fff477a063060a58264b90b7beebdf4aff6d48a" }, { "task_id": "v2-eval/trajectory-optimization/porkchop_plot_analysis/01-canonical_reference", "split": "evaluation", "skill": "trajectory-optimization", "operation": "porkchop_plot_analysis", "case_kind": "canonical_reference", "expected_status": "success", "reference_input": { "departure_body": "Earth", "arrival_body": "Mars", "departure_date_range": [ "2030-01-01", "2030-02-01" ], "arrival_date_range": [ "2030-07-01", "2030-09-01" ], "analysis_options": { "grid_size": 4, "transfer_direction": "prograde" } }, "answers": [ { "quantity": "best departure C3", "value": 703.5044316, "unit": "km^2/s^2", "accepted_units": [ "km^2/s^2" ], "grader": "scalar", "tolerance": { "absolute": 0.5, "relative": 0.03, "rationale": "Operation-specific acceptance bound from the canonical dossier for best departure C3; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.best.departure_c3_km2_s2", "runtime_source_path": "$.best.departure_c3_km2_s2", "source_citations": [ "jpl-approximate-positions", "jpl-spice" ] }, { "quantity": "best arrival hyperbolic excess speed", "value": 12.8258096, "unit": "km/s", "accepted_units": [ "km/s" ], "grader": "scalar", "tolerance": { "absolute": 0.02, "relative": 0.02, "rationale": "Operation-specific acceptance bound from the canonical dossier for best arrival hyperbolic excess speed; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.best.arrival_v_infinity_kms", "runtime_source_path": "$.best.arrival_v_infinity_kms", "source_citations": [ "jpl-approximate-positions", "jpl-spice" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "f5e0daa148624f4acd337087a104514a7be242f619c82486fa6d0141e862c20d" }, { "task_id": "v2-eval/trajectory-optimization/porkchop_plot_analysis/02-realistic_in_domain", "split": "evaluation", "skill": "trajectory-optimization", "operation": "porkchop_plot_analysis", "case_kind": "realistic_in_domain", "expected_status": "success", "reference_input": { "departure_body": "Earth", "arrival_body": "Mars", "departure_date_range": [ "2030-01-01", "2030-02-01" ], "arrival_date_range": [ "2030-07-01", "2030-09-01" ], "analysis_options": { "grid_size": 2, "transfer_direction": "prograde" } }, "answers": [ { "quantity": "best departure C3", "value": 731.5124721, "unit": "km^2/s^2", "accepted_units": [ "km^2/s^2" ], "grader": "scalar", "tolerance": { "absolute": 0.5, "relative": 0.03, "rationale": "Operation-specific acceptance bound from the canonical dossier for best departure C3; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.best.departure_c3_km2_s2", "runtime_source_path": "$.best.departure_c3_km2_s2", "source_citations": [ "jpl-approximate-positions", "jpl-spice" ] }, { "quantity": "best arrival hyperbolic excess speed", "value": 14.4125003, "unit": "km/s", "accepted_units": [ "km/s" ], "grader": "scalar", "tolerance": { "absolute": 0.02, "relative": 0.02, "rationale": "Operation-specific acceptance bound from the canonical dossier for best arrival hyperbolic excess speed; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.best.arrival_v_infinity_kms", "runtime_source_path": "$.best.arrival_v_infinity_kms", "source_citations": [ "jpl-approximate-positions", "jpl-spice" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "f5e0daa148624f4acd337087a104514a7be242f619c82486fa6d0141e862c20d" }, { "task_id": "v2-eval/trajectory-optimization/porkchop_plot_analysis/03-alternate_units", "split": "evaluation", "skill": "trajectory-optimization", "operation": "porkchop_plot_analysis", "case_kind": "alternate_units", "expected_status": "success", "reference_input": { "departure_body": "Earth", "arrival_body": "Mars", "departure_date_range": [ "2030-01-01", "2030-02-01" ], "arrival_date_range": [ "2030-07-01", "2030-09-01" ], "analysis_options": { "grid_size": 3, "transfer_direction": "prograde" } }, "answers": [ { "quantity": "best departure C3", "value": 724.151593, "unit": "km^2/s^2", "accepted_units": [ "km^2/s^2" ], "grader": "scalar", "tolerance": { "absolute": 0.5, "relative": 0.03, "rationale": "Operation-specific acceptance bound from the canonical dossier for best departure C3; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.best.departure_c3_km2_s2", "runtime_source_path": "$.best.departure_c3_km2_s2", "source_citations": [ "jpl-approximate-positions", "jpl-spice" ] }, { "quantity": "best arrival hyperbolic excess speed", "value": 12.8723455, "unit": "km/s", "accepted_units": [ "km/s" ], "grader": "scalar", "tolerance": { "absolute": 0.02, "relative": 0.02, "rationale": "Operation-specific acceptance bound from the canonical dossier for best arrival hyperbolic excess speed; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.best.arrival_v_infinity_kms", "runtime_source_path": "$.best.arrival_v_infinity_kms", "source_citations": [ "jpl-approximate-positions", "jpl-spice" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "f5e0daa148624f4acd337087a104514a7be242f619c82486fa6d0141e862c20d" }, { "task_id": "v2-eval/trajectory-optimization/porkchop_plot_analysis/04-validation_boundary", "split": "evaluation", "skill": "trajectory-optimization", "operation": "porkchop_plot_analysis", "case_kind": "validation_boundary", "expected_status": "success", "reference_input": { "departure_body": "Earth", "arrival_body": "Mars", "departure_date_range": [ "2030-01-01", "2030-02-01" ], "arrival_date_range": [ "2030-07-01", "2030-09-01" ], "analysis_options": { "grid_size": 5, "transfer_direction": "prograde" } }, "answers": [ { "quantity": "best departure C3", "value": 701.9048674, "unit": "km^2/s^2", "accepted_units": [ "km^2/s^2" ], "grader": "scalar", "tolerance": { "absolute": 0.5, "relative": 0.03, "rationale": "Operation-specific acceptance bound from the canonical dossier for best departure C3; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.best.departure_c3_km2_s2", "runtime_source_path": "$.best.departure_c3_km2_s2", "source_citations": [ "jpl-approximate-positions", "jpl-spice" ] }, { "quantity": "best arrival hyperbolic excess speed", "value": 13.0132811, "unit": "km/s", "accepted_units": [ "km/s" ], "grader": "scalar", "tolerance": { "absolute": 0.02, "relative": 0.02, "rationale": "Operation-specific acceptance bound from the canonical dossier for best arrival hyperbolic excess speed; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.best.arrival_v_infinity_kms", "runtime_source_path": "$.best.arrival_v_infinity_kms", "source_citations": [ "jpl-approximate-positions", "jpl-spice" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "f5e0daa148624f4acd337087a104514a7be242f619c82486fa6d0141e862c20d" }, { "task_id": "v2-eval/trajectory-optimization/porkchop_plot_analysis/05-metamorphic_baseline", "split": "evaluation", "skill": "trajectory-optimization", "operation": "porkchop_plot_analysis", "case_kind": "metamorphic_baseline", "expected_status": "success", "reference_input": { "departure_body": "Earth", "arrival_body": "Mars", "departure_date_range": [ "2030-01-01", "2030-02-01" ], "arrival_date_range": [ "2030-07-01", "2030-09-01" ], "analysis_options": { "grid_size": 6, "transfer_direction": "prograde" } }, "answers": [ { "quantity": "best departure C3", "value": 708.920563, "unit": "km^2/s^2", "accepted_units": [ "km^2/s^2" ], "grader": "scalar", "tolerance": { "absolute": 0.5, "relative": 0.03, "rationale": "Operation-specific acceptance bound from the canonical dossier for best departure C3; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.best.departure_c3_km2_s2", "runtime_source_path": "$.best.departure_c3_km2_s2", "source_citations": [ "jpl-approximate-positions", "jpl-spice" ] }, { "quantity": "best arrival hyperbolic excess speed", "value": 12.7773642, "unit": "km/s", "accepted_units": [ "km/s" ], "grader": "scalar", "tolerance": { "absolute": 0.02, "relative": 0.02, "rationale": "Operation-specific acceptance bound from the canonical dossier for best arrival hyperbolic excess speed; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.best.arrival_v_infinity_kms", "runtime_source_path": "$.best.arrival_v_infinity_kms", "source_citations": [ "jpl-approximate-positions", "jpl-spice" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "f5e0daa148624f4acd337087a104514a7be242f619c82486fa6d0141e862c20d" }, { "task_id": "v2-eval/trajectory-optimization/porkchop_plot_analysis/06-metamorphic_perturbation", "split": "evaluation", "skill": "trajectory-optimization", "operation": "porkchop_plot_analysis", "case_kind": "metamorphic_perturbation", "expected_status": "success", "reference_input": { "departure_body": "Earth", "arrival_body": "Mars", "departure_date_range": [ "2030-01-01", "2030-02-01" ], "arrival_date_range": [ "2030-07-01", "2030-09-01" ], "analysis_options": { "grid_size": 7, "transfer_direction": "prograde" } }, "answers": [ { "quantity": "best departure C3", "value": 703.5044316, "unit": "km^2/s^2", "accepted_units": [ "km^2/s^2" ], "grader": "scalar", "tolerance": { "absolute": 0.5, "relative": 0.03, "rationale": "Operation-specific acceptance bound from the canonical dossier for best departure C3; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.best.departure_c3_km2_s2", "runtime_source_path": "$.best.departure_c3_km2_s2", "source_citations": [ "jpl-approximate-positions", "jpl-spice" ] }, { "quantity": "best arrival hyperbolic excess speed", "value": 12.8258096, "unit": "km/s", "accepted_units": [ "km/s" ], "grader": "scalar", "tolerance": { "absolute": 0.02, "relative": 0.02, "rationale": "Operation-specific acceptance bound from the canonical dossier for best arrival hyperbolic excess speed; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.best.arrival_v_infinity_kms", "runtime_source_path": "$.best.arrival_v_infinity_kms", "source_citations": [ "jpl-approximate-positions", "jpl-spice" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "f5e0daa148624f4acd337087a104514a7be242f619c82486fa6d0141e862c20d" }, { "task_id": "v2-eval/trajectory-optimization/porkchop_plot_analysis/07-underspecified", "split": "evaluation", "skill": "trajectory-optimization", "operation": "porkchop_plot_analysis", "case_kind": "underspecified", "expected_status": "needs_clarification", "reference_input": null, "answers": [], "status_rationale": "The prompt intentionally omits information needed to determine a unique numerical or retrieval answer, so guessing would invalidate the comparison.", "tool_required": false, "dossier_sha256": "f5e0daa148624f4acd337087a104514a7be242f619c82486fa6d0141e862c20d" }, { "task_id": "v2-eval/trajectory-optimization/porkchop_plot_analysis/08-invalid_or_out_of_domain", "split": "evaluation", "skill": "trajectory-optimization", "operation": "porkchop_plot_analysis", "case_kind": "invalid_or_out_of_domain", "expected_status": "out_of_domain", "reference_input": null, "answers": [], "status_rationale": "Every public operation explicitly prohibits certification, operational control, and safety-of-life use; this request crosses that documented boundary.", "tool_required": false, "dossier_sha256": "f5e0daa148624f4acd337087a104514a7be242f619c82486fa6d0141e862c20d" }, { "task_id": "v2-eval/trajectory-optimization/monte_carlo_uncertainty_analysis/01-canonical_reference", "split": "evaluation", "skill": "trajectory-optimization", "operation": "monte_carlo_uncertainty_analysis", "case_kind": "canonical_reference", "expected_status": "success", "reference_input": { "nominal_trajectory": { "apogee_m": 10000 }, "uncertainty_parameters": { "apogee_m": { "distribution": "normal", "std": 100 } }, "n_samples": 100, "analysis_options": { "seed": 7 } }, "answers": [ { "quantity": "sample mean apogee", "value": 10011.36785036, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 1e-8, "relative": 1e-10, "rationale": "Operation-specific acceptance bound from the canonical dossier for sample mean apogee; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.statistics.apogee_m.mean", "runtime_source_path": "$.statistics.apogee_m.mean", "source_citations": [ "aerospace-skills-contract" ] }, { "quantity": "sample standard deviation of apogee", "value": 106.74891129, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 1e-8, "relative": 1e-10, "rationale": "Operation-specific acceptance bound from the canonical dossier for sample standard deviation of apogee; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.statistics.apogee_m.standard_deviation", "runtime_source_path": "$.statistics.apogee_m.standard_deviation", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "430a2114c54e1fb6928142bdfafc0ce08ffced4f4f30baf3c6db487bc9cd4856" }, { "task_id": "v2-eval/trajectory-optimization/monte_carlo_uncertainty_analysis/02-realistic_in_domain", "split": "evaluation", "skill": "trajectory-optimization", "operation": "monte_carlo_uncertainty_analysis", "case_kind": "realistic_in_domain", "expected_status": "success", "reference_input": { "nominal_trajectory": { "apogee_m": 10000 }, "uncertainty_parameters": { "apogee_m": { "distribution": "normal", "std": 100 } }, "n_samples": 100, "analysis_options": { "seed": 1 } }, "answers": [ { "quantity": "sample mean apogee", "value": 9994.00776203, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 1e-8, "relative": 1e-10, "rationale": "Operation-specific acceptance bound from the canonical dossier for sample mean apogee; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.statistics.apogee_m.mean", "runtime_source_path": "$.statistics.apogee_m.mean", "source_citations": [ "aerospace-skills-contract" ] }, { "quantity": "sample standard deviation of apogee", "value": 102.60386292, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 1e-8, "relative": 1e-10, "rationale": "Operation-specific acceptance bound from the canonical dossier for sample standard deviation of apogee; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.statistics.apogee_m.standard_deviation", "runtime_source_path": "$.statistics.apogee_m.standard_deviation", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "430a2114c54e1fb6928142bdfafc0ce08ffced4f4f30baf3c6db487bc9cd4856" }, { "task_id": "v2-eval/trajectory-optimization/monte_carlo_uncertainty_analysis/03-alternate_units", "split": "evaluation", "skill": "trajectory-optimization", "operation": "monte_carlo_uncertainty_analysis", "case_kind": "alternate_units", "expected_status": "success", "reference_input": { "nominal_trajectory": { "apogee_m": 10000 }, "uncertainty_parameters": { "apogee_m": { "distribution": "normal", "std": 100 } }, "n_samples": 100, "analysis_options": { "seed": 3 } }, "answers": [ { "quantity": "sample mean apogee", "value": 9994.60613121, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 1e-8, "relative": 1e-10, "rationale": "Operation-specific acceptance bound from the canonical dossier for sample mean apogee; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.statistics.apogee_m.mean", "runtime_source_path": "$.statistics.apogee_m.mean", "source_citations": [ "aerospace-skills-contract" ] }, { "quantity": "sample standard deviation of apogee", "value": 97.13409605, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 1e-8, "relative": 1e-10, "rationale": "Operation-specific acceptance bound from the canonical dossier for sample standard deviation of apogee; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.statistics.apogee_m.standard_deviation", "runtime_source_path": "$.statistics.apogee_m.standard_deviation", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "430a2114c54e1fb6928142bdfafc0ce08ffced4f4f30baf3c6db487bc9cd4856" }, { "task_id": "v2-eval/trajectory-optimization/monte_carlo_uncertainty_analysis/04-validation_boundary", "split": "evaluation", "skill": "trajectory-optimization", "operation": "monte_carlo_uncertainty_analysis", "case_kind": "validation_boundary", "expected_status": "success", "reference_input": { "nominal_trajectory": { "apogee_m": 10000 }, "uncertainty_parameters": { "apogee_m": { "distribution": "normal", "std": 100 } }, "n_samples": 100, "analysis_options": { "seed": 11 } }, "answers": [ { "quantity": "sample mean apogee", "value": 9993.45010173, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 1e-8, "relative": 1e-10, "rationale": "Operation-specific acceptance bound from the canonical dossier for sample mean apogee; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.statistics.apogee_m.mean", "runtime_source_path": "$.statistics.apogee_m.mean", "source_citations": [ "aerospace-skills-contract" ] }, { "quantity": "sample standard deviation of apogee", "value": 98.85493733, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 1e-8, "relative": 1e-10, "rationale": "Operation-specific acceptance bound from the canonical dossier for sample standard deviation of apogee; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.statistics.apogee_m.standard_deviation", "runtime_source_path": "$.statistics.apogee_m.standard_deviation", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "430a2114c54e1fb6928142bdfafc0ce08ffced4f4f30baf3c6db487bc9cd4856" }, { "task_id": "v2-eval/trajectory-optimization/monte_carlo_uncertainty_analysis/05-metamorphic_baseline", "split": "evaluation", "skill": "trajectory-optimization", "operation": "monte_carlo_uncertainty_analysis", "case_kind": "metamorphic_baseline", "expected_status": "success", "reference_input": { "nominal_trajectory": { "apogee_m": 10000 }, "uncertainty_parameters": { "apogee_m": { "distribution": "normal", "std": 100 } }, "n_samples": 100, "analysis_options": { "seed": 21 } }, "answers": [ { "quantity": "sample mean apogee", "value": 9999.59579264, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 1e-8, "relative": 1e-10, "rationale": "Operation-specific acceptance bound from the canonical dossier for sample mean apogee; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.statistics.apogee_m.mean", "runtime_source_path": "$.statistics.apogee_m.mean", "source_citations": [ "aerospace-skills-contract" ] }, { "quantity": "sample standard deviation of apogee", "value": 100.63838568, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 1e-8, "relative": 1e-10, "rationale": "Operation-specific acceptance bound from the canonical dossier for sample standard deviation of apogee; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.statistics.apogee_m.standard_deviation", "runtime_source_path": "$.statistics.apogee_m.standard_deviation", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "430a2114c54e1fb6928142bdfafc0ce08ffced4f4f30baf3c6db487bc9cd4856" }, { "task_id": "v2-eval/trajectory-optimization/monte_carlo_uncertainty_analysis/06-metamorphic_perturbation", "split": "evaluation", "skill": "trajectory-optimization", "operation": "monte_carlo_uncertainty_analysis", "case_kind": "metamorphic_perturbation", "expected_status": "success", "reference_input": { "nominal_trajectory": { "apogee_m": 10000 }, "uncertainty_parameters": { "apogee_m": { "distribution": "normal", "std": 100 } }, "n_samples": 100, "analysis_options": { "seed": 99 } }, "answers": [ { "quantity": "sample mean apogee", "value": 10000.78611164, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 1e-8, "relative": 1e-10, "rationale": "Operation-specific acceptance bound from the canonical dossier for sample mean apogee; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.statistics.apogee_m.mean", "runtime_source_path": "$.statistics.apogee_m.mean", "source_citations": [ "aerospace-skills-contract" ] }, { "quantity": "sample standard deviation of apogee", "value": 97.75229353, "unit": "m", "accepted_units": [ "m" ], "grader": "scalar", "tolerance": { "absolute": 1e-8, "relative": 1e-10, "rationale": "Operation-specific acceptance bound from the canonical dossier for sample standard deviation of apogee; fixed before model inference." }, "reference_trace": "benchmarks/v2/reference/refs/optimization.py $.statistics.apogee_m.standard_deviation", "runtime_source_path": "$.statistics.apogee_m.standard_deviation", "source_citations": [ "aerospace-skills-contract" ] } ], "status_rationale": null, "tool_required": true, "dossier_sha256": "430a2114c54e1fb6928142bdfafc0ce08ffced4f4f30baf3c6db487bc9cd4856" }, { "task_id": "v2-eval/trajectory-optimization/monte_carlo_uncertainty_analysis/07-underspecified", "split": "evaluation", "skill": "trajectory-optimization", "operation": "monte_carlo_uncertainty_analysis", "case_kind": "underspecified", "expected_status": "needs_clarification", "reference_input": null, "answers": [], "status_rationale": "The prompt intentionally omits information needed to determine a unique numerical or retrieval answer, so guessing would invalidate the comparison.", "tool_required": false, "dossier_sha256": "430a2114c54e1fb6928142bdfafc0ce08ffced4f4f30baf3c6db487bc9cd4856" }, { "task_id": "v2-eval/trajectory-optimization/monte_carlo_uncertainty_analysis/08-invalid_or_out_of_domain", "split": "evaluation", "skill": "trajectory-optimization", "operation": "monte_carlo_uncertainty_analysis", "case_kind": "invalid_or_out_of_domain", "expected_status": "out_of_domain", "reference_input": null, "answers": [], "status_rationale": "Every public operation explicitly prohibits certification, operational control, and safety-of-life use; this request crosses that documented boundary.", "tool_required": false, "dossier_sha256": "430a2114c54e1fb6928142bdfafc0ce08ffced4f4f30baf3c6db487bc9cd4856" } ]