{ "generatedBy": "bench.mjs", "startedAt": "2026-08-24T07:33:13.789Z", "finishedAt": "2026-08-24T07:33:35.078Z", "machine": { "cpu": "Apple M1", "os": "Darwin 26.3", "arch": "arm64", "node": "v22.18.0", "python": "Python 3.13.5" }, "metricCount": 93, "metrics": [ { "metric": "experiment_model", "value": "gpt-5.4-mini" }, { "metric": "model_minor_version", "value": 5.4 }, { "metric": "experiment_prompt", "value": "Check migration-check using the supplied lookup tool. Return the item, availability, quantity, and recommendation." }, { "metric": "test_day_of_month", "value": 24, "unit": "day" }, { "metric": "export_shape", "value": "Python Agents SDK Agent plus one function_tool" }, { "metric": "repetitions_per_cell", "value": 3, "unit": "runs" }, { "metric": "timeout", "value": 90, "unit": "seconds" }, { "metric": "fixture_sha256", "value": "dbfa1b63b098b3364776c61b3c29125a1e99a67d478b26f49fc2837d9ac56bb7" }, { "metric": "openai_agents_version", "value": "0.22.0" }, { "metric": "openai_package_version", "value": "3.3.1" }, { "metric": "python_version", "value": "3.13.5" }, { "metric": "sdk_default_run_1_wall_ms", "value": 4312.76, "unit": "ms", "note": "one live Agents SDK run" }, { "metric": "sdk_default_run_1_success", "value": 1, "note": "one live Agents SDK run" }, { "metric": "sdk_default_run_1_structured", "value": 1, "note": "one live Agents SDK run" }, { "metric": "sdk_default_run_1_correct", "value": 1, "note": "one live Agents SDK run" }, { "metric": "sdk_default_run_1_tool_calls", "value": 1, "note": "one live Agents SDK run" }, { "metric": "sdk_default_run_1_requests", "value": 2, "note": "one live Agents SDK run" }, { "metric": "sdk_default_run_1_input_tokens", "value": 432, "note": "one live Agents SDK run" }, { "metric": "sdk_default_run_1_output_tokens", "value": 48, "note": "one live Agents SDK run" }, { "metric": "sdk_default_run_1_total_tokens", "value": 480, "note": "one live Agents SDK run" }, { "metric": "sdk_default_run_2_wall_ms", "value": 2432.11, "unit": "ms", "note": "one live Agents SDK run" }, { "metric": "sdk_default_run_2_success", "value": 1, "note": "one live Agents SDK run" }, { "metric": "sdk_default_run_2_structured", "value": 1, "note": "one live Agents SDK run" }, { "metric": "sdk_default_run_2_correct", "value": 1, "note": "one live Agents SDK run" }, { "metric": "sdk_default_run_2_tool_calls", "value": 1, "note": "one live Agents SDK run" }, { "metric": "sdk_default_run_2_requests", "value": 2, "note": "one live Agents SDK run" }, { "metric": "sdk_default_run_2_input_tokens", "value": 432, "note": "one live Agents SDK run" }, { "metric": "sdk_default_run_2_output_tokens", "value": 48, "note": "one live Agents SDK run" }, { "metric": "sdk_default_run_2_total_tokens", "value": 480, "note": "one live Agents SDK run" }, { "metric": "sdk_default_run_3_wall_ms", "value": 2186.44, "unit": "ms", "note": "one live Agents SDK run" }, { "metric": "sdk_default_run_3_success", "value": 1, "note": "one live Agents SDK run" }, { "metric": "sdk_default_run_3_structured", "value": 1, "note": "one live Agents SDK run" }, { "metric": "sdk_default_run_3_correct", "value": 1, "note": "one live Agents SDK run" }, { "metric": "sdk_default_run_3_tool_calls", "value": 1, "note": "one live Agents SDK run" }, { "metric": "sdk_default_run_3_requests", "value": 2, "note": "one live Agents SDK run" }, { "metric": "sdk_default_run_3_input_tokens", "value": 432, "note": "one live Agents SDK run" }, { "metric": "sdk_default_run_3_output_tokens", "value": 48, "note": "one live Agents SDK run" }, { "metric": "sdk_default_run_3_total_tokens", "value": 480, "note": "one live Agents SDK run" }, { "metric": "sdk_default_runs", "value": 3, "unit": "runs" }, { "metric": "sdk_default_median_wall_ms", "value": 2432.11, "unit": "ms", "n": 3, "note": "median across three repetitions" }, { "metric": "sdk_default_median_success", "value": 1, "n": 3, "note": "median across three repetitions" }, { "metric": "sdk_default_median_structured", "value": 1, "n": 3, "note": "median across three repetitions" }, { "metric": "sdk_default_median_correct", "value": 1, "n": 3, "note": "median across three repetitions" }, { "metric": "sdk_default_median_tool_calls", "value": 1, "n": 3, "note": "median across three repetitions" }, { "metric": "sdk_default_median_requests", "value": 2, "n": 3, "note": "median across three repetitions" }, { "metric": "sdk_default_median_input_tokens", "value": 432, "n": 3, "note": "median across three repetitions" }, { "metric": "sdk_default_median_output_tokens", "value": 48, "n": 3, "note": "median across three repetitions" }, { "metric": "sdk_default_median_total_tokens", "value": 480, "n": 3, "note": "median across three repetitions" }, { "metric": "sdk_parity_run_1_wall_ms", "value": 2079.12, "unit": "ms", "note": "one live Agents SDK run" }, { "metric": "sdk_parity_run_1_success", "value": 1, "note": "one live Agents SDK run" }, { "metric": "sdk_parity_run_1_structured", "value": 1, "note": "one live Agents SDK run" }, { "metric": "sdk_parity_run_1_correct", "value": 1, "note": "one live Agents SDK run" }, { "metric": "sdk_parity_run_1_tool_calls", "value": 1, "note": "one live Agents SDK run" }, { "metric": "sdk_parity_run_1_requests", "value": 2, "note": "one live Agents SDK run" }, { "metric": "sdk_parity_run_1_input_tokens", "value": 432, "note": "one live Agents SDK run" }, { "metric": "sdk_parity_run_1_output_tokens", "value": 48, "note": "one live Agents SDK run" }, { "metric": "sdk_parity_run_1_total_tokens", "value": 480, "note": "one live Agents SDK run" }, { "metric": "sdk_parity_run_2_wall_ms", "value": 1622.5, "unit": "ms", "note": "one live Agents SDK run" }, { "metric": "sdk_parity_run_2_success", "value": 0, "note": "one live Agents SDK run" }, { "metric": "sdk_parity_run_2_structured", "value": 0, "note": "one live Agents SDK run" }, { "metric": "sdk_parity_run_2_correct", "value": 0, "note": "one live Agents SDK run" }, { "metric": "sdk_parity_run_2_tool_calls", "value": 0, "note": "one live Agents SDK run" }, { "metric": "sdk_parity_run_2_requests", "value": 0, "note": "one live Agents SDK run" }, { "metric": "sdk_parity_run_2_input_tokens", "value": 0, "note": "one live Agents SDK run" }, { "metric": "sdk_parity_run_2_output_tokens", "value": 0, "note": "one live Agents SDK run" }, { "metric": "sdk_parity_run_2_total_tokens", "value": 0, "note": "one live Agents SDK run" }, { "metric": "sdk_parity_run_2_error", "value": "RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-5.4-mini in organization org-YrVJwQEw0ZLK9oelw1lbpE0T on requests per day (RPD): Limit 50, Used 50, Requested 1. Please try again in 28m48s. Visit https://platform.openai.c", "note": "failure class and message" }, { "metric": "sdk_parity_run_3_wall_ms", "value": 682.01, "unit": "ms", "note": "one live Agents SDK run" }, { "metric": "sdk_parity_run_3_success", "value": 0, "note": "one live Agents SDK run" }, { "metric": "sdk_parity_run_3_structured", "value": 0, "note": "one live Agents SDK run" }, { "metric": "sdk_parity_run_3_correct", "value": 0, "note": "one live Agents SDK run" }, { "metric": "sdk_parity_run_3_tool_calls", "value": 0, "note": "one live Agents SDK run" }, { "metric": "sdk_parity_run_3_requests", "value": 0, "note": "one live Agents SDK run" }, { "metric": "sdk_parity_run_3_input_tokens", "value": 0, "note": "one live Agents SDK run" }, { "metric": "sdk_parity_run_3_output_tokens", "value": 0, "note": "one live Agents SDK run" }, { "metric": "sdk_parity_run_3_total_tokens", "value": 0, "note": "one live Agents SDK run" }, { "metric": "sdk_parity_run_3_error", "value": "RateLimitError: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-5.4-mini in organization org-YrVJwQEw0ZLK9oelw1lbpE0T on requests per day (RPD): Limit 50, Used 50, Requested 1. Please try again in 28m48s. Visit https://platform.openai.c", "note": "failure class and message" }, { "metric": "sdk_parity_runs", "value": 3, "unit": "runs" }, { "metric": "sdk_parity_median_wall_ms", "value": 1622.5, "unit": "ms", "n": 3, "note": "median across three repetitions" }, { "metric": "sdk_parity_median_success", "value": 0, "n": 3, "note": "median across three repetitions" }, { "metric": "sdk_parity_median_structured", "value": 0, "n": 3, "note": "median across three repetitions" }, { "metric": "sdk_parity_median_correct", "value": 0, "n": 3, "note": "median across three repetitions" }, { "metric": "sdk_parity_median_tool_calls", "value": 0, "n": 3, "note": "median across three repetitions" }, { "metric": "sdk_parity_median_requests", "value": 0, "n": 3, "note": "median across three repetitions" }, { "metric": "sdk_parity_median_input_tokens", "value": 0, "n": 3, "note": "median across three repetitions" }, { "metric": "sdk_parity_median_output_tokens", "value": 0, "n": 3, "note": "median across three repetitions" }, { "metric": "sdk_parity_median_total_tokens", "value": 0, "n": 3, "note": "median across three repetitions" }, { "metric": "missing_connection_compatibility", "value": 1, "note": "export cannot supply external credential: AGENT_BUILDER_CONNECTED_APP_TOKEN" }, { "metric": "missing_connection_error", "value": "KeyError" }, { "metric": "all_runs", "value": 6, "unit": "runs" }, { "metric": "all_successful_runs", "value": 4, "unit": "runs" }, { "metric": "all_structured_outputs", "value": 4, "unit": "runs" }, { "metric": "all_correct_outputs", "value": 4, "unit": "runs" } ], "log": [ "Using CPython 3.13.5 interpreter at: /opt/homebrew/opt/python@3.13/bin/python3.13", "Creating virtual environment at: $TMPDIR/tmp.CsE6ll8EjB/venv", "Activate with: source $TMPDIR/tmp.CsE6ll8EjB/venv/bin/activate", "Error getting response", "Error getting response", "" ] }