{
  "generatedBy": "bench.mjs",
  "startedAt": "2026-09-30T17:34:25.793Z",
  "finishedAt": "2026-09-30T17:34:41.177Z",
  "machine": {
    "cpu": "Apple M1",
    "os": "Darwin 26.3",
    "arch": "arm64",
    "node": "v22.18.0",
    "python": "Python 3.13.5"
  },
  "metricCount": 91,
  "metrics": [
    {
      "metric": "release_asset",
      "value": "llama-b11146-bin-macos-arm64.tar.gz",
      "unit": "text",
      "note": "GitHub release asset for stable v0.5.0 (nightly-tag.txt points at b11146)"
    },
    {
      "metric": "tarball_mb",
      "value": 10.7,
      "unit": "MB",
      "note": "release tarball size"
    },
    {
      "metric": "unpacked_mb",
      "value": 27,
      "unit": "MB",
      "note": "unpacked release directory"
    },
    {
      "metric": "binary_count",
      "value": 22,
      "unit": "count",
      "note": "llama* executables in the macOS release"
    },
    {
      "metric": "model_file_mb",
      "value": 468.6,
      "unit": "MB",
      "note": "qwen2.5-0.5b-instruct-q4_k_m.gguf"
    },
    {
      "metric": "server_version",
      "value": "version: 0.5.0-dev (build 11146, commit 7fe450e19)",
      "unit": "text",
      "note": "llama-server --version"
    },
    {
      "metric": "build_number",
      "value": 11146,
      "unit": "build",
      "note": "numeric build from --version"
    },
    {
      "metric": "llama_commands",
      "value": "serve cli download version licenses help",
      "unit": "text",
      "note": "commands listed by ./llama --help"
    },
    {
      "metric": "serve_help_identical",
      "value": "yes",
      "unit": "bool",
      "note": "llama serve --help vs llama-server --help, log lines stripped"
    },
    {
      "metric": "help_lines",
      "value": 731,
      "unit": "lines",
      "note": "llama-server --help output"
    },
    {
      "metric": "help_flag_rows",
      "value": 259,
      "unit": "count",
      "note": "option rows starting with a dash in llama-server --help"
    },
    {
      "metric": "help_env_vars",
      "value": 148,
      "unit": "count",
      "note": "options that print an (env: ...) variable"
    },
    {
      "metric": "help_ctx",
      "value": "-c, --ctx-size N size of the prompt context (default: 0, 0 = loaded from model)  (env: LLAMA_ARG_CTX_SIZE) -n, --predict, --n-predict N number of tokens to predict (default: -1, -1 = infinity)  (env: LLAMA_ARG_N_PREDICT) -b, --batch-size N logical maximum batch size (default: 2048)",
      "unit": "text",
      "note": "help text"
    },
    {
      "metric": "help_ngl",
      "value": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number,  'auto', or 'all' (default: auto)  (env: LLAMA_ARG_N_GPU_LAYERS) -sm, --split-mode {none,layer,row,tensor}  how to split the model across multiple GPUs, one of:",
      "unit": "text",
      "note": "help text"
    },
    {
      "metric": "help_np",
      "value": "-np, --parallel N number of server slots (default: -1, -1 = auto)  (env: LLAMA_ARG_N_PARALLEL) -cb, --cont-batching, -nocb, --no-cont-batching  whether to enable continuous batching (a.k.a dynamic batching)  (default: enabled)",
      "unit": "text",
      "note": "help text"
    },
    {
      "metric": "help_host",
      "value": "--host HOST IP addresses to listen on, comma-separated, or UNIX socket paths  ending in .sock; with multiple TCP addresses, :: binds IPv6 only;  overlapping addresses result in undefined behavior (default:  127.0.0.1)  (env: LLAMA_ARG_HOST)",
      "unit": "text",
      "note": "help text"
    },
    {
      "metric": "help_port",
      "value": "--port PORT port to listen (default: 8080)  (env: LLAMA_ARG_PORT) --reuse-port allow multiple sockets to bind to the same port (default: disabled)  (env: LLAMA_ARG_REUSE_PORT) --path PATH path to serve static files from (default: )",
      "unit": "text",
      "note": "help text"
    },
    {
      "metric": "help_api_key",
      "value": "--api-key KEY API key to use for authentication, multiple keys can be provided as a  comma-separated list (default: none)  (env: LLAMA_API_KEY) --api-key-file FNAME path to file containing API keys, one per line; lines starting with a  hash are treated as comments (default: none)",
      "unit": "text",
      "note": "help text"
    },
    {
      "metric": "help_jinja",
      "value": "--jinja, --no-jinja whether to use jinja template engine for chat (default: enabled)  (env: LLAMA_ARG_JINJA) --reasoning-format FORMAT controls whether thought tags are allowed and/or extracted from the  response, and in which format they're returned; one of:  - none: leaves thoughts unparsed in `message.content`",
      "unit": "text",
      "note": "help text"
    },
    {
      "metric": "help_ctk",
      "value": "-ctk, --cache-type-k TYPE KV cache data type for K  allowed values: f32, f16, bf16, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1  (default: f16)  (env: LLAMA_ARG_CACHE_TYPE_K) -ctv, --cache-type-v TYPE KV cache data type for V",
      "unit": "text",
      "note": "help text"
    },
    {
      "metric": "help_hf",
      "value": "-hf, -hfr, --hf-repo <user>/<model>[:quant]  Hugging Face model repository; quant is optional, case-insensitive,  default to Q4_K_M, or falls back to the first file in the repo if  Q4_K_M doesn't exist.  mmproj is also downloaded automatically if available. to disable, add -- -hff, --hf-file FILE Hugging Face model file. If specified, it will override the quant in  --hf-repo (default: unused)  (en",
      "unit": "text",
      "note": "help text"
    },
    {
      "metric": "help_webui",
      "value": "--ui, --webui, --no-ui, --no-webui whether to enable the Web UI (default: enabled)  (env: LLAMA_ARG_UI) --embedding, --embeddings restrict to only support embedding use case; use only with dedicated  embedding models (default: disabled)  (env: LLAMA_ARG_EMBEDDINGS)",
      "unit": "text",
      "note": "help text"
    },
    {
      "metric": "help_metrics",
      "value": "--metrics enable prometheus compatible metrics endpoint (default: disabled)  (env: LLAMA_ARG_ENDPOINT_METRICS) --props enable changing global properties via POST /props (default: disabled)  (env: LLAMA_ARG_ENDPOINT_PROPS) --slots, --no-slots expose slots monitoring endpoint (default: enabled)",
      "unit": "text",
      "note": "help text"
    },
    {
      "metric": "help_slots",
      "value": "--slots, --no-slots expose slots monitoring endpoint (default: enabled)  (env: LLAMA_ARG_ENDPOINT_SLOTS) --slot-save-path PATH path to save slot kv cache (default: disabled) --media-path PATH directory for loading local media files; files can be accessed via  file:// URLs using relative paths (default: disabled)",
      "unit": "text",
      "note": "help text"
    },
    {
      "metric": "help_fa",
      "value": "-fa, --flash-attn [on|off|auto] set Flash Attention use ('on', 'off', or 'auto', default: 'auto')  (env: LLAMA_ARG_FLASH_ATTN) --perf, --no-perf whether to enable internal libllama performance timings (default:  false)  (env: LLAMA_ARG_PERF)",
      "unit": "text",
      "note": "help text"
    },
    {
      "metric": "help_fit",
      "value": "-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory ('on' or  'off', default: 'on')  (env: LLAMA_ARG_FIT) -fitt, --fit-target MiB0,MiB1,MiB2,...  target margin per device for --fit, comma-separated list of values,",
      "unit": "text",
      "note": "help text"
    },
    {
      "metric": "help_alias",
      "value": "-a, --alias STRING set model name aliases, comma-separated (to be used by API)  (env: LLAMA_ARG_ALIAS) --tags STRING set model tags, comma-separated (informational, not used for routing)  (env: LLAMA_ARG_TAGS) --embd-normalize N normalisation for embeddings (default: 2) (-1=none, 0=max absolute",
      "unit": "text",
      "note": "help text"
    },
    {
      "metric": "help_timeout",
      "value": "-to, --timeout N server read/write timeout in seconds (default: 3600)  (env: LLAMA_ARG_TIMEOUT) --sse-ping-interval N server SSE ping interval in seconds (-1 = disabled, default: 30)  (env: LLAMA_ARG_SSE_PING_INTERVAL) --threads-http N number of threads used to process HTTP requests (default: -1)",
      "unit": "text",
      "note": "help text"
    },
    {
      "metric": "default_port",
      "value": 8080,
      "unit": "port",
      "note": "--port default from help"
    },
    {
      "metric": "s1_port",
      "value": 8089,
      "unit": "port",
      "note": "port used for server 1"
    },
    {
      "metric": "s1_command",
      "value": "llama-server -m qwen2.5-0.5b-instruct-q4_k_m.gguf -a qwen2.5-0.5b -c 4096 -ngl all -np 2 --host 127.0.0.1 --port 8089 --api-key <key> --jinja -ctk q8_0 -ctv q8_0 --metrics",
      "unit": "text",
      "note": "server 1 launch command"
    },
    {
      "metric": "s1_ready_ms",
      "value": 793,
      "unit": "ms",
      "note": "launch to first 200 on /health, n=1, Apple M1 8GB"
    },
    {
      "metric": "s1_rss_mb",
      "value": 610,
      "unit": "MB",
      "note": "resident memory of llama-server after load"
    },
    {
      "metric": "s1_slot_log",
      "value": "n_slots = 2, n_ctx_slot = 2048, kv_unified = 'false'",
      "unit": "text",
      "note": "server log line"
    },
    {
      "metric": "s1_listen_log",
      "value": "listening on http://127.0.0.1:8089",
      "unit": "text",
      "note": "server log line"
    },
    {
      "metric": "s1_listen_socket",
      "value": "127.0.0.1:8089",
      "unit": "text",
      "note": "lsof LISTEN address for server 1"
    },
    {
      "metric": "health_nokey",
      "value": "{\"status\":\"ok\"} HTTP 200",
      "unit": "text",
      "note": "GET /health without a key"
    },
    {
      "metric": "v1_health_nokey_code",
      "value": 200,
      "unit": "http",
      "note": "GET /v1/health without a key"
    },
    {
      "metric": "models_nokey",
      "value": "{\"error\":{\"message\":\"Invalid API Key\",\"type\":\"authentication_error\",\"code\":401}} HTTP 401",
      "unit": "text",
      "note": "GET /v1/models without a key"
    },
    {
      "metric": "models_wrongkey_code",
      "value": 401,
      "unit": "http",
      "note": "GET /v1/models with a wrong key"
    },
    {
      "metric": "models_key_code",
      "value": 200,
      "unit": "http",
      "note": "GET /v1/models with the key"
    },
    {
      "metric": "models_id",
      "value": "qwen2.5-0.5b",
      "unit": "text",
      "note": "model id reported (set by -a)"
    },
    {
      "metric": "models_n_ctx_train",
      "value": 32768,
      "unit": "tokens",
      "note": "model's trained context"
    },
    {
      "metric": "models_n_ctx",
      "value": 2048,
      "unit": "tokens",
      "note": "n_ctx reported in /v1/models meta"
    },
    {
      "metric": "models_owned_by",
      "value": "llamacpp",
      "unit": "text"
    },
    {
      "metric": "props_nokey_code",
      "value": 401,
      "unit": "http",
      "note": "GET /props without a key"
    },
    {
      "metric": "slots_nokey_code",
      "value": 401,
      "unit": "http",
      "note": "GET /slots without a key"
    },
    {
      "metric": "metrics_nokey_code",
      "value": 401,
      "unit": "http",
      "note": "GET /metrics without a key"
    },
    {
      "metric": "webui_nokey_code",
      "value": 200,
      "unit": "http",
      "note": "GET / (web UI) without a key, browser-like headers"
    },
    {
      "metric": "webui_content_type",
      "value": "text/html; charset=utf-8",
      "unit": "text",
      "note": "web UI content type"
    },
    {
      "metric": "webui_no_gzip_body",
      "value": "Error: gzip is not supported by this browser",
      "unit": "text",
      "note": "GET / with no Accept-Encoding header"
    },
    {
      "metric": "chat_nokey",
      "value": "{\"error\":{\"message\":\"Invalid API Key\",\"type\":\"authentication_error\",\"code\":401}} HTTP 401",
      "unit": "text",
      "note": "POST /v1/chat/completions without a key"
    },
    {
      "metric": "anthropic_header_code",
      "value": 200,
      "unit": "http",
      "note": "POST /v1/messages with x-api-key header"
    },
    {
      "metric": "messages_nokey_code",
      "value": 401,
      "unit": "http",
      "note": "POST /v1/messages without a key"
    },
    {
      "metric": "completion_nokey_code",
      "value": 401,
      "unit": "http",
      "note": "POST /completion without a key"
    },
    {
      "metric": "health_key_code",
      "value": 200,
      "unit": "http",
      "note": "GET /health with the key"
    },
    {
      "metric": "webui_key_code",
      "value": 200,
      "unit": "http",
      "note": "GET / (web UI) with the key, browser-like headers"
    },
    {
      "metric": "chat_key_code",
      "value": 200,
      "unit": "http",
      "note": "POST /v1/chat/completions with the key"
    },
    {
      "metric": "completion_key_code",
      "value": 200,
      "unit": "http",
      "note": "POST /completion with the key"
    },
    {
      "metric": "props_key_code",
      "value": 200,
      "unit": "http",
      "note": "GET /props with the key"
    },
    {
      "metric": "slots_key_code",
      "value": 200,
      "unit": "http",
      "note": "GET /slots with the key"
    },
    {
      "metric": "chat_key_content",
      "value": "Paris",
      "unit": "text",
      "note": "POST /v1/chat/completions with the key, temperature 0"
    },
    {
      "metric": "chat_fingerprint",
      "value": "b11146-7fe450e19",
      "unit": "text",
      "note": "system_fingerprint in chat response"
    },
    {
      "metric": "chat_prompt_tokens",
      "value": 38,
      "unit": "tokens",
      "note": "usage.prompt_tokens"
    },
    {
      "metric": "chat_has_timings",
      "value": "yes",
      "unit": "bool",
      "note": "llama.cpp timings object in an OpenAI response"
    },
    {
      "metric": "tool_finish_reason",
      "value": "tool_calls",
      "unit": "text",
      "note": "tool-calling request with --jinja"
    },
    {
      "metric": "tool_call",
      "value": "{\"name\": \"get_weather\", \"arguments\": \"{\\\"city\\\": \\\"Paris\\\"}\"}",
      "unit": "json",
      "note": "first tool call returned"
    },
    {
      "metric": "decode_tps_median",
      "value": 94.2,
      "unit": "tok/s",
      "note": "median predicted_per_second over 5 /completion runs, n_predict 64, Apple M1, all layers on Metal"
    },
    {
      "metric": "completion_content",
      "value": "Paris. It is the largest city in",
      "unit": "text",
      "note": "POST /completion native endpoint"
    },
    {
      "metric": "completion_keys",
      "value": "id_slot tokens_predicted tokens_evaluated stop_type timings",
      "unit": "text",
      "note": "native-only fields present"
    },
    {
      "metric": "concurrent_slot_ids",
      "value": "[0, 1]",
      "unit": "text",
      "note": "id_slot of two simultaneous /completion requests with -np 2"
    },
    {
      "metric": "slots_count",
      "value": 2,
      "unit": "count",
      "note": "GET /slots entries with -np 2"
    },
    {
      "metric": "slot_n_ctx",
      "value": 2048,
      "unit": "tokens",
      "note": "per-slot n_ctx with -c 4096 -np 2"
    },
    {
      "metric": "props_total_slots",
      "value": 2,
      "unit": "count",
      "note": "/props total_slots"
    },
    {
      "metric": "props_build_info",
      "value": "b11146-7fe450e19",
      "unit": "text",
      "note": "/props build_info"
    },
    {
      "metric": "props_ctx",
      "value": 2048,
      "unit": "tokens",
      "note": "/props default_generation_settings.n_ctx"
    },
    {
      "metric": "metrics_key_code",
      "value": 200,
      "unit": "http",
      "note": "GET /metrics with the key (--metrics on)"
    },
    {
      "metric": "metrics_names",
      "value": "llamacpp:prompt_tokens_total llamacpp:prompt_tokens_cached_total llamacpp:prompt_seconds_total llamacpp:tokens_predicted_total llamacpp:tokens_predicted_seconds_total llamacpp:n_decode_total llamacpp:n_tokens_max llamacpp:spec_decode_num_draft_tokens_total llamacpp:spec_decode_num_accepted_tokens_total llamacpp:spec_decode_num_drafts_total llamacpp:prompt_tokens_seconds llamacpp:predicted_tokens_seconds llamacpp:requests_processing llamacpp:requests_deferred llamacpp:n_busy_slots_per_decode",
      "unit": "text",
      "note": "metric names exposed"
    },
    {
      "metric": "metrics_count",
      "value": 15,
      "unit": "count",
      "note": "metric series in /metrics"
    },
    {
      "metric": "openai_sdk_version",
      "value": "3.22.1",
      "unit": "text",
      "note": "openai package version installed in venv"
    },
    {
      "metric": "openai_sdk_reply",
      "value": "waiting",
      "unit": "text",
      "note": "chat.completions.create via OpenAI SDK with base_url=/v1"
    },
    {
      "metric": "s2_command",
      "value": "llama serve -m qwen2.5-0.5b-instruct-q4_k_m.gguf --port 8090 --no-slots",
      "unit": "text",
      "note": "server 2 launch command"
    },
    {
      "metric": "s2_ready_ms",
      "value": 1030,
      "unit": "ms",
      "note": "launch to first 200 on /health, n=1"
    },
    {
      "metric": "s2_slot_log",
      "value": "n_slots = 4, n_ctx_slot = 32768, kv_unified = 'true'",
      "unit": "text",
      "note": "default slots and per-slot context when -c and -np are unset"
    },
    {
      "metric": "s2_listen_socket",
      "value": "127.0.0.1:8090",
      "unit": "text",
      "note": "default bind address"
    },
    {
      "metric": "s2_models_nokey_code",
      "value": 200,
      "unit": "http",
      "note": "GET /v1/models with no --api-key set"
    },
    {
      "metric": "s2_model_id",
      "value": "/tmp/aw-llama-server/qwen2.5-0.5b-instruct-q4_k_m.gguf",
      "unit": "text",
      "note": "model id without -a"
    },
    {
      "metric": "s2_slots_code",
      "value": 501,
      "unit": "http",
      "note": "GET /slots with --no-slots"
    },
    {
      "metric": "s2_metrics_code",
      "value": 501,
      "unit": "http",
      "note": "GET /metrics without --metrics"
    },
    {
      "metric": "s2_props_ctx",
      "value": 32768,
      "unit": "tokens",
      "note": "per-slot n_ctx with defaults"
    },
    {
      "metric": "s2_total_slots",
      "value": 4,
      "unit": "count",
      "note": "total_slots with -np unset (auto)"
    }
  ],
  "log": [
    "[harness] starting server 1: llama-server -m qwen2.5-0.5b-instruct-q4_k_m.gguf -a qwen2.5-0.5b -c 4096 -ngl all -np 2 --host 127.0.0.1 --port 8089 --api-key <key> --jinja -ctk q8_0 -ctv q8_0 --metrics",
    "[harness] OpenAI SDK venv",
    "[harness] starting server 2 via llama serve with defaults",
    "[harness] done",
    "[harness] cleanup",
    ""
  ]
}