AgenticWire test record: claude-agent-sdk-vs-claude-code
Article: https://www.agenticwire.news/article/claude-agent-sdk-vs-claude-code
Machine and versions: see results.json (machine field).

Harness script (experiment/run.sh):

#!/usr/bin/env bash
# Claude Agent SDK vs Claude Code CLI: controlled task comparison.
#
# What this measures, and what it deliberately does not:
#   - The two surfaces wrap the same engine, so this is NOT a model-quality benchmark. It measures
#     what the SURFACE costs you: defaults, prompt-cache behaviour, wall time, tool-call count,
#     token/cost totals, and whether the task actually completed.
#   - Two configurations are reported separately, because "same engine" does not mean same defaults:
#       default  : CLI `claude -p "<task>"`, SDK `query({prompt})`. Out-of-the-box, nothing tuned.
#       parity   : same model, same allowedTools, same maxTurns, same cwd, permissions bypassed
#                  on both. This is the only apples-to-apples number.
#
# Fixture: a deterministic local git repo built by this script (no network, no third-party repo that
# can change or vanish). Same content and same commit on every run. Each run gets a FRESH COPY so
# one run's edits cannot contaminate the next.
#
# Tasks, each with an objective pass/fail rather than a judgement call:
#   inspect   : locate a definition            -> answer must name the right file
#   fix       : make a failing test pass       -> the test runner must exit 0 AND tests/ must be unmodified
#   multistep : grep + transform + write file  -> TODOS.md must exist with the right entry count
#
# Re-run from a clean checkout:
#   node blog-app/scripts/pipeline/bench.mjs drafts/claude-agent-sdk-vs-claude-code
# Env: RUNS (default 3), TASKS (default "inspect fix multistep"), CONFIGS (default "default parity")
set -uo pipefail
cd "$(dirname "$0")"
HERE="$PWD"
log() { echo "$@" >&2; }
emit() { echo "$1"; }

RUNS="${RUNS:-3}"
TASKS="${TASKS:-inspect fix multistep}"
CONFIGS="${CONFIGS:-default parity}"
MODEL="claude-sonnet-5"
TOOLS="Read,Write,Edit,Bash,Glob,Grep"
MAX_TURNS=30

command -v claude >/dev/null || { log "claude CLI not on PATH"; exit 1; }
command -v python3 >/dev/null || { log "python3 not on PATH"; exit 1; }

WORK="${TMPDIR:-/tmp}/cagsdk-bench-$$"
mkdir -p "$WORK"
trap 'rm -rf "$WORK"' EXIT
RAW="$WORK/raw.jsonl"; : > "$RAW"

# ---------- versions under test ----------
CLI_V=$(claude --version 2>/dev/null | awk '{print $1}')
emit "{\"metric\":\"cli_version_tested\",\"value\":\"$CLI_V\",\"note\":\"claude --version at run time\"}"

cd "$WORK"
npm init -y >/dev/null 2>&1
npm install @anthropic-ai/claude-agent-sdk >/dev/null 2>&1
SDK_V=$(node -e "console.log(require('$WORK/node_modules/@anthropic-ai/claude-agent-sdk/package.json').version)")
SDK_EXPORTS=$(node -e "console.log(Object.keys(require('$WORK/node_modules/@anthropic-ai/claude-agent-sdk')).length)")
emit "{\"metric\":\"sdk_version_tested\",\"value\":\"$SDK_V\",\"note\":\"fresh npm install at run time\"}"
emit "{\"metric\":\"sdk_export_count\",\"value\":$SDK_EXPORTS,\"note\":\"Object.keys on the module namespace\"}"

# --- npm registry history: keeps the live article's 2026-08-03 Correction block traceable ---
REG=$(curl -sS -m 20 https://registry.npmjs.org/@anthropic-ai/claude-agent-sdk || echo '{}')
V020_DATE=$(printf '%s' "$REG" | python3 -c "import json,sys;print(json.load(sys.stdin).get('time',{}).get('0.2.0','unknown'))" 2>/dev/null || echo unknown)
emit "{\"metric\":\"sdk_0_2_0_publish_date\",\"value\":\"$V020_DATE\",\"note\":\"npm registry time field\"}"
PUBDAY=$(printf '%s' "$REG" | python3 -c "
import json,sys
t=json.load(sys.stdin).get('time',{})
same=sorted(v for v,ts in t.items() if ts.startswith('2026-06-25') and v not in ('created','modified'))
print(same[-1] if same else 'none')
" 2>/dev/null || echo unknown)
emit "{\"metric\":\"sdk_version_published_on_article_publish_date\",\"value\":\"$PUBDAY\",\"note\":\"latest version npm shows published on 2026-06-25, the article's publish date\"}"

# ---------- deterministic fixture repo ----------
FIX="$WORK/fixture"
mkdir -p "$FIX/src/statkit" "$FIX/tests"
cat > "$FIX/src/statkit/stats.py" <<'PY'
"""Small statistics helpers."""


def mean(values):
    if not values:
        raise ValueError("mean() requires at least one value")
    return sum(values) / len(values)


def median(values):
    # TODO: handle the even-length case properly
    if not values:
        raise ValueError("median() requires at least one value")
    ordered = sorted(values)
    return ordered[len(ordered) // 2]


def variance(values):
    # TODO: switch to a single-pass algorithm
    m = mean(values)
    return sum((v - m) ** 2 for v in values) / len(values)
PY
cat > "$FIX/src/statkit/__init__.py" <<'PY'
from .stats import mean, median, variance

__all__ = ["mean", "median", "variance"]
PY
# Self-contained runner: `python3 tests/test_stats.py` exits non-zero on failure. No pytest, so the
# fixture is reproducible from a clean checkout with nothing but python3 on PATH.
cat > "$FIX/tests/test_stats.py" <<'PY'
import sys, os
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src"))

from statkit import mean, median, variance

CASES = [
    ("mean", lambda: mean([1, 2, 3]), 2),
    ("median_odd", lambda: median([3, 1, 2]), 2),
    # median of [1, 2, 3, 4] is 2.5, not 3
    ("median_even", lambda: median([1, 2, 3, 4]), 2.5),
    ("variance", lambda: variance([2, 2, 2]), 0),
]

if __name__ == "__main__":
    failed = 0
    for name, fn, want in CASES:
        try:
            got = fn()
        except Exception as exc:
            print(f"FAIL {name}: raised {exc!r}")
            failed += 1
            continue
        if got != want:
            print(f"FAIL {name}: got {got!r}, want {want!r}")
            failed += 1
        else:
            print(f"ok   {name}")
    print(f"{len(CASES) - failed}/{len(CASES)} passed")
    sys.exit(1 if failed else 0)
PY
cat > "$FIX/README.md" <<'MD'
# statkit

Tiny statistics helpers used as a benchmark fixture.

<!-- TODO: document the public API -->
MD
( cd "$FIX" && git init -q && git add -A \
  && GIT_AUTHOR_NAME=bench GIT_AUTHOR_EMAIL=bench@local \
     GIT_COMMITTER_NAME=bench GIT_COMMITTER_EMAIL=bench@local \
     GIT_AUTHOR_DATE="2026-01-01T00:00:00Z" GIT_COMMITTER_DATE="2026-01-01T00:00:00Z" \
     git commit -qm "fixture" )
FIX_SHA=$( cd "$FIX" && git rev-parse HEAD )
emit "{\"metric\":\"fixture_commit\",\"value\":\"$FIX_SHA\",\"note\":\"deterministic fixture repo, same content every run\"}"
TODO_COUNT=$( grep -rc "TODO" "$FIX" --include=*.py --include=*.md 2>/dev/null | awk -F: '{s+=$2} END {print s}' )
emit "{\"metric\":\"fixture_todo_count\",\"value\":$TODO_COUNT,\"note\":\"ground truth for the multistep task\"}"

# ---------- task prompts ----------
mkdir -p "$WORK/prompts"
cat > "$WORK/prompts/inspect.txt" <<'P'
In this repository, which file defines the median function? Reply with only the file path relative to the repository root, nothing else.
P
cat > "$WORK/prompts/fix.txt" <<'P'
The test suite in this repository fails. Run it, find the bug in the source code, and fix it so all tests pass. Do not modify any file inside the tests/ directory.
P
cat > "$WORK/prompts/multistep.txt" <<'P'
Find every TODO comment in this repository. Write them to a new file TODOS.md at the repository root, one per line, sorted by file path, each line formatted exactly as: path:line: text
P

# ---------- objective grading ----------
grade() { # grade <task> <runDir> <answerFile> -> "1" pass / "0" fail
  local task="$1" dir="$2" ans="$3"
  case "$task" in
    inspect)
      grep -qi "stats.py" "$ans" && echo 1 || echo 0 ;;
    fix)
      # tests must pass AND the agent must not have edited the tests to get there
      local dirty
      dirty=$( cd "$dir" && git status --porcelain -- tests/ | wc -l | tr -d ' ' )
      if [ "$dirty" != "0" ]; then echo 0; return; fi
      ( cd "$dir" && python3 tests/test_stats.py >/dev/null 2>&1 ) && echo 1 || echo 0 ;;
    multistep)
      if [ ! -f "$dir/TODOS.md" ]; then echo 0; return; fi
      local n
      n=$( grep -c ":" "$dir/TODOS.md" 2>/dev/null || echo 0 )
      [ "$n" -ge "$TODO_COUNT" ] && echo 1 || echo 0 ;;
    *) echo 0 ;;
  esac
}

# Sanity: the fixture must actually start red, or the 'fix' task grades a no-op as success.
( cd "$FIX" && python3 tests/test_stats.py >/dev/null 2>&1 ) \
  && { log "FATAL: fixture tests already pass, 'fix' task would be meaningless"; exit 1; } \
  || log "fixture starts red as expected"

# ---------- run matrix ----------
for task in $TASKS; do
  for config in $CONFIGS; do
    for i in $( seq 1 "$RUNS" ); do
      # --- CLI ---
      RD="$WORK/run-cli-$task-$config-$i"
      cp -R "$FIX" "$RD"
      A="$RD/.agent-answer"
      T0=$(python3 -c 'import time;print(int(time.time()*1000))')
      if [ "$config" = "parity" ]; then
        OUT=$( cd "$RD" && claude -p "$(cat "$WORK/prompts/$task.txt")" \
                 --output-format json --model "$MODEL" --allowed-tools "$TOOLS" \
                 --max-turns "$MAX_TURNS" --dangerously-skip-permissions </dev/null 2>/dev/null )
      else
        OUT=$( cd "$RD" && claude -p "$(cat "$WORK/prompts/$task.txt")" \
                 --output-format json --dangerously-skip-permissions </dev/null 2>/dev/null )
      fi
      T1=$(python3 -c 'import time;print(int(time.time()*1000))')
      echo "$OUT" > "$RD/.raw.json"
      if python3 - "$RD/.raw.json" <<'PY'
import json,sys
try: d=json.load(open(sys.argv[1]))
except Exception: d={}
sys.exit(0 if d.get("terminal_reason") == "api_error" or d.get("api_error_status") else 1)
PY
      then
        log "FATAL: cli/$config/$task hit API error: $(python3 -c 'import json,sys; d=json.load(open(sys.argv[1])); print(d.get("result") or d.get("api_error_status") or "unknown")' "$RD/.raw.json")"
        exit 2
      fi
      python3 - "$RD/.raw.json" "$A" <<'PY'
import json,sys
try:
    d=json.load(open(sys.argv[1]))
except Exception:
    d={}
open(sys.argv[2],"w").write(str(d.get("result","")))
PY
      PASS=$( grade "$task" "$RD" "$A" )
      python3 - "$RD/.raw.json" "$task" "$config" "$PASS" "$T0" "$T1" >> "$RAW" <<'PY'
import json,sys
p,task,config,ok,t0,t1=sys.argv[1:7]
try: d=json.load(open(p))
except Exception: d={}
u=d.get("usage") or {}
mu=d.get("modelUsage") or {}
print(json.dumps({"surface":"cli","config":config,"task":task,
 "wall_ms": d.get("duration_ms") or (int(t1)-int(t0)),
 "turns": d.get("num_turns"), "tool_calls": None,
 "cost_usd": d.get("total_cost_usd"),
 "input_tokens": u.get("input_tokens"), "output_tokens": u.get("output_tokens"),
 "cache_creation_tokens": u.get("cache_creation_input_tokens"),
 "cache_read_tokens": u.get("cache_read_input_tokens"),
 "is_error": bool(d.get("is_error")), "stop_reason": d.get("subtype"),
 "terminal_reason": d.get("terminal_reason"), "api_error_status": d.get("api_error_status"),
 "model": (list(mu.keys())[0] if mu else None), "passed": ok=="1"}))
PY
      log "  cli/$config/$task run $i pass=$PASS"

      # --- SDK ---
      RD="$WORK/run-sdk-$task-$config-$i"
      cp -R "$FIX" "$RD"
      # One invocation only: stdout is the measurement line, stderr goes to the run log.
      LINE=$( cd "$WORK" && SDK_ROOT="$WORK" node "$HERE/sdk-run.mjs" \
                "$RD" "$config" "$WORK/prompts/$task.txt" 2>>"$WORK/sdk-errors.log" )
      if echo "$LINE" | python3 -c '
import json,sys
d=json.loads(sys.stdin.read() or "{}")
sys.exit(0 if d.get("terminal_reason") == "api_error" or d.get("api_error_status") else 1)
'; then
        log "FATAL: sdk/$config/$task hit API error"
        exit 2
      fi
      echo "$LINE" | python3 -c "
import json,sys
d=json.loads(sys.stdin.read() or '{}')
open('$RD/.agent-answer','w').write(d.get('text',''))
" 2>/dev/null || : > "$RD/.agent-answer"
      PASS=$( grade "$task" "$RD" "$RD/.agent-answer" )
      echo "$LINE" | python3 -c "
import json,sys
d=json.loads(sys.stdin.read() or '{}')
d['task']='$task'; d['passed']=('$PASS'=='1'); d.pop('text',None)
print(json.dumps(d))
" >> "$RAW"
      log "  sdk/$config/$task run $i pass=$PASS"
    done
  done
done

# ---------- aggregate: median + range per cell ----------
python3 - "$RAW" <<'PY'
import json,sys,statistics as st
rows=[json.loads(l) for l in open(sys.argv[1]) if l.strip()]
def med(vs):
    vs=[v for v in vs if isinstance(v,(int,float))]
    return round(st.median(vs),4) if vs else None
def rng(vs):
    vs=[v for v in vs if isinstance(v,(int,float))]
    return f"{round(min(vs),4)}-{round(max(vs),4)}" if vs else None
cells={}
for r in rows: cells.setdefault((r["surface"],r["config"],r["task"]),[]).append(r)
for (surface,config,task),rs in sorted(cells.items()):
    n=len(rs); pfx=f"{surface}_{config}_{task}"
    for field,unit in (("wall_ms","ms"),("cost_usd","usd"),("turns","turns"),
                       ("tool_calls","calls"),("cache_creation_tokens","tokens"),
                       ("cache_read_tokens","tokens"),("output_tokens","tokens")):
        m=med([r.get(field) for r in rs])
        if m is None: continue
        print(json.dumps({"metric":f"{pfx}_{field}_median","value":m,"unit":unit,
                          "note":f"median of {n} runs; range {rng([r.get(field) for r in rs])}","n":n}))
    print(json.dumps({"metric":f"{pfx}_pass_count","value":sum(1 for r in rs if r.get('passed')),
                      "unit":"runs","note":f"objective grading, out of {n}","n":n}))
    print(json.dumps({"metric":f"{pfx}_error_count","value":sum(1 for r in rs if r.get('is_error')),
                      "unit":"runs","note":f"surface reported is_error, out of {n}","n":n}))
    mdl=[r.get("model") for r in rs if r.get("model")]
    if mdl: print(json.dumps({"metric":f"{pfx}_model","value":mdl[0],"note":"model actually used"}))
PY

log "done"