Skip to content

Commit 336ef4e

Browse files
paultancre-btclaudelforst
authored
fix(anthropic): capture extended-thinking reasoning tokens (#746)
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com> Co-authored-by: lforst <8118419+lforst@users.noreply.github.com>
1 parent 46660f5 commit 336ef4e

7 files changed

Lines changed: 629 additions & 14 deletions

‎py/src/braintrust/integrations/anthropic/_utils.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,13 @@ def extract_anthropic_usage(
9191
if cache_creation_breakdown:
9292
metrics.pop("prompt_cache_creation_tokens", None)
9393

94+
output_tokens_details = _try_to_dict(usage.get("output_tokens_details"))
95+
if include_output and output_tokens_details is not None:
96+
# Anthropic reports extended-thinking usage as a subset of output_tokens, under the
97+
# nested output_tokens_details.thinking_tokens field. Surface it as the normalized
98+
# completion_reasoning_tokens metric (matching the OpenAI/pydantic-ai integrations).
99+
_set_numeric_metric(metrics, "completion_reasoning_tokens", output_tokens_details.get("thinking_tokens"))
100+
94101
server_tool_use = _try_to_dict(usage.get("server_tool_use"))
95102
if server_tool_use is not None:
96103
for source_name, value in server_tool_use.items():

‎py/src/braintrust/integrations/anthropic/cassettes/0.48.0/test_anthropic_messages_create_reasoning_tokens_metrics.yaml‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
interactions:
2+
- request:
3+
body: '{"max_tokens":2048,"messages":[{"role":"user","content":"What is 17 * 23?
4+
Think it through, then give the number."}],"model":"claude-haiku-4-5-20251001","thinking":{"type":"enabled","budget_tokens":1024}}'
5+
headers:
6+
accept:
7+
- application/json
8+
accept-encoding:
9+
- gzip, deflate
10+
anthropic-version:
11+
- '2023-06-01'
12+
connection:
13+
- keep-alive
14+
content-length:
15+
- '204'
16+
content-type:
17+
- application/json
18+
host:
19+
- api.anthropic.com
20+
user-agent:
21+
- Anthropic/Python 0.48.0
22+
x-stainless-arch:
23+
- x64
24+
x-stainless-async:
25+
- 'false'
26+
x-stainless-lang:
27+
- python
28+
x-stainless-os:
29+
- Linux
30+
x-stainless-package-version:
31+
- 0.48.0
32+
x-stainless-read-timeout:
33+
- '600'
34+
x-stainless-retry-count:
35+
- '0'
36+
x-stainless-runtime:
37+
- CPython
38+
x-stainless-runtime-version:
39+
- 3.14.6
40+
x-stainless-timeout:
41+
- '600'
42+
method: POST
43+
uri: https://api.anthropic.com/v1/messages
44+
response:
45+
body:
46+
string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011Cep4FvAdw5k9kFJqR5trP","type":"message","role":"assistant","content":[{"type":"thinking","thinking":"Let
47+
me calculate 17 * 23.\n\nI can break this down:\n17 * 23 = 17 * (20 + 3)\n=
48+
17 * 20 + 17 * 3\n= 340 + 51\n= 391\n\nLet me verify this another way:\n17
49+
* 23 = (20 - 3) * 23\n= 20 * 23 - 3 * 23\n= 460 - 69\n= 391\n\nYes, that''s
50+
correct.","signature":"Et4DCpoBCBEYAipAjRXpS/Xogu4z9JT1WNaqulh8un2XjgYshUyW62VC4OrbP71M+2O94HdPIAHCHwaXBSIIoaIEwjI7fFwhxbcdezIZY2xhdWRlLWhhaWt1LTQtNS0yMDI1MTAwMTgAQgh0aGlua2luZ1okMjc3OTY2NjgtNzM1MS00MGFjLWFjYzQtMDI0YWVlODk5NWE1qAH50/rUBhIMshMqDkSSKZFHM0PfGgxVK+BEEKCBAZKWsXEiMItCFjLNfYjkvT7BEERaQ4kaCA853Aa40E80dS1EDMNX9TqS5nyvVbA6swRCIlFF8CrwAe4QWNw6X4IKk4glFsus0MK8ZU7JUE+bdCueBUScZEWzX1MaZnweSdw+tZm9oRnJiJDNEjIMCXS17T0ua3RlAnMypfcwg8T/Rk7oYwYux5LnEwOfNTLAZvt4r01OqFTvfwn/hDDC7JvgsIA6Ol101LzqsO1lom9K6idbPk+FunOkk7peVoI8U5KFQlh8jBxLLAcZhPXDuuCoVEKwvB1e6/SbjPk9JNo1hO3hhXzCwxECK1NInWBKYZZz54Q1Bo/FfbLl4x8g+fJAiqdQgWzG4jlkmLmd0KMcF3ikqSgxMfY+MQLgRvfRt79xy8r5YhF7rRgB"},{"type":"text","text":"Let
51+
me work through this:\n\n17 * 23 = 17 * (20 + 3)\n= (17 * 20) + (17 * 3)\n=
52+
340 + 51\n= **391**"}],"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":54,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":191,"output_tokens_details":{"thinking_tokens":130},"service_tier":"standard","inference_geo":"not_available"}}'
53+
headers:
54+
anthropic-organization-id:
55+
- 27796668-7351-40ac-acc4-024aee8995a5
56+
anthropic-ratelimit-input-tokens-limit:
57+
- '10000000'
58+
anthropic-ratelimit-input-tokens-remaining:
59+
- '10000000'
60+
anthropic-ratelimit-input-tokens-reset:
61+
- '2026-09-07T12:11:36Z'
62+
anthropic-ratelimit-output-tokens-limit:
63+
- '2000000'
64+
anthropic-ratelimit-output-tokens-remaining:
65+
- '2000000'
66+
anthropic-ratelimit-output-tokens-reset:
67+
- '2026-09-07T12:11:37Z'
68+
anthropic-ratelimit-requests-limit:
69+
- '20000'
70+
anthropic-ratelimit-requests-remaining:
71+
- '19999'
72+
anthropic-ratelimit-requests-reset:
73+
- '2026-09-07T12:11:35Z'
74+
anthropic-ratelimit-tokens-limit:
75+
- '12000000'
76+
anthropic-ratelimit-tokens-remaining:
77+
- '12000000'
78+
anthropic-ratelimit-tokens-reset:
79+
- '2026-09-07T12:11:36Z'
80+
anthropic-workspace-id:
81+
- wrkspc_01J8hau4yJ2SKvtbeP9drm9A
82+
cf-cache-status:
83+
- DYNAMIC
84+
cf-ray:
85+
- a3759ded3f1bc96a-IAD
86+
connection:
87+
- keep-alive
88+
content-length:
89+
- '1536'
90+
content-security-policy:
91+
- default-src 'none'; frame-ancestors 'none'
92+
content-type:
93+
- application/json
94+
date:
95+
- Mon, 07 Sep 2026 12:11:37 GMT
96+
request-id:
97+
- req_011Cep4FuaQrAQvogvgT22gJ
98+
server:
99+
- cloudflare
100+
strict-transport-security:
101+
- max-age=31536000; includeSubDomains; preload
102+
traceresponse:
103+
- 00-dfd54e78541d06a1123d9af064aec955-51ee7da98cfd57f9-01
104+
transfer-encoding:
105+
- chunked
106+
vary:
107+
- Accept-Encoding
108+
x-robots-tag:
109+
- none
110+
status:
111+
code: 200
112+
message: OK
113+
version: 1

‎py/src/braintrust/integrations/anthropic/cassettes/0.48.0/test_anthropic_messages_stream_reasoning_tokens_metrics.yaml‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
interactions:
2+
- request:
3+
body: '{"max_tokens":2048,"messages":[{"role":"user","content":"What is 17 * 23?
4+
Think it through, then give the number."}],"model":"claude-haiku-4-5-20251001","stream":true,"thinking":{"type":"enabled","budget_tokens":1024}}'
5+
headers:
6+
accept:
7+
- application/json
8+
accept-encoding:
9+
- gzip, deflate
10+
anthropic-version:
11+
- '2023-06-01'
12+
connection:
13+
- keep-alive
14+
content-length:
15+
- '218'
16+
content-type:
17+
- application/json
18+
host:
19+
- api.anthropic.com
20+
user-agent:
21+
- Anthropic/Python 0.48.0
22+
x-stainless-arch:
23+
- x64
24+
x-stainless-async:
25+
- 'false'
26+
x-stainless-lang:
27+
- python
28+
x-stainless-os:
29+
- Linux
30+
x-stainless-package-version:
31+
- 0.48.0
32+
x-stainless-read-timeout:
33+
- '600'
34+
x-stainless-retry-count:
35+
- '0'
36+
x-stainless-runtime:
37+
- CPython
38+
x-stainless-runtime-version:
39+
- 3.14.6
40+
x-stainless-timeout:
41+
- NOT_GIVEN
42+
method: POST
43+
uri: https://api.anthropic.com/v1/messages
44+
response:
45+
body:
46+
string: "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011Cep4CYQrfbDgk2SQV7Pg6\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":54,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":1,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}
47+
\ }\n\nevent: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"thinking\",\"thinking\":\"\",\"signature\":\"\"}
48+
\ }\n\nevent: ping\ndata: {\"type\": \"ping\"}\n\nevent: content_block_delta\ndata:
49+
{\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\"Let\"}
50+
\ }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\"
51+
me calculate 17 * 23.\\n\\nI can break this down:\\n17 * 23 = 17 * (20 + 3)\\n=
52+
17 * 20 + 17\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\"
53+
* 3\\n= 340 + 51\\n= 391\\n\\nAlternatively:\\n17 * 23 = (10 + 7) * 23\\n=
54+
10 * \"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\"23
55+
+ 7 * 23\\n= 230 + 161\\n= 391\\n\\nLet me verify once more:\\n17 * 23\\n=
56+
17 * 20 + 17\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\"
57+
* 3\\n= 340 + 51\\n= 391\\n\\nYes, 391 is correct.\"} }\n\nevent:
58+
content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"signature_delta\",\"signature\":\"EpQECpoBCBEYAipANHewmm1S9pfqd1f1NsPz65CoWBhVOSwzlz/tlzNmt6Gy8dLhvJIBWf+6g+1OtMYk6Y0MChVFM+UgtHHETYe5uTIZY2xhdWRlLWhhaWt1LTQtNS0yMDI1MTAwMTgAQgh0aGlua2luZ1okMjc3OTY2NjgtNzM1MS00MGFjLWFjYzQtMDI0YWVlODk5NWE1qAHL0/rUBhIMDR1e8EblmM3G5OWPGgw7w7Gd652yQDx11zIiMIx53xS7pyMhznnwemeMXxcCpDElb+NGKf7IsSyk11z5GQV3Ai4lekthU4Dj0NqOYyqmAokwZrvXTihY4pgUsBYv5debbZRd/SwxB0m5YVmFx5cACZ59uK62Pjtw0rtEONiwuCzVCNpoJIPLzHrxPCLEQBhmBAyOyMEK+yq1H+de7FHKc7vHqdE9GBbjYQZyHvHG/X7mtUNU43DT2G59r2XqGV2T+WMWTEckg0l5SMVIwmu+RBEV/Tx5rNdzCPaVwucgiir28W3lqIIRJfto072ISzAf2t0tRck/WE+6Uv76qjHY95OTTmBNxqCtCnD6NL+RK6oeRTCfL4mYbyWol0pjCnu+8Mu2D231pq3Xy4BUac2x54H1i5uw8EbtsJ3bTwolgOQMP/L5cyB7Ea6hAhAqeuJKmB0VyHZe/CfPbxBBwrXt126jv/SHuARq8ehpEmfWYT2FAVFbfRgB\"}
59+
\ }\n\nevent: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0
60+
\ }\n\nevent: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":1,\"content_block\":{\"type\":\"text\",\"text\":\"\"}
61+
\ }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"text_delta\",\"text\":\"#
62+
Calculating 17 \xD7 23\\n\\nBreaking\"} }\n\nevent: content_block_delta\ndata:
63+
{\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"text_delta\",\"text\":\"
64+
it down:\\n- 17 \xD7 23 = 17 \xD7 (20 + 3)\\n- = (17 \xD7 20) + (17 \xD7 3)\\n-
65+
= 340 \"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"text_delta\",\"text\":\"+
66+
51\\n- = **391**\\n\\nThe answer is **391**.\"} }\n\nevent: content_block_stop\ndata:
67+
{\"type\":\"content_block_stop\",\"index\":1}\n\nevent: message_delta\ndata:
68+
{\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"stop_details\":null},\"usage\":{\"input_tokens\":54,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"output_tokens\":250,\"output_tokens_details\":{\"thinking_tokens\":169}}
69+
\ }\n\nevent: message_stop\ndata: {\"type\":\"message_stop\" }\n\n"
70+
headers:
71+
anthropic-organization-id:
72+
- 27796668-7351-40ac-acc4-024aee8995a5
73+
anthropic-ratelimit-input-tokens-limit:
74+
- '10000000'
75+
anthropic-ratelimit-input-tokens-remaining:
76+
- '10000000'
77+
anthropic-ratelimit-input-tokens-reset:
78+
- '2026-09-07T12:10:50Z'
79+
anthropic-ratelimit-output-tokens-limit:
80+
- '2000000'
81+
anthropic-ratelimit-output-tokens-remaining:
82+
- '2000000'
83+
anthropic-ratelimit-output-tokens-reset:
84+
- '2026-09-07T12:10:50Z'
85+
anthropic-ratelimit-requests-limit:
86+
- '20000'
87+
anthropic-ratelimit-requests-remaining:
88+
- '19999'
89+
anthropic-ratelimit-requests-reset:
90+
- '2026-09-07T12:10:50Z'
91+
anthropic-ratelimit-tokens-limit:
92+
- '12000000'
93+
anthropic-ratelimit-tokens-remaining:
94+
- '12000000'
95+
anthropic-ratelimit-tokens-reset:
96+
- '2026-09-07T12:10:50Z'
97+
anthropic-workspace-id:
98+
- wrkspc_01J8hau4yJ2SKvtbeP9drm9A
99+
cache-control:
100+
- no-cache
101+
cf-cache-status:
102+
- DYNAMIC
103+
cf-ray:
104+
- a3759ccebeba4c80-IAD
105+
connection:
106+
- keep-alive
107+
content-length:
108+
- '3650'
109+
content-security-policy:
110+
- default-src 'none'; frame-ancestors 'none'
111+
content-type:
112+
- text/event-stream; charset=utf-8
113+
date:
114+
- Mon, 07 Sep 2026 12:10:50 GMT
115+
request-id:
116+
- req_011Cep4CXdj2dbuiKhQRa6T1
117+
server:
118+
- cloudflare
119+
strict-transport-security:
120+
- max-age=31536000; includeSubDomains; preload
121+
traceresponse:
122+
- 00-91d778ba9a76bedf9b72507b165b7243-44d7f2c5c22c3c82-01
123+
transfer-encoding:
124+
- chunked
125+
vary:
126+
- Accept-Encoding
127+
x-robots-tag:
128+
- none
129+
status:
130+
code: 200
131+
message: OK
132+
version: 1

‎py/src/braintrust/integrations/anthropic/cassettes/latest/test_anthropic_messages_create_reasoning_tokens_metrics.yaml‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
interactions:
2+
- request:
3+
body: '{"max_tokens":2048,"messages":[{"role":"user","content":"What is 17 * 23?
4+
Think it through, then give the number."}],"model":"claude-haiku-4-5-20251001","thinking":{"type":"enabled","budget_tokens":1024}}'
5+
headers:
6+
accept:
7+
- application/json
8+
accept-encoding:
9+
- gzip, deflate
10+
anthropic-version:
11+
- '2023-06-01'
12+
connection:
13+
- keep-alive
14+
content-length:
15+
- '204'
16+
content-type:
17+
- application/json
18+
host:
19+
- api.anthropic.com
20+
user-agent:
21+
- Anthropic/Python 1.2.0
22+
x-stainless-arch:
23+
- arm64
24+
x-stainless-async:
25+
- 'false'
26+
x-stainless-lang:
27+
- python
28+
x-stainless-os:
29+
- MacOS
30+
x-stainless-package-version:
31+
- 1.2.0
32+
x-stainless-read-timeout:
33+
- '600'
34+
x-stainless-retry-count:
35+
- '0'
36+
x-stainless-runtime:
37+
- CPython
38+
x-stainless-runtime-version:
39+
- 3.11.15
40+
x-stainless-timeout:
41+
- '600'
42+
method: POST
43+
uri: https://api.anthropic.com/v1/messages
44+
response:
45+
body:
46+
string: "{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011CeoqtD3dWGwJ1229TnnFD\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[{\"type\":\"thinking\",\"thinking\":\"17
47+
* 23\\n\\nLet me break this down:\\n17 * 23 = 17 * (20 + 3)\\n= 17 * 20 +
48+
17 * 3\\n= 340 + 51\\n= 391\\n\\nLet me double-check:\\n17 * 20 = 340 ✓\\n17
49+
* 3 = 51 ✓\\n340 + 51 = 391 ✓\\n\\nAlternatively:\\n17 * 23 = (10
50+
+ 7) * 23\\n= 10 * 23 + 7 * 23\\n= 230 + 161\\n= 391 ✓\",\"signature\":\"EvYDCpoBCBEYAipAJUDXcciCV/oALWk53KOqQ43mNIbhG1QWcimrkKC9UtYmzjGqPkUAY1yaP3URbyLFmY2GtC2L4DyY/bHDYtghlzIZY2xhdWRlLWhhaWt1LTQtNS0yMDI1MTAwMTgAQgh0aGlua2luZ1okMjc3OTY2NjgtNzM1MS00MGFjLWFjYzQtMDI0YWVlODk5NWE1qAHxh/rUBhIMNZJuVeYYxJ2RsKIrGgycuj7DxA2X5mSez7AiMAVKuQ01okNXZPxQL+fnZDp8HaivH4MGgb1aYDygkv98bXje2HFwREsLluc81yimZyqIAiF5KqKOkCSAWX0dgug2D7e0SUwMql070+i+6dJ1Kx9b518BWF0gWOFrHlCZ39VptVamutI7P+Gv1aXd8W8eNANN222vGX5D5nqRz/rY4Z+lLld4mfr4jUrMiMUenJpOaiO33ZdQ10jPmn0OZVAfUHcQByJQkbrP1Hqzr77ahsPchSesoYYD83smavcj1XMLxu2VXZ6NKO1gnxSjJ6nMCfC44CZBim/3k2qYz5ABIiJqLPA44okcRKxrF5qWF3Nc/2bwQEs+f/csAAMAKvMjP47pibT7tFvOYxhS71YI1BLBU4ka5SxF0u/NxqC3YOAgwv4Wc6UcvCGZSsvRsQxvwumeAr+H0a/6GRgB\"},{\"type\":\"text\",\"text\":\"17
51+
* 23:\\n\\nI'll break this into parts:\\n- 17 * 20 = 340\\n- 17 * 3 = 51\\n-
52+
340 + 51 = 391\\n\\n**391**\"}],\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":54,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":227,\"output_tokens_details\":{\"thinking_tokens\":168},\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}"
53+
headers:
54+
connection:
55+
- keep-alive
56+
content-type:
57+
- application/json
58+
date:
59+
- Mon, 07 Sep 2026 09:29:21 GMT
60+
strict-transport-security:
61+
- max-age=31536000; includeSubDomains
62+
transfer-encoding:
63+
- chunked
64+
vary:
65+
- accept-encoding
66+
status:
67+
code: 200
68+
message: OK
69+
version: 1

0 commit comments

Comments
 (0)