lanyuanxiaoyao
e1af978c56
- 修复 Anthropic Count Tokens 响应验证器，检查嵌套结构 - 补充 OpenAI service_tier: default 测试 - 补充 Anthropic output_config 带 effort 字段测试 - 补充 OpenAI reasoning_effort: low/high 测试 - 补充 Anthropic service_tier: standard_only 测试 - 修复流式响应 choices 数量验证逻辑，跳过空数组
1106 lines
36 KiB
Python
1106 lines
36 KiB
Python
#!/usr/bin/env python3
|
||
"""Anthropic 兼容性接口测试脚本
|
||
|
||
用法:
|
||
python3 scripts/anthropic_detect.py --base_url <url> [options]
|
||
|
||
示例:
|
||
python3 scripts/anthropic_detect.py --base_url https://api.example.com
|
||
python3 scripts/anthropic_detect.py --base_url https://api.example.com --api_key sk-xxx --model claude-sonnet-4-5
|
||
python3 scripts/anthropic_detect.py --base_url https://api.example.com --stream --tools --vision
|
||
"""
|
||
|
||
import json
|
||
import argparse
|
||
from typing import Dict, List, Tuple, Any
|
||
from core import (
|
||
create_ssl_context,
|
||
TestCase,
|
||
run_test_suite,
|
||
validate_response_structure,
|
||
)
|
||
|
||
ANTHROPIC_VERSION = "2023-06-01"
|
||
|
||
|
||
def build_headers(api_key: str) -> Dict[str, str]:
|
||
"""构建 Anthropic API 请求头"""
|
||
h = {
|
||
"Content-Type": "application/json",
|
||
"anthropic-version": ANTHROPIC_VERSION,
|
||
}
|
||
if api_key:
|
||
h["x-api-key"] = api_key
|
||
return h
|
||
|
||
|
||
# ==================== Anthropic 响应验证函数 ====================
|
||
|
||
def validate_anthropic_models_list_response(response_text: str) -> Tuple[bool, List[str]]:
|
||
"""验证 Anthropic Models List 响应
|
||
|
||
根据API文档，响应应包含：
|
||
- data: array of ModelInfo
|
||
- first_id: string (可选)
|
||
- has_more: boolean
|
||
- last_id: string (可选)
|
||
"""
|
||
errors = []
|
||
|
||
try:
|
||
data = json.loads(response_text)
|
||
except json.JSONDecodeError as e:
|
||
return False, [f"响应不是有效的JSON: {e}"]
|
||
|
||
# 检查必需字段
|
||
required_fields = ["data", "has_more"]
|
||
for field in required_fields:
|
||
if field not in data:
|
||
errors.append(f"缺少必需字段: {field}")
|
||
|
||
# 检查 data 数组
|
||
if "data" in data:
|
||
if not isinstance(data["data"], list):
|
||
errors.append(f"字段 'data' 类型错误: 期望 list, 实际 {type(data['data']).__name__}")
|
||
else:
|
||
for i, model in enumerate(data["data"]):
|
||
if not isinstance(model, dict):
|
||
errors.append(f"data[{i}] 不是对象")
|
||
continue
|
||
|
||
# 检查 model 对象的必需字段
|
||
model_required = ["id", "type", "display_name", "created_at"]
|
||
for field in model_required:
|
||
if field not in model:
|
||
errors.append(f"data[{i}] 缺少必需字段: {field}")
|
||
|
||
# 检查 type 字段值
|
||
if "type" in model and model["type"] != "model":
|
||
errors.append(f"data[{i}].type 值错误: 期望 'model', 实际 '{model['type']}'")
|
||
|
||
return len(errors) == 0, errors
|
||
|
||
|
||
def validate_anthropic_model_retrieve_response(response_text: str) -> Tuple[bool, List[str]]:
|
||
"""验证 Anthropic Model Retrieve 响应
|
||
|
||
根据API文档，响应应包含：
|
||
- id: string
|
||
- type: "model"
|
||
- display_name: string
|
||
- created_at: string
|
||
- max_input_tokens: number
|
||
- max_tokens: number
|
||
"""
|
||
required_fields = ["id", "type", "display_name", "created_at"]
|
||
field_types = {
|
||
"id": str,
|
||
"type": str,
|
||
"display_name": str,
|
||
"created_at": str
|
||
}
|
||
enum_values = {
|
||
"type": ["model"]
|
||
}
|
||
|
||
return validate_response_structure(response_text, required_fields, field_types, enum_values)
|
||
|
||
|
||
def validate_anthropic_messages_response(response_text: str) -> Tuple[bool, List[str]]:
|
||
"""验证 Anthropic Messages 响应
|
||
|
||
根据API文档，响应应包含：
|
||
- id: string
|
||
- type: "message"
|
||
- role: "assistant"
|
||
- content: array
|
||
- model: string
|
||
- stop_reason: string (可选)
|
||
- usage: object
|
||
"""
|
||
errors = []
|
||
|
||
try:
|
||
data = json.loads(response_text)
|
||
except json.JSONDecodeError as e:
|
||
return False, [f"响应不是有效的JSON: {e}"]
|
||
|
||
# 检查必需字段
|
||
required_fields = ["id", "type", "role", "content", "model"]
|
||
for field in required_fields:
|
||
if field not in data:
|
||
errors.append(f"缺少必需字段: {field}")
|
||
|
||
# 检查特定字段值
|
||
if "type" in data and data["type"] != "message":
|
||
errors.append(f"字段 'type' 值错误: 期望 'message', 实际 '{data['type']}'")
|
||
|
||
if "role" in data and data["role"] != "assistant":
|
||
errors.append(f"字段 'role' 值错误: 期望 'assistant', 实际 '{data['role']}'")
|
||
|
||
# 检查 content 数组
|
||
if "content" in data:
|
||
if not isinstance(data["content"], list):
|
||
errors.append(f"字段 'content' 类型错误: 期望 list, 实际 {type(data['content']).__name__}")
|
||
else:
|
||
for i, block in enumerate(data["content"]):
|
||
if not isinstance(block, dict):
|
||
errors.append(f"content[{i}] 不是对象")
|
||
continue
|
||
|
||
# 检查 content block 的必需字段
|
||
if "type" not in block:
|
||
errors.append(f"content[{i}] 缺少必需字段: type")
|
||
|
||
# 检查 usage 对象
|
||
if "usage" in data and data["usage"] is not None:
|
||
if not isinstance(data["usage"], dict):
|
||
errors.append(f"字段 'usage' 类型错误: 期望 object, 实际 {type(data['usage']).__name__}")
|
||
else:
|
||
usage_fields = ["input_tokens", "output_tokens"]
|
||
for field in usage_fields:
|
||
if field not in data["usage"]:
|
||
errors.append(f"usage 缺少必需字段: {field}")
|
||
|
||
return len(errors) == 0, errors
|
||
|
||
|
||
def validate_anthropic_count_tokens_response(response_text: str) -> Tuple[bool, List[str]]:
|
||
"""验证 Anthropic Count Tokens 响应
|
||
|
||
根据API文档，响应应包含：
|
||
- message_tokens_count: object { input_tokens }
|
||
"""
|
||
errors = []
|
||
|
||
try:
|
||
data = json.loads(response_text)
|
||
except json.JSONDecodeError as e:
|
||
return False, [f"响应不是有效的JSON: {e}"]
|
||
|
||
# 检查嵌套结构
|
||
if "message_tokens_count" not in data:
|
||
errors.append("缺少必需字段: message_tokens_count")
|
||
else:
|
||
mtc = data["message_tokens_count"]
|
||
if not isinstance(mtc, dict):
|
||
errors.append(f"字段 'message_tokens_count' 类型错误: 期望 object, 实际 {type(mtc).__name__}")
|
||
else:
|
||
if "input_tokens" not in mtc:
|
||
errors.append("message_tokens_count 缺少必需字段: input_tokens")
|
||
elif not isinstance(mtc["input_tokens"], (int, float)):
|
||
errors.append(f"message_tokens_count.input_tokens 类型错误: 期望 number, 实际 {type(mtc['input_tokens']).__name__}")
|
||
|
||
return len(errors) == 0, errors
|
||
|
||
|
||
def validate_anthropic_streaming_response(response_text: str) -> Tuple[bool, List[str]]:
|
||
"""验证 Anthropic 流式响应
|
||
|
||
流式响应使用 SSE 格式，每行以 "data: " 开头。
|
||
事件类型包括：message_start, content_block_start, content_block_delta, content_block_stop, message_delta, message_stop
|
||
|
||
验证要点：
|
||
- 每个事件是有效的 JSON
|
||
- 包含 message_start 和 message_stop 事件
|
||
- message_start 事件包含完整的 message 对象
|
||
|
||
Args:
|
||
response_text: SSE 格式的响应文本
|
||
|
||
Returns:
|
||
(是否验证通过, 错误信息列表)
|
||
"""
|
||
from core import parse_sse_events
|
||
|
||
errors = []
|
||
events = parse_sse_events(response_text)
|
||
|
||
if not events:
|
||
errors.append("未收到任何 SSE 事件")
|
||
return False, errors
|
||
|
||
has_message_start = False
|
||
has_message_stop = False
|
||
|
||
for i, event_data in enumerate(events):
|
||
try:
|
||
event = json.loads(event_data)
|
||
except json.JSONDecodeError as e:
|
||
errors.append(f"事件[{i}] 不是有效的JSON: {e}")
|
||
continue
|
||
|
||
if "type" not in event:
|
||
errors.append(f"事件[{i}] 缺少必需字段: type")
|
||
continue
|
||
|
||
event_type = event["type"]
|
||
|
||
if event_type == "message_start":
|
||
has_message_start = True
|
||
if "message" not in event:
|
||
errors.append(f"message_start 事件缺少 message 字段")
|
||
elif not isinstance(event["message"], dict):
|
||
errors.append(f"message_start 事件的 message 不是对象")
|
||
else:
|
||
msg = event["message"]
|
||
if "id" not in msg:
|
||
errors.append(f"message_start.message 缺少 id 字段")
|
||
if "type" not in msg:
|
||
errors.append(f"message_start.message 缺少 type 字段")
|
||
elif msg["type"] != "message":
|
||
errors.append(f"message_start.message.type 值错误: 期望 'message', 实际 '{msg['type']}'")
|
||
if "role" not in msg:
|
||
errors.append(f"message_start.message 缺少 role 字段")
|
||
elif msg["role"] != "assistant":
|
||
errors.append(f"message_start.message.role 值错误: 期望 'assistant', 实际 '{msg['role']}'")
|
||
if "content" not in msg:
|
||
errors.append(f"message_start.message 缺少 content 字段")
|
||
elif not isinstance(msg["content"], list):
|
||
errors.append(f"message_start.message.content 类型错误: 期望 list")
|
||
|
||
elif event_type == "message_stop":
|
||
has_message_stop = True
|
||
|
||
elif event_type == "content_block_delta":
|
||
if "delta" not in event:
|
||
errors.append(f"content_block_delta 事件缺少 delta 字段")
|
||
|
||
if not has_message_start:
|
||
errors.append("缺少 message_start 事件")
|
||
if not has_message_stop:
|
||
errors.append("缺少 message_stop 事件")
|
||
|
||
return len(errors) == 0, errors
|
||
|
||
|
||
def main():
|
||
parser = argparse.ArgumentParser(
|
||
description="Anthropic 兼容性接口测试",
|
||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||
)
|
||
parser.add_argument("--base_url", required=True, help="API 基础地址 (如 https://api.example.com)")
|
||
parser.add_argument("--api_key", default="", help="API 密钥 (默认空)")
|
||
parser.add_argument("--model", default="claude-sonnet-4-5", help="模型名称 (默认 claude-sonnet-4-5)")
|
||
parser.add_argument("--vision", action="store_true", help="执行视觉相关测试")
|
||
parser.add_argument("--stream", action="store_true", help="执行流式响应测试")
|
||
parser.add_argument("--tools", action="store_true", help="执行工具调用测试")
|
||
parser.add_argument("--thinking", action="store_true", help="执行扩展思维测试")
|
||
parser.add_argument("--all", action="store_true", help="开启所有扩展测试")
|
||
args = parser.parse_args()
|
||
|
||
if args.all:
|
||
args.vision = True
|
||
args.stream = True
|
||
args.tools = True
|
||
args.thinking = True
|
||
|
||
base_url = args.base_url.rstrip("/")
|
||
api_key = args.api_key
|
||
model = args.model
|
||
ssl_ctx = create_ssl_context()
|
||
headers = build_headers(api_key)
|
||
headers_bad_auth = build_headers("invalid-key-xxx")
|
||
|
||
messages_url = f"{base_url}/v1/messages"
|
||
models_url = f"{base_url}/v1/models"
|
||
count_tokens_url = f"{base_url}/v1/messages/count_tokens"
|
||
|
||
# --- 收集测试用例 ---
|
||
cases: List[TestCase] = []
|
||
|
||
# ==== Models API ====
|
||
cases.append(TestCase(
|
||
desc="获取模型列表 (GET /v1/models)",
|
||
method="GET",
|
||
url=models_url,
|
||
headers=headers,
|
||
validator=validate_anthropic_models_list_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="获取模型列表（分页 limit=3）(GET /v1/models?limit=3)",
|
||
method="GET",
|
||
url=f"{models_url}?limit=3",
|
||
headers=headers,
|
||
validator=validate_anthropic_models_list_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="获取指定模型详情 (GET /v1/models/{model})",
|
||
method="GET",
|
||
url=f"{models_url}/{model}",
|
||
headers=headers,
|
||
validator=validate_anthropic_model_retrieve_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="获取不存在的模型 (GET /v1/models/nonexistent-model-xxx)",
|
||
method="GET",
|
||
url=f"{models_url}/nonexistent-model-xxx",
|
||
headers=headers
|
||
))
|
||
|
||
# ==== Messages API: 正面用例 ====
|
||
cases.append(TestCase(
|
||
desc="基本对话（仅 user）",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 5,
|
||
"messages": [{"role": "user", "content": "Hi"}]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="system prompt + user 对话",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 5,
|
||
"system": "You are a helpful assistant.",
|
||
"messages": [{"role": "user", "content": "1+1="}]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="system prompt 数组格式（带缓存控制）",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 5,
|
||
"system": [
|
||
{"type": "text", "text": "You are a helpful assistant.", "cache_control": {"type": "ephemeral"}}
|
||
],
|
||
"messages": [{"role": "user", "content": "Hi"}]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="多轮对话（含 assistant 历史）",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 5,
|
||
"messages": [
|
||
{"role": "user", "content": "Hi"},
|
||
{"role": "assistant", "content": "Hello!"},
|
||
{"role": "user", "content": "1+1="}
|
||
]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="assistant prefill（部分回复填充）",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 1,
|
||
"messages": [
|
||
{"role": "user", "content": "What is latin for Ant? (A) Apoidea (B) Rhopalocera (C) Formicidae"},
|
||
{"role": "assistant", "content": "The answer is ("}
|
||
]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="content 数组格式（多个 text block）",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 5,
|
||
"messages": [{"role": "user", "content": [
|
||
{"type": "text", "text": "Hello"},
|
||
{"type": "text", "text": "1+1=?"}
|
||
]}]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="temperature + top_p",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 5,
|
||
"temperature": 0.5,
|
||
"top_p": 0.9,
|
||
"messages": [{"role": "user", "content": "Hi"}]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="temperature = 0（类确定性输出）",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 5,
|
||
"temperature": 0,
|
||
"messages": [{"role": "user", "content": "1+1="}]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="top_k 参数",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 5,
|
||
"top_k": 40,
|
||
"messages": [{"role": "user", "content": "Hi"}]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="max_tokens 限制",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 10,
|
||
"messages": [{"role": "user", "content": "讲一个故事"}]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="stop_sequences",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 20,
|
||
"stop_sequences": ["5"],
|
||
"messages": [{"role": "user", "content": "数数: 1,2,3,"}]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="metadata 参数（user_id）",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 5,
|
||
"metadata": {"user_id": "test-user-001"},
|
||
"messages": [{"role": "user", "content": "Hi"}]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="assistant content 数组格式（text + tool_use 块）",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 20,
|
||
"messages": [
|
||
{"role": "user", "content": "帮我查一下北京的天气"},
|
||
{"role": "assistant", "content": [
|
||
{"type": "text", "text": "好的，让我查一下。"},
|
||
{"type": "tool_use", "id": "toolu_prev_001", "name": "get_weather", "input": {"location": "Beijing"}}
|
||
]},
|
||
{"role": "user", "content": [
|
||
{"type": "tool_result", "tool_use_id": "toolu_prev_001", "content": [
|
||
{"type": "text", "text": "{\"temperature\": 22, \"condition\": \"晴\"}"}
|
||
]}
|
||
]}
|
||
]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
|
||
# ==== Count Tokens API ====
|
||
cases.append(TestCase(
|
||
desc="计数 Token (POST /v1/messages/count_tokens)",
|
||
method="POST",
|
||
url=count_tokens_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"messages": [{"role": "user", "content": "Hello, how are you?"}]
|
||
},
|
||
validator=validate_anthropic_count_tokens_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="计数 Token（带 system + tools）",
|
||
method="POST",
|
||
url=count_tokens_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"system": "You are a helpful assistant.",
|
||
"messages": [{"role": "user", "content": "Hi"}],
|
||
"tools": [{
|
||
"name": "get_weather",
|
||
"description": "获取天气",
|
||
"input_schema": {
|
||
"type": "object",
|
||
"properties": {"location": {"type": "string"}},
|
||
"required": ["location"]
|
||
}
|
||
}]
|
||
}
|
||
))
|
||
cases.append(TestCase(
|
||
desc="计数 Token 缺少 model（负面）",
|
||
method="POST",
|
||
url=count_tokens_url,
|
||
headers=headers,
|
||
body={
|
||
"messages": [{"role": "user", "content": "Hi"}]
|
||
}
|
||
))
|
||
|
||
# ==== Messages API: 负面用例 ====
|
||
cases.append(TestCase(
|
||
desc="缺少 x-api-key header（无认证）",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers={
|
||
"Content-Type": "application/json",
|
||
"anthropic-version": ANTHROPIC_VERSION,
|
||
},
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 5,
|
||
"messages": [{"role": "user", "content": "Hi"}]
|
||
}
|
||
))
|
||
cases.append(TestCase(
|
||
desc="错误的 anthropic-version header",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers={
|
||
"Content-Type": "application/json",
|
||
"anthropic-version": "0000-00-00",
|
||
"x-api-key": api_key,
|
||
} if api_key else {
|
||
"Content-Type": "application/json",
|
||
"anthropic-version": "0000-00-00",
|
||
},
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 5,
|
||
"messages": [{"role": "user", "content": "Hi"}]
|
||
}
|
||
))
|
||
cases.append(TestCase(
|
||
desc="缺少 model 参数",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"max_tokens": 5,
|
||
"messages": [{"role": "user", "content": "Hi"}]
|
||
}
|
||
))
|
||
cases.append(TestCase(
|
||
desc="缺少 messages 参数",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 5
|
||
}
|
||
))
|
||
cases.append(TestCase(
|
||
desc="缺少 max_tokens 参数",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"messages": [{"role": "user", "content": "Hi"}]
|
||
}
|
||
))
|
||
cases.append(TestCase(
|
||
desc="messages 为空数组",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 5,
|
||
"messages": []
|
||
}
|
||
))
|
||
cases.append(TestCase(
|
||
desc="无效 API key",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers_bad_auth,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 5,
|
||
"messages": [{"role": "user", "content": "Hi"}]
|
||
}
|
||
))
|
||
cases.append(TestCase(
|
||
desc="不存在的模型",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": "nonexistent-model-xxx",
|
||
"max_tokens": 5,
|
||
"messages": [{"role": "user", "content": "Hi"}]
|
||
}
|
||
))
|
||
cases.append(TestCase(
|
||
desc="畸形 JSON body",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body="invalid json{"
|
||
))
|
||
cases.append(TestCase(
|
||
desc="无效 role（非法消息角色）",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 5,
|
||
"messages": [{"role": "system", "content": "You are helpful"}]
|
||
}
|
||
))
|
||
cases.append(TestCase(
|
||
desc="max_tokens 为负数",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": -1,
|
||
"messages": [{"role": "user", "content": "Hi"}]
|
||
}
|
||
))
|
||
cases.append(TestCase(
|
||
desc="max_tokens = 0",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 0,
|
||
"messages": [{"role": "user", "content": "Hi"}]
|
||
}
|
||
))
|
||
cases.append(TestCase(
|
||
desc="temperature 超出范围 (2.0)",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 5,
|
||
"temperature": 2.0,
|
||
"messages": [{"role": "user", "content": "Hi"}]
|
||
}
|
||
))
|
||
|
||
# ==== --vision ====
|
||
if args.vision:
|
||
cases.append(TestCase(
|
||
desc="图片 URL 输入 (--vision)",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 10,
|
||
"messages": [{"role": "user", "content": [
|
||
{"type": "text", "text": "用一个词描述这张图"},
|
||
{"type": "image", "source": {
|
||
"type": "url",
|
||
"url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/"
|
||
"Gfp-wisconsin-madison-the-nature-boardwalk.jpg/"
|
||
"2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
|
||
}}
|
||
]}]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
|
||
# ==== --stream ====
|
||
if args.stream:
|
||
cases.append(TestCase(
|
||
desc="基本流式 (--stream)",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 5,
|
||
"stream": True,
|
||
"messages": [{"role": "user", "content": "Hi"}]
|
||
},
|
||
stream=True,
|
||
validator=validate_anthropic_streaming_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="流式 + system prompt (--stream)",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 5,
|
||
"stream": True,
|
||
"system": "Reply in one word.",
|
||
"messages": [{"role": "user", "content": "1+1="}]
|
||
},
|
||
stream=True,
|
||
validator=validate_anthropic_streaming_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="流式 + stop_sequences (--stream)",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 20,
|
||
"stream": True,
|
||
"stop_sequences": ["5"],
|
||
"messages": [{"role": "user", "content": "数数: 1,2,3,"}]
|
||
},
|
||
stream=True,
|
||
validator=validate_anthropic_streaming_response
|
||
))
|
||
|
||
# ==== --tools ====
|
||
if args.tools:
|
||
tool_weather = {
|
||
"name": "get_weather",
|
||
"description": "获取指定城市的天气",
|
||
"input_schema": {
|
||
"type": "object",
|
||
"properties": {
|
||
"location": {"type": "string", "description": "城市名称"}
|
||
},
|
||
"required": ["location"]
|
||
}
|
||
}
|
||
cases.append(TestCase(
|
||
desc="工具调用 tool_choice: auto (--tools)",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 50,
|
||
"tools": [tool_weather],
|
||
"tool_choice": {"type": "auto"},
|
||
"messages": [{"role": "user", "content": "北京天气怎么样？"}]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="工具调用 tool_choice: any (--tools)",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 50,
|
||
"tools": [tool_weather],
|
||
"tool_choice": {"type": "any"},
|
||
"messages": [{"role": "user", "content": "北京天气怎么样？"}]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="指定工具调用 tool_choice: {name} (--tools)",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 50,
|
||
"tools": [tool_weather],
|
||
"tool_choice": {"type": "tool", "name": "get_weather"},
|
||
"messages": [{"role": "user", "content": "北京天气怎么样？"}]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="tool_choice: none (--tools)",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 20,
|
||
"tools": [tool_weather],
|
||
"tool_choice": {"type": "none"},
|
||
"messages": [{"role": "user", "content": "北京天气怎么样？"}]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="多轮工具调用（tool_result 返回）(--tools)",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 20,
|
||
"tools": [tool_weather],
|
||
"messages": [
|
||
{"role": "user", "content": "北京天气怎么样？"},
|
||
{"role": "assistant", "content": [
|
||
{"type": "text", "text": "让我查一下。"},
|
||
{"type": "tool_use", "id": "toolu_001", "name": "get_weather", "input": {"location": "Beijing"}}
|
||
]},
|
||
{"role": "user", "content": [
|
||
{"type": "tool_result", "tool_use_id": "toolu_001", "content": "{\"temperature\": 22, \"condition\": \"晴\"}"}
|
||
]}
|
||
]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="多轮工具调用（tool_result 带 is_error）(--tools)",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 20,
|
||
"tools": [tool_weather],
|
||
"messages": [
|
||
{"role": "user", "content": "北京天气怎么样？"},
|
||
{"role": "assistant", "content": [
|
||
{"type": "tool_use", "id": "toolu_002", "name": "get_weather", "input": {"location": "Beijing"}}
|
||
]},
|
||
{"role": "user", "content": [
|
||
{"type": "tool_result", "tool_use_id": "toolu_002", "is_error": True, "content": "天气服务不可用"}
|
||
]}
|
||
]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="tool_choice 指向不存在的工具（负面）(--tools)",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 50,
|
||
"tools": [tool_weather],
|
||
"tool_choice": {"type": "tool", "name": "nonexistent_tool_xxx"},
|
||
"messages": [{"role": "user", "content": "北京天气怎么样？"}]
|
||
}
|
||
))
|
||
cases.append(TestCase(
|
||
desc="多工具定义 (--tools)",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 50,
|
||
"tools": [
|
||
tool_weather,
|
||
{
|
||
"name": "get_time",
|
||
"description": "获取指定城市的当前时间",
|
||
"input_schema": {
|
||
"type": "object",
|
||
"properties": {
|
||
"location": {"type": "string", "description": "城市名称"}
|
||
},
|
||
"required": ["location"]
|
||
}
|
||
}
|
||
],
|
||
"tool_choice": {"type": "auto"},
|
||
"messages": [{"role": "user", "content": "北京现在几点了？天气怎么样？"}]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
|
||
# ==== --thinking ====
|
||
if args.thinking:
|
||
cases.append(TestCase(
|
||
desc="扩展思维 enabled (--thinking)",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 200,
|
||
"thinking": {"type": "enabled", "budget_tokens": 100},
|
||
"messages": [{"role": "user", "content": "1+1=?"}]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="扩展思维 adaptive (--thinking)",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 200,
|
||
"thinking": {"type": "adaptive", "budget_tokens": 100},
|
||
"messages": [{"role": "user", "content": "1+1=?"}]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
|
||
# ==== --stream + --tools 组合 ====
|
||
if args.stream and args.tools:
|
||
tool_weather_stream = {
|
||
"name": "get_weather",
|
||
"description": "获取指定城市的天气",
|
||
"input_schema": {
|
||
"type": "object",
|
||
"properties": {
|
||
"location": {"type": "string", "description": "城市名称"}
|
||
},
|
||
"required": ["location"]
|
||
}
|
||
}
|
||
cases.append(TestCase(
|
||
desc="流式工具调用 (--stream --tools)",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 50,
|
||
"stream": True,
|
||
"tools": [tool_weather_stream],
|
||
"tool_choice": {"type": "auto"},
|
||
"messages": [{"role": "user", "content": "北京天气怎么样？"}]
|
||
},
|
||
stream=True,
|
||
validator=validate_anthropic_streaming_response
|
||
))
|
||
|
||
# ==== 高级参数测试 ====
|
||
# cache_control: 缓存控制
|
||
cases.append(TestCase(
|
||
desc="cache_control 缓存控制",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 10,
|
||
"cache_control": {"type": "ephemeral"},
|
||
"messages": [{"role": "user", "content": "Hello"}]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
|
||
# output_config: 输出配置
|
||
cases.append(TestCase(
|
||
desc="output_config 输出配置",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 10,
|
||
"output_config": {"format": "text"},
|
||
"messages": [{"role": "user", "content": "Hi"}]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="output_config 带 effort",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 10,
|
||
"output_config": {"format": "text", "effort": "low"},
|
||
"messages": [{"role": "user", "content": "Hi"}]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
|
||
# service_tier: 服务层级
|
||
cases.append(TestCase(
|
||
desc="service_tier: auto",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 5,
|
||
"service_tier": "auto",
|
||
"messages": [{"role": "user", "content": "Hello"}]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="service_tier: standard_only",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 5,
|
||
"service_tier": "standard_only",
|
||
"messages": [{"role": "user", "content": "Hello"}]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
|
||
# ==== Models API 分页测试 ====
|
||
cases.append(TestCase(
|
||
desc="Models API 分页 limit=5",
|
||
method="GET",
|
||
url=f"{models_url}?limit=5",
|
||
headers=headers
|
||
))
|
||
|
||
# ==== 执行测试 ====
|
||
flags = []
|
||
if args.vision:
|
||
flags.append("vision")
|
||
if args.stream:
|
||
flags.append("stream")
|
||
if args.tools:
|
||
flags.append("tools")
|
||
if args.thinking:
|
||
flags.append("thinking")
|
||
|
||
run_test_suite(
|
||
cases=cases,
|
||
ssl_ctx=ssl_ctx,
|
||
title="Anthropic 兼容性测试",
|
||
base_url=base_url,
|
||
model=model,
|
||
flags=flags
|
||
)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|