lanyuanxiaoyao
44d6af026a
- 提取共享定义（tool_weather, image_url, json_schema_math）到功能块前 - 流式用例精简为代表子集：核心 6-8 个 + 扩展各 1-2 个 + 高级参数代表 - OpenAI: 15 个流式用例（核心 8 + vision/tools/logprobs/json_schema + 高级参数） - Anthropic: 11 个流式用例（核心 6 + vision/tools/thinking + 高级参数） - 更新 README：新增流式测试覆盖原则、parse_sse_events 函数说明
1160 lines
39 KiB
Python
1160 lines
39 KiB
Python
#!/usr/bin/env python3
|
||
"""Anthropic 兼容性接口测试脚本
|
||
|
||
用法:
|
||
python3 scripts/anthropic_detect.py --base_url <url> [options]
|
||
|
||
示例:
|
||
python3 scripts/anthropic_detect.py --base_url https://api.example.com
|
||
python3 scripts/anthropic_detect.py --base_url https://api.example.com --api_key sk-xxx --model claude-sonnet-4-5
|
||
python3 scripts/anthropic_detect.py --base_url https://api.example.com --stream --tools --vision
|
||
"""
|
||
|
||
import json
|
||
import argparse
|
||
from typing import Dict, List, Tuple, Any
|
||
from core import (
|
||
create_ssl_context,
|
||
TestCase,
|
||
run_test_suite,
|
||
validate_response_structure,
|
||
)
|
||
|
||
ANTHROPIC_VERSION = "2023-06-01"
|
||
|
||
|
||
def build_headers(api_key: str) -> Dict[str, str]:
|
||
"""构建 Anthropic API 请求头"""
|
||
h = {
|
||
"Content-Type": "application/json",
|
||
"anthropic-version": ANTHROPIC_VERSION,
|
||
}
|
||
if api_key:
|
||
h["x-api-key"] = api_key
|
||
return h
|
||
|
||
|
||
# ==================== Anthropic 响应验证函数 ====================
|
||
|
||
def validate_anthropic_models_list_response(response_text: str) -> Tuple[bool, List[str]]:
|
||
"""验证 Anthropic Models List 响应
|
||
|
||
根据API文档，响应应包含：
|
||
- data: array of ModelInfo
|
||
- first_id: string (可选)
|
||
- has_more: boolean
|
||
- last_id: string (可选)
|
||
"""
|
||
errors = []
|
||
|
||
try:
|
||
data = json.loads(response_text)
|
||
except json.JSONDecodeError as e:
|
||
return False, [f"响应不是有效的JSON: {e}"]
|
||
|
||
# 检查必需字段
|
||
required_fields = ["data", "has_more"]
|
||
for field in required_fields:
|
||
if field not in data:
|
||
errors.append(f"缺少必需字段: {field}")
|
||
|
||
# 检查 data 数组
|
||
if "data" in data:
|
||
if not isinstance(data["data"], list):
|
||
errors.append(f"字段 'data' 类型错误: 期望 list, 实际 {type(data['data']).__name__}")
|
||
else:
|
||
for i, model in enumerate(data["data"]):
|
||
if not isinstance(model, dict):
|
||
errors.append(f"data[{i}] 不是对象")
|
||
continue
|
||
|
||
# 检查 model 对象的必需字段
|
||
model_required = ["id", "type", "display_name", "created_at"]
|
||
for field in model_required:
|
||
if field not in model:
|
||
errors.append(f"data[{i}] 缺少必需字段: {field}")
|
||
|
||
# 检查 type 字段值
|
||
if "type" in model and model["type"] != "model":
|
||
errors.append(f"data[{i}].type 值错误: 期望 'model', 实际 '{model['type']}'")
|
||
|
||
return len(errors) == 0, errors
|
||
|
||
|
||
def validate_anthropic_model_retrieve_response(response_text: str) -> Tuple[bool, List[str]]:
|
||
"""验证 Anthropic Model Retrieve 响应
|
||
|
||
根据API文档，响应应包含：
|
||
- id: string
|
||
- type: "model"
|
||
- display_name: string
|
||
- created_at: string
|
||
- max_input_tokens: number
|
||
- max_tokens: number
|
||
"""
|
||
required_fields = ["id", "type", "display_name", "created_at"]
|
||
field_types = {
|
||
"id": str,
|
||
"type": str,
|
||
"display_name": str,
|
||
"created_at": str
|
||
}
|
||
enum_values = {
|
||
"type": ["model"]
|
||
}
|
||
|
||
return validate_response_structure(response_text, required_fields, field_types, enum_values)
|
||
|
||
|
||
def validate_anthropic_messages_response(response_text: str) -> Tuple[bool, List[str]]:
|
||
"""验证 Anthropic Messages 响应
|
||
|
||
根据API文档，响应应包含：
|
||
- id: string
|
||
- type: "message"
|
||
- role: "assistant"
|
||
- content: array
|
||
- model: string
|
||
- stop_reason: string (可选)
|
||
- usage: object
|
||
"""
|
||
errors = []
|
||
|
||
try:
|
||
data = json.loads(response_text)
|
||
except json.JSONDecodeError as e:
|
||
return False, [f"响应不是有效的JSON: {e}"]
|
||
|
||
# 检查必需字段
|
||
required_fields = ["id", "type", "role", "content", "model"]
|
||
for field in required_fields:
|
||
if field not in data:
|
||
errors.append(f"缺少必需字段: {field}")
|
||
|
||
# 检查特定字段值
|
||
if "type" in data and data["type"] != "message":
|
||
errors.append(f"字段 'type' 值错误: 期望 'message', 实际 '{data['type']}'")
|
||
|
||
if "role" in data and data["role"] != "assistant":
|
||
errors.append(f"字段 'role' 值错误: 期望 'assistant', 实际 '{data['role']}'")
|
||
|
||
# 检查 content 数组
|
||
if "content" in data:
|
||
if not isinstance(data["content"], list):
|
||
errors.append(f"字段 'content' 类型错误: 期望 list, 实际 {type(data['content']).__name__}")
|
||
else:
|
||
for i, block in enumerate(data["content"]):
|
||
if not isinstance(block, dict):
|
||
errors.append(f"content[{i}] 不是对象")
|
||
continue
|
||
|
||
# 检查 content block 的必需字段
|
||
if "type" not in block:
|
||
errors.append(f"content[{i}] 缺少必需字段: type")
|
||
|
||
# 检查 usage 对象
|
||
if "usage" in data and data["usage"] is not None:
|
||
if not isinstance(data["usage"], dict):
|
||
errors.append(f"字段 'usage' 类型错误: 期望 object, 实际 {type(data['usage']).__name__}")
|
||
else:
|
||
usage_fields = ["input_tokens", "output_tokens"]
|
||
for field in usage_fields:
|
||
if field not in data["usage"]:
|
||
errors.append(f"usage 缺少必需字段: {field}")
|
||
|
||
return len(errors) == 0, errors
|
||
|
||
|
||
def validate_anthropic_count_tokens_response(response_text: str) -> Tuple[bool, List[str]]:
|
||
"""验证 Anthropic Count Tokens 响应
|
||
|
||
根据API文档，响应应包含：
|
||
- message_tokens_count: object { input_tokens }
|
||
"""
|
||
errors = []
|
||
|
||
try:
|
||
data = json.loads(response_text)
|
||
except json.JSONDecodeError as e:
|
||
return False, [f"响应不是有效的JSON: {e}"]
|
||
|
||
# 检查嵌套结构
|
||
if "message_tokens_count" not in data:
|
||
errors.append("缺少必需字段: message_tokens_count")
|
||
else:
|
||
mtc = data["message_tokens_count"]
|
||
if not isinstance(mtc, dict):
|
||
errors.append(f"字段 'message_tokens_count' 类型错误: 期望 object, 实际 {type(mtc).__name__}")
|
||
else:
|
||
if "input_tokens" not in mtc:
|
||
errors.append("message_tokens_count 缺少必需字段: input_tokens")
|
||
elif not isinstance(mtc["input_tokens"], (int, float)):
|
||
errors.append(f"message_tokens_count.input_tokens 类型错误: 期望 number, 实际 {type(mtc['input_tokens']).__name__}")
|
||
|
||
return len(errors) == 0, errors
|
||
|
||
|
||
def validate_anthropic_streaming_response(response_text: str) -> Tuple[bool, List[str]]:
|
||
"""验证 Anthropic 流式响应
|
||
|
||
流式响应使用 SSE 格式，每行以 "data: " 开头。
|
||
事件类型包括：message_start, content_block_start, content_block_delta, content_block_stop, message_delta, message_stop
|
||
|
||
验证要点：
|
||
- 每个事件是有效的 JSON
|
||
- 包含 message_start 和 message_stop 事件
|
||
- message_start 事件包含完整的 message 对象
|
||
|
||
Args:
|
||
response_text: SSE 格式的响应文本
|
||
|
||
Returns:
|
||
(是否验证通过, 错误信息列表)
|
||
"""
|
||
from core import parse_sse_events
|
||
|
||
errors = []
|
||
events = parse_sse_events(response_text)
|
||
|
||
if not events:
|
||
errors.append("未收到任何 SSE 事件")
|
||
return False, errors
|
||
|
||
has_message_start = False
|
||
has_message_stop = False
|
||
|
||
for i, event_data in enumerate(events):
|
||
try:
|
||
event = json.loads(event_data)
|
||
except json.JSONDecodeError as e:
|
||
errors.append(f"事件[{i}] 不是有效的JSON: {e}")
|
||
continue
|
||
|
||
if "type" not in event:
|
||
errors.append(f"事件[{i}] 缺少必需字段: type")
|
||
continue
|
||
|
||
event_type = event["type"]
|
||
|
||
if event_type == "message_start":
|
||
has_message_start = True
|
||
if "message" not in event:
|
||
errors.append(f"message_start 事件缺少 message 字段")
|
||
elif not isinstance(event["message"], dict):
|
||
errors.append(f"message_start 事件的 message 不是对象")
|
||
else:
|
||
msg = event["message"]
|
||
if "id" not in msg:
|
||
errors.append(f"message_start.message 缺少 id 字段")
|
||
if "type" not in msg:
|
||
errors.append(f"message_start.message 缺少 type 字段")
|
||
elif msg["type"] != "message":
|
||
errors.append(f"message_start.message.type 值错误: 期望 'message', 实际 '{msg['type']}'")
|
||
if "role" not in msg:
|
||
errors.append(f"message_start.message 缺少 role 字段")
|
||
elif msg["role"] != "assistant":
|
||
errors.append(f"message_start.message.role 值错误: 期望 'assistant', 实际 '{msg['role']}'")
|
||
if "content" not in msg:
|
||
errors.append(f"message_start.message 缺少 content 字段")
|
||
elif not isinstance(msg["content"], list):
|
||
errors.append(f"message_start.message.content 类型错误: 期望 list")
|
||
|
||
elif event_type == "message_stop":
|
||
has_message_stop = True
|
||
|
||
elif event_type == "content_block_start":
|
||
if "index" not in event:
|
||
errors.append(f"content_block_start 事件缺少 index 字段")
|
||
if "content_block" not in event:
|
||
errors.append(f"content_block_start 事件缺少 content_block 字段")
|
||
elif not isinstance(event["content_block"], dict):
|
||
errors.append(f"content_block_start 事件的 content_block 不是对象")
|
||
else:
|
||
cb = event["content_block"]
|
||
if "type" not in cb:
|
||
errors.append(f"content_block_start.content_block 缺少 type 字段")
|
||
|
||
elif event_type == "content_block_delta":
|
||
if "delta" not in event:
|
||
errors.append(f"content_block_delta 事件缺少 delta 字段")
|
||
|
||
if not has_message_start:
|
||
errors.append("缺少 message_start 事件")
|
||
if not has_message_stop:
|
||
errors.append("缺少 message_stop 事件")
|
||
|
||
return len(errors) == 0, errors
|
||
|
||
|
||
def main():
|
||
parser = argparse.ArgumentParser(
|
||
description="Anthropic 兼容性接口测试",
|
||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||
)
|
||
parser.add_argument("--base_url", required=True, help="API 基础地址 (如 https://api.example.com)")
|
||
parser.add_argument("--api_key", default="", help="API 密钥 (默认空)")
|
||
parser.add_argument("--model", default="claude-sonnet-4-5", help="模型名称 (默认 claude-sonnet-4-5)")
|
||
parser.add_argument("--vision", action="store_true", help="执行视觉相关测试")
|
||
parser.add_argument("--stream", action="store_true", help="执行流式响应测试")
|
||
parser.add_argument("--tools", action="store_true", help="执行工具调用测试")
|
||
parser.add_argument("--thinking", action="store_true", help="执行扩展思维测试")
|
||
parser.add_argument("--all", action="store_true", help="开启所有扩展测试")
|
||
args = parser.parse_args()
|
||
|
||
if args.all:
|
||
args.vision = True
|
||
args.stream = True
|
||
args.tools = True
|
||
args.thinking = True
|
||
|
||
base_url = args.base_url.rstrip("/")
|
||
api_key = args.api_key
|
||
model = args.model
|
||
ssl_ctx = create_ssl_context()
|
||
headers = build_headers(api_key)
|
||
headers_bad_auth = build_headers("invalid-key-xxx")
|
||
|
||
messages_url = f"{base_url}/v1/messages"
|
||
models_url = f"{base_url}/v1/models"
|
||
count_tokens_url = f"{base_url}/v1/messages/count_tokens"
|
||
|
||
# ---- 共享定义（供流式和非流式用例共同使用）----
|
||
image_url = (
|
||
"https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/"
|
||
"Gfp-wisconsin-madison-the-nature-boardwalk.jpg/"
|
||
"2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
|
||
)
|
||
tool_weather = {
|
||
"name": "get_weather",
|
||
"description": "获取指定城市的天气",
|
||
"input_schema": {
|
||
"type": "object",
|
||
"properties": {
|
||
"location": {"type": "string", "description": "城市名称"}
|
||
},
|
||
"required": ["location"]
|
||
}
|
||
}
|
||
|
||
# --- 收集测试用例 ---
|
||
cases: List[TestCase] = []
|
||
|
||
# ==== Models API ====
|
||
cases.append(TestCase(
|
||
desc="获取模型列表 (GET /v1/models)",
|
||
method="GET",
|
||
url=models_url,
|
||
headers=headers,
|
||
validator=validate_anthropic_models_list_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="获取模型列表（分页 limit=3）(GET /v1/models?limit=3)",
|
||
method="GET",
|
||
url=f"{models_url}?limit=3",
|
||
headers=headers,
|
||
validator=validate_anthropic_models_list_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="获取指定模型详情 (GET /v1/models/{model})",
|
||
method="GET",
|
||
url=f"{models_url}/{model}",
|
||
headers=headers,
|
||
validator=validate_anthropic_model_retrieve_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="获取不存在的模型 (GET /v1/models/nonexistent-model-xxx)",
|
||
method="GET",
|
||
url=f"{models_url}/nonexistent-model-xxx",
|
||
headers=headers
|
||
))
|
||
|
||
# ==== Messages API: 正面用例 ====
|
||
cases.append(TestCase(
|
||
desc="基本对话（仅 user）",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 5,
|
||
"messages": [{"role": "user", "content": "Hi"}]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="system prompt + user 对话",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 5,
|
||
"system": "You are a helpful assistant.",
|
||
"messages": [{"role": "user", "content": "1+1="}]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="system prompt 数组格式（带缓存控制）",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 5,
|
||
"system": [
|
||
{"type": "text", "text": "You are a helpful assistant.", "cache_control": {"type": "ephemeral"}}
|
||
],
|
||
"messages": [{"role": "user", "content": "Hi"}]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="多轮对话（含 assistant 历史）",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 5,
|
||
"messages": [
|
||
{"role": "user", "content": "Hi"},
|
||
{"role": "assistant", "content": "Hello!"},
|
||
{"role": "user", "content": "1+1="}
|
||
]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="assistant prefill（部分回复填充）",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 1,
|
||
"messages": [
|
||
{"role": "user", "content": "What is latin for Ant? (A) Apoidea (B) Rhopalocera (C) Formicidae"},
|
||
{"role": "assistant", "content": "The answer is ("}
|
||
]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="content 数组格式（多个 text block）",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 5,
|
||
"messages": [{"role": "user", "content": [
|
||
{"type": "text", "text": "Hello"},
|
||
{"type": "text", "text": "1+1=?"}
|
||
]}]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="temperature + top_p",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 5,
|
||
"temperature": 0.5,
|
||
"top_p": 0.9,
|
||
"messages": [{"role": "user", "content": "Hi"}]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="temperature = 0（类确定性输出）",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 5,
|
||
"temperature": 0,
|
||
"messages": [{"role": "user", "content": "1+1="}]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="top_k 参数",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 5,
|
||
"top_k": 40,
|
||
"messages": [{"role": "user", "content": "Hi"}]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="max_tokens 限制",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 10,
|
||
"messages": [{"role": "user", "content": "讲一个故事"}]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="stop_sequences",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 20,
|
||
"stop_sequences": ["5"],
|
||
"messages": [{"role": "user", "content": "数数: 1,2,3,"}]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="metadata 参数（user_id）",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 5,
|
||
"metadata": {"user_id": "test-user-001"},
|
||
"messages": [{"role": "user", "content": "Hi"}]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="assistant content 数组格式（text + tool_use 块）",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 20,
|
||
"messages": [
|
||
{"role": "user", "content": "帮我查一下北京的天气"},
|
||
{"role": "assistant", "content": [
|
||
{"type": "text", "text": "好的，让我查一下。"},
|
||
{"type": "tool_use", "id": "toolu_prev_001", "name": "get_weather", "input": {"location": "Beijing"}}
|
||
]},
|
||
{"role": "user", "content": [
|
||
{"type": "tool_result", "tool_use_id": "toolu_prev_001", "content": [
|
||
{"type": "text", "text": "{\"temperature\": 22, \"condition\": \"晴\"}"}
|
||
]}
|
||
]}
|
||
]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
|
||
# ==== Count Tokens API ====
|
||
cases.append(TestCase(
|
||
desc="计数 Token (POST /v1/messages/count_tokens)",
|
||
method="POST",
|
||
url=count_tokens_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"messages": [{"role": "user", "content": "Hello, how are you?"}]
|
||
},
|
||
validator=validate_anthropic_count_tokens_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="计数 Token（带 system + tools）",
|
||
method="POST",
|
||
url=count_tokens_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"system": "You are a helpful assistant.",
|
||
"messages": [{"role": "user", "content": "Hi"}],
|
||
"tools": [{
|
||
"name": "get_weather",
|
||
"description": "获取天气",
|
||
"input_schema": {
|
||
"type": "object",
|
||
"properties": {"location": {"type": "string"}},
|
||
"required": ["location"]
|
||
}
|
||
}]
|
||
}
|
||
))
|
||
cases.append(TestCase(
|
||
desc="计数 Token 缺少 model（负面）",
|
||
method="POST",
|
||
url=count_tokens_url,
|
||
headers=headers,
|
||
body={
|
||
"messages": [{"role": "user", "content": "Hi"}]
|
||
}
|
||
))
|
||
|
||
# ==== Messages API: 负面用例 ====
|
||
cases.append(TestCase(
|
||
desc="缺少 x-api-key header（无认证）",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers={
|
||
"Content-Type": "application/json",
|
||
"anthropic-version": ANTHROPIC_VERSION,
|
||
},
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 5,
|
||
"messages": [{"role": "user", "content": "Hi"}]
|
||
}
|
||
))
|
||
cases.append(TestCase(
|
||
desc="错误的 anthropic-version header",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers={
|
||
"Content-Type": "application/json",
|
||
"anthropic-version": "0000-00-00",
|
||
"x-api-key": api_key,
|
||
} if api_key else {
|
||
"Content-Type": "application/json",
|
||
"anthropic-version": "0000-00-00",
|
||
},
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 5,
|
||
"messages": [{"role": "user", "content": "Hi"}]
|
||
}
|
||
))
|
||
cases.append(TestCase(
|
||
desc="缺少 model 参数",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"max_tokens": 5,
|
||
"messages": [{"role": "user", "content": "Hi"}]
|
||
}
|
||
))
|
||
cases.append(TestCase(
|
||
desc="缺少 messages 参数",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 5
|
||
}
|
||
))
|
||
cases.append(TestCase(
|
||
desc="缺少 max_tokens 参数",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"messages": [{"role": "user", "content": "Hi"}]
|
||
}
|
||
))
|
||
cases.append(TestCase(
|
||
desc="messages 为空数组",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 5,
|
||
"messages": []
|
||
}
|
||
))
|
||
cases.append(TestCase(
|
||
desc="无效 API key",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers_bad_auth,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 5,
|
||
"messages": [{"role": "user", "content": "Hi"}]
|
||
}
|
||
))
|
||
cases.append(TestCase(
|
||
desc="不存在的模型",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": "nonexistent-model-xxx",
|
||
"max_tokens": 5,
|
||
"messages": [{"role": "user", "content": "Hi"}]
|
||
}
|
||
))
|
||
cases.append(TestCase(
|
||
desc="畸形 JSON body",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body="invalid json{"
|
||
))
|
||
cases.append(TestCase(
|
||
desc="无效 role（非法消息角色）",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 5,
|
||
"messages": [{"role": "system", "content": "You are helpful"}]
|
||
}
|
||
))
|
||
cases.append(TestCase(
|
||
desc="max_tokens 为负数",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": -1,
|
||
"messages": [{"role": "user", "content": "Hi"}]
|
||
}
|
||
))
|
||
cases.append(TestCase(
|
||
desc="max_tokens = 0",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 0,
|
||
"messages": [{"role": "user", "content": "Hi"}]
|
||
}
|
||
))
|
||
cases.append(TestCase(
|
||
desc="temperature 超出范围 (2.0)",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 5,
|
||
"temperature": 2.0,
|
||
"messages": [{"role": "user", "content": "Hi"}]
|
||
}
|
||
))
|
||
|
||
# ==== --vision ====
|
||
if args.vision:
|
||
cases.append(TestCase(
|
||
desc="图片 URL 输入 (--vision)",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 10,
|
||
"messages": [{"role": "user", "content": [
|
||
{"type": "text", "text": "用一个词描述这张图"},
|
||
{"type": "image", "source": {
|
||
"type": "url",
|
||
"url": image_url
|
||
}}
|
||
]}]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
|
||
# ==== --stream ====
|
||
if args.stream:
|
||
# 核心用例
|
||
cases.append(TestCase(
|
||
desc="流式基本对话",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={"model": model, "max_tokens": 5, "stream": True, "messages": [{"role": "user", "content": "Hi"}]},
|
||
stream=True,
|
||
validator=validate_anthropic_streaming_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="流式 + system prompt",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={"model": model, "max_tokens": 5, "stream": True, "system": "有帮助的助手", "messages": [{"role": "user", "content": "Hi"}]},
|
||
stream=True,
|
||
validator=validate_anthropic_streaming_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="流式多轮对话",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={"model": model, "max_tokens": 5, "stream": True, "messages": [{"role": "user", "content": "Hi"}, {"role": "assistant", "content": "Hello"}, {"role": "user", "content": "1+1"}]},
|
||
stream=True,
|
||
validator=validate_anthropic_streaming_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="流式 temperature + top_p",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={"model": model, "max_tokens": 5, "stream": True, "temperature": 0.5, "top_p": 0.9, "messages": [{"role": "user", "content": "Hi"}]},
|
||
stream=True,
|
||
validator=validate_anthropic_streaming_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="流式 max_tokens",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={"model": model, "max_tokens": 3, "stream": True, "messages": [{"role": "user", "content": "Hi"}]},
|
||
stream=True,
|
||
validator=validate_anthropic_streaming_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="流式 stop_sequences",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={"model": model, "max_tokens": 10, "stream": True, "stop_sequences": ["5"], "messages": [{"role": "user", "content": "数数: 1,2,3,"}]},
|
||
stream=True,
|
||
validator=validate_anthropic_streaming_response
|
||
))
|
||
|
||
# 流式 + vision
|
||
if args.vision:
|
||
cases.append(TestCase(
|
||
desc="流式图片输入",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={"model": model, "max_tokens": 10, "stream": True, "messages": [{"role": "user", "content": [{"type": "text", "text": "描述图"}, {"type": "image", "source": {"type": "url", "url": image_url}}]}]},
|
||
stream=True,
|
||
validator=validate_anthropic_streaming_response
|
||
))
|
||
|
||
# 流式 + tools
|
||
if args.tools:
|
||
cases.append(TestCase(
|
||
desc="流式工具调用 auto",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={"model": model, "max_tokens": 50, "stream": True, "tools": [tool_weather], "tool_choice": {"type": "auto"}, "messages": [{"role": "user", "content": "北京天气？"}]},
|
||
stream=True,
|
||
validator=validate_anthropic_streaming_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="流式多轮工具调用",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={"model": model, "max_tokens": 20, "stream": True, "tools": [tool_weather], "messages": [{"role": "user", "content": "北京天气？"}, {"role": "assistant", "content": [{"type": "tool_use", "id": "toolu_001", "name": "get_weather", "input": {"location": "Beijing"}}]}, {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_001", "content": '{"temp": 22}'}]}]},
|
||
stream=True,
|
||
validator=validate_anthropic_streaming_response
|
||
))
|
||
|
||
# 流式 + thinking
|
||
if args.thinking:
|
||
cases.append(TestCase(
|
||
desc="流式扩展思维",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={"model": model, "max_tokens": 100, "stream": True, "thinking": {"type": "enabled", "budget_tokens": 50}, "messages": [{"role": "user", "content": "1+1=?"}]},
|
||
stream=True,
|
||
validator=validate_anthropic_streaming_response
|
||
))
|
||
|
||
# 流式高级参数
|
||
cases.append(TestCase(
|
||
desc="流式 service_tier: auto",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={"model": model, "max_tokens": 5, "stream": True, "service_tier": "auto", "messages": [{"role": "user", "content": "Hi"}]},
|
||
stream=True,
|
||
validator=validate_anthropic_streaming_response
|
||
))
|
||
|
||
# ==== --tools ====
|
||
if args.tools:
|
||
cases.append(TestCase(
|
||
desc="工具调用 tool_choice: auto (--tools)",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 50,
|
||
"tools": [tool_weather],
|
||
"tool_choice": {"type": "auto"},
|
||
"messages": [{"role": "user", "content": "北京天气怎么样？"}]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="工具调用 tool_choice: any (--tools)",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 50,
|
||
"tools": [tool_weather],
|
||
"tool_choice": {"type": "any"},
|
||
"messages": [{"role": "user", "content": "北京天气怎么样？"}]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="指定工具调用 tool_choice: {name} (--tools)",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 50,
|
||
"tools": [tool_weather],
|
||
"tool_choice": {"type": "tool", "name": "get_weather"},
|
||
"messages": [{"role": "user", "content": "北京天气怎么样？"}]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="tool_choice: none (--tools)",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 20,
|
||
"tools": [tool_weather],
|
||
"tool_choice": {"type": "none"},
|
||
"messages": [{"role": "user", "content": "北京天气怎么样？"}]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="多轮工具调用（tool_result 返回）(--tools)",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 20,
|
||
"tools": [tool_weather],
|
||
"messages": [
|
||
{"role": "user", "content": "北京天气怎么样？"},
|
||
{"role": "assistant", "content": [
|
||
{"type": "text", "text": "让我查一下。"},
|
||
{"type": "tool_use", "id": "toolu_001", "name": "get_weather", "input": {"location": "Beijing"}}
|
||
]},
|
||
{"role": "user", "content": [
|
||
{"type": "tool_result", "tool_use_id": "toolu_001", "content": "{\"temperature\": 22, \"condition\": \"晴\"}"}
|
||
]}
|
||
]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="多轮工具调用（tool_result 带 is_error）(--tools)",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 20,
|
||
"tools": [tool_weather],
|
||
"messages": [
|
||
{"role": "user", "content": "北京天气怎么样？"},
|
||
{"role": "assistant", "content": [
|
||
{"type": "tool_use", "id": "toolu_002", "name": "get_weather", "input": {"location": "Beijing"}}
|
||
]},
|
||
{"role": "user", "content": [
|
||
{"type": "tool_result", "tool_use_id": "toolu_002", "is_error": True, "content": "天气服务不可用"}
|
||
]}
|
||
]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="tool_choice 指向不存在的工具（负面）(--tools)",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 50,
|
||
"tools": [tool_weather],
|
||
"tool_choice": {"type": "tool", "name": "nonexistent_tool_xxx"},
|
||
"messages": [{"role": "user", "content": "北京天气怎么样？"}]
|
||
}
|
||
))
|
||
cases.append(TestCase(
|
||
desc="多工具定义 (--tools)",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 50,
|
||
"tools": [
|
||
tool_weather,
|
||
{
|
||
"name": "get_time",
|
||
"description": "获取指定城市的当前时间",
|
||
"input_schema": {
|
||
"type": "object",
|
||
"properties": {
|
||
"location": {"type": "string", "description": "城市名称"}
|
||
},
|
||
"required": ["location"]
|
||
}
|
||
}
|
||
],
|
||
"tool_choice": {"type": "auto"},
|
||
"messages": [{"role": "user", "content": "北京现在几点了？天气怎么样？"}]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
|
||
# ==== --thinking ====
|
||
if args.thinking:
|
||
cases.append(TestCase(
|
||
desc="扩展思维 enabled (--thinking)",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 200,
|
||
"thinking": {"type": "enabled", "budget_tokens": 100},
|
||
"messages": [{"role": "user", "content": "1+1=?"}]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="扩展思维 adaptive (--thinking)",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 200,
|
||
"thinking": {"type": "adaptive", "budget_tokens": 100},
|
||
"messages": [{"role": "user", "content": "1+1=?"}]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
|
||
# ==== 高级参数测试 ====
|
||
# cache_control: 缓存控制
|
||
cases.append(TestCase(
|
||
desc="cache_control 缓存控制",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 10,
|
||
"cache_control": {"type": "ephemeral"},
|
||
"messages": [{"role": "user", "content": "Hello"}]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
|
||
# output_config: 输出配置
|
||
cases.append(TestCase(
|
||
desc="output_config 输出配置",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 10,
|
||
"output_config": {"format": "text"},
|
||
"messages": [{"role": "user", "content": "Hi"}]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="output_config 带 effort",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 10,
|
||
"output_config": {"format": "text", "effort": "low"},
|
||
"messages": [{"role": "user", "content": "Hi"}]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
|
||
# service_tier: 服务层级
|
||
cases.append(TestCase(
|
||
desc="service_tier: auto",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 5,
|
||
"service_tier": "auto",
|
||
"messages": [{"role": "user", "content": "Hello"}]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
cases.append(TestCase(
|
||
desc="service_tier: standard_only",
|
||
method="POST",
|
||
url=messages_url,
|
||
headers=headers,
|
||
body={
|
||
"model": model,
|
||
"max_tokens": 5,
|
||
"service_tier": "standard_only",
|
||
"messages": [{"role": "user", "content": "Hello"}]
|
||
},
|
||
validator=validate_anthropic_messages_response
|
||
))
|
||
|
||
# ==== Models API 分页测试 ====
|
||
cases.append(TestCase(
|
||
desc="Models API 分页 limit=5",
|
||
method="GET",
|
||
url=f"{models_url}?limit=5",
|
||
headers=headers
|
||
))
|
||
|
||
# ==== 执行测试 ====
|
||
flags = []
|
||
if args.vision:
|
||
flags.append("vision")
|
||
if args.stream:
|
||
flags.append("stream")
|
||
if args.tools:
|
||
flags.append("tools")
|
||
if args.thinking:
|
||
flags.append("thinking")
|
||
|
||
run_test_suite(
|
||
cases=cases,
|
||
ssl_ctx=ssl_ctx,
|
||
title="Anthropic 兼容性测试",
|
||
base_url=base_url,
|
||
model=model,
|
||
flags=flags
|
||
)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|