feat: 优化兼容性检测脚本
- 重命名脚本为 detect_xxx.py 格式 - 移除所有装饰线，精简输出格式 - 请求/响应输出增加 URL/Headers/入参/响应 标题标记 - 为所有正面用例添加响应验证器 - 补充 OpenAI 版缺失的负面测试（max_tokens 负数/0、temperature 越界） - 移除未使用的 format_validation_errors 导入 - 新增 scripts/README.md 文档
This commit is contained in:
984
scripts/detect_anthropic.py
Normal file
984
scripts/detect_anthropic.py
Normal file
@@ -0,0 +1,984 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Anthropic 兼容性接口测试脚本
|
||||
|
||||
用法:
|
||||
python3 scripts/anthropic_detect.py --base_url <url> [options]
|
||||
|
||||
示例:
|
||||
python3 scripts/anthropic_detect.py --base_url https://api.example.com
|
||||
python3 scripts/anthropic_detect.py --base_url https://api.example.com --api_key sk-xxx --model claude-sonnet-4-5
|
||||
python3 scripts/anthropic_detect.py --base_url https://api.example.com --stream --tools --vision
|
||||
"""
|
||||
|
||||
import json
|
||||
import argparse
|
||||
from typing import Dict, List, Tuple, Any
|
||||
from core import (
|
||||
create_ssl_context,
|
||||
TestCase,
|
||||
run_test_suite,
|
||||
validate_response_structure,
|
||||
)
|
||||
|
||||
ANTHROPIC_VERSION = "2023-06-01"
|
||||
|
||||
|
||||
def build_headers(api_key: str) -> Dict[str, str]:
|
||||
"""构建 Anthropic API 请求头"""
|
||||
h = {
|
||||
"Content-Type": "application/json",
|
||||
"anthropic-version": ANTHROPIC_VERSION,
|
||||
}
|
||||
if api_key:
|
||||
h["x-api-key"] = api_key
|
||||
return h
|
||||
|
||||
|
||||
# ==================== Anthropic 响应验证函数 ====================
|
||||
|
||||
def validate_anthropic_models_list_response(response_text: str) -> Tuple[bool, List[str]]:
|
||||
"""验证 Anthropic Models List 响应
|
||||
|
||||
根据API文档，响应应包含：
|
||||
- data: array of ModelInfo
|
||||
- first_id: string (可选)
|
||||
- has_more: boolean
|
||||
- last_id: string (可选)
|
||||
"""
|
||||
errors = []
|
||||
|
||||
try:
|
||||
data = json.loads(response_text)
|
||||
except json.JSONDecodeError as e:
|
||||
return False, [f"响应不是有效的JSON: {e}"]
|
||||
|
||||
# 检查必需字段
|
||||
required_fields = ["data", "has_more"]
|
||||
for field in required_fields:
|
||||
if field not in data:
|
||||
errors.append(f"缺少必需字段: {field}")
|
||||
|
||||
# 检查 data 数组
|
||||
if "data" in data:
|
||||
if not isinstance(data["data"], list):
|
||||
errors.append(f"字段 'data' 类型错误: 期望 list, 实际 {type(data['data']).__name__}")
|
||||
else:
|
||||
for i, model in enumerate(data["data"]):
|
||||
if not isinstance(model, dict):
|
||||
errors.append(f"data[{i}] 不是对象")
|
||||
continue
|
||||
|
||||
# 检查 model 对象的必需字段
|
||||
model_required = ["id", "type", "display_name", "created_at"]
|
||||
for field in model_required:
|
||||
if field not in model:
|
||||
errors.append(f"data[{i}] 缺少必需字段: {field}")
|
||||
|
||||
# 检查 type 字段值
|
||||
if "type" in model and model["type"] != "model":
|
||||
errors.append(f"data[{i}].type 值错误: 期望 'model', 实际 '{model['type']}'")
|
||||
|
||||
return len(errors) == 0, errors
|
||||
|
||||
|
||||
def validate_anthropic_model_retrieve_response(response_text: str) -> Tuple[bool, List[str]]:
|
||||
"""验证 Anthropic Model Retrieve 响应
|
||||
|
||||
根据API文档，响应应包含：
|
||||
- id: string
|
||||
- type: "model"
|
||||
- display_name: string
|
||||
- created_at: string
|
||||
- max_input_tokens: number
|
||||
- max_tokens: number
|
||||
"""
|
||||
required_fields = ["id", "type", "display_name", "created_at"]
|
||||
field_types = {
|
||||
"id": str,
|
||||
"type": str,
|
||||
"display_name": str,
|
||||
"created_at": str
|
||||
}
|
||||
enum_values = {
|
||||
"type": ["model"]
|
||||
}
|
||||
|
||||
return validate_response_structure(response_text, required_fields, field_types, enum_values)
|
||||
|
||||
|
||||
def validate_anthropic_messages_response(response_text: str) -> Tuple[bool, List[str]]:
|
||||
"""验证 Anthropic Messages 响应
|
||||
|
||||
根据API文档，响应应包含：
|
||||
- id: string
|
||||
- type: "message"
|
||||
- role: "assistant"
|
||||
- content: array
|
||||
- model: string
|
||||
- stop_reason: string (可选)
|
||||
- usage: object
|
||||
"""
|
||||
errors = []
|
||||
|
||||
try:
|
||||
data = json.loads(response_text)
|
||||
except json.JSONDecodeError as e:
|
||||
return False, [f"响应不是有效的JSON: {e}"]
|
||||
|
||||
# 检查必需字段
|
||||
required_fields = ["id", "type", "role", "content", "model"]
|
||||
for field in required_fields:
|
||||
if field not in data:
|
||||
errors.append(f"缺少必需字段: {field}")
|
||||
|
||||
# 检查特定字段值
|
||||
if "type" in data and data["type"] != "message":
|
||||
errors.append(f"字段 'type' 值错误: 期望 'message', 实际 '{data['type']}'")
|
||||
|
||||
if "role" in data and data["role"] != "assistant":
|
||||
errors.append(f"字段 'role' 值错误: 期望 'assistant', 实际 '{data['role']}'")
|
||||
|
||||
# 检查 content 数组
|
||||
if "content" in data:
|
||||
if not isinstance(data["content"], list):
|
||||
errors.append(f"字段 'content' 类型错误: 期望 list, 实际 {type(data['content']).__name__}")
|
||||
else:
|
||||
for i, block in enumerate(data["content"]):
|
||||
if not isinstance(block, dict):
|
||||
errors.append(f"content[{i}] 不是对象")
|
||||
continue
|
||||
|
||||
# 检查 content block 的必需字段
|
||||
if "type" not in block:
|
||||
errors.append(f"content[{i}] 缺少必需字段: type")
|
||||
|
||||
# 检查 usage 对象
|
||||
if "usage" in data and data["usage"] is not None:
|
||||
if not isinstance(data["usage"], dict):
|
||||
errors.append(f"字段 'usage' 类型错误: 期望 object, 实际 {type(data['usage']).__name__}")
|
||||
else:
|
||||
usage_fields = ["input_tokens", "output_tokens"]
|
||||
for field in usage_fields:
|
||||
if field not in data["usage"]:
|
||||
errors.append(f"usage 缺少必需字段: {field}")
|
||||
|
||||
return len(errors) == 0, errors
|
||||
|
||||
|
||||
def validate_anthropic_count_tokens_response(response_text: str) -> Tuple[bool, List[str]]:
|
||||
"""验证 Anthropic Count Tokens 响应
|
||||
|
||||
根据API文档，响应应包含：
|
||||
- input_tokens: number
|
||||
"""
|
||||
required_fields = ["input_tokens"]
|
||||
field_types = {
|
||||
"input_tokens": (int, float)
|
||||
}
|
||||
|
||||
return validate_response_structure(response_text, required_fields, field_types)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Anthropic 兼容性接口测试",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
parser.add_argument("--base_url", required=True, help="API 基础地址 (如 https://api.example.com)")
|
||||
parser.add_argument("--api_key", default="", help="API 密钥 (默认空)")
|
||||
parser.add_argument("--model", default="claude-sonnet-4-5", help="模型名称 (默认 claude-sonnet-4-5)")
|
||||
parser.add_argument("--vision", action="store_true", help="执行视觉相关测试")
|
||||
parser.add_argument("--stream", action="store_true", help="执行流式响应测试")
|
||||
parser.add_argument("--tools", action="store_true", help="执行工具调用测试")
|
||||
parser.add_argument("--thinking", action="store_true", help="执行扩展思维测试")
|
||||
parser.add_argument("--all", action="store_true", help="开启所有扩展测试")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.all:
|
||||
args.vision = True
|
||||
args.stream = True
|
||||
args.tools = True
|
||||
args.thinking = True
|
||||
|
||||
base_url = args.base_url.rstrip("/")
|
||||
api_key = args.api_key
|
||||
model = args.model
|
||||
ssl_ctx = create_ssl_context()
|
||||
headers = build_headers(api_key)
|
||||
headers_bad_auth = build_headers("invalid-key-xxx")
|
||||
|
||||
messages_url = f"{base_url}/v1/messages"
|
||||
models_url = f"{base_url}/v1/models"
|
||||
count_tokens_url = f"{base_url}/v1/messages/count_tokens"
|
||||
|
||||
# --- 收集测试用例 ---
|
||||
cases: List[TestCase] = []
|
||||
|
||||
# ==== Models API ====
|
||||
cases.append(TestCase(
|
||||
desc="获取模型列表 (GET /v1/models)",
|
||||
method="GET",
|
||||
url=models_url,
|
||||
headers=headers,
|
||||
validator=validate_anthropic_models_list_response
|
||||
))
|
||||
cases.append(TestCase(
|
||||
desc="获取模型列表（分页 limit=3）(GET /v1/models?limit=3)",
|
||||
method="GET",
|
||||
url=f"{models_url}?limit=3",
|
||||
headers=headers,
|
||||
validator=validate_anthropic_models_list_response
|
||||
))
|
||||
cases.append(TestCase(
|
||||
desc="获取指定模型详情 (GET /v1/models/{model})",
|
||||
method="GET",
|
||||
url=f"{models_url}/{model}",
|
||||
headers=headers,
|
||||
validator=validate_anthropic_model_retrieve_response
|
||||
))
|
||||
cases.append(TestCase(
|
||||
desc="获取不存在的模型 (GET /v1/models/nonexistent-model-xxx)",
|
||||
method="GET",
|
||||
url=f"{models_url}/nonexistent-model-xxx",
|
||||
headers=headers
|
||||
))
|
||||
|
||||
# ==== Messages API: 正面用例 ====
|
||||
cases.append(TestCase(
|
||||
desc="基本对话（仅 user）",
|
||||
method="POST",
|
||||
url=messages_url,
|
||||
headers=headers,
|
||||
body={
|
||||
"model": model,
|
||||
"max_tokens": 5,
|
||||
"messages": [{"role": "user", "content": "Hi"}]
|
||||
},
|
||||
validator=validate_anthropic_messages_response
|
||||
))
|
||||
cases.append(TestCase(
|
||||
desc="system prompt + user 对话",
|
||||
method="POST",
|
||||
url=messages_url,
|
||||
headers=headers,
|
||||
body={
|
||||
"model": model,
|
||||
"max_tokens": 5,
|
||||
"system": "You are a helpful assistant.",
|
||||
"messages": [{"role": "user", "content": "1+1="}]
|
||||
},
|
||||
validator=validate_anthropic_messages_response
|
||||
))
|
||||
cases.append(TestCase(
|
||||
desc="system prompt 数组格式（带缓存控制）",
|
||||
method="POST",
|
||||
url=messages_url,
|
||||
headers=headers,
|
||||
body={
|
||||
"model": model,
|
||||
"max_tokens": 5,
|
||||
"system": [
|
||||
{"type": "text", "text": "You are a helpful assistant.", "cache_control": {"type": "ephemeral"}}
|
||||
],
|
||||
"messages": [{"role": "user", "content": "Hi"}]
|
||||
},
|
||||
validator=validate_anthropic_messages_response
|
||||
))
|
||||
cases.append(TestCase(
|
||||
desc="多轮对话（含 assistant 历史）",
|
||||
method="POST",
|
||||
url=messages_url,
|
||||
headers=headers,
|
||||
body={
|
||||
"model": model,
|
||||
"max_tokens": 5,
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hi"},
|
||||
{"role": "assistant", "content": "Hello!"},
|
||||
{"role": "user", "content": "1+1="}
|
||||
]
|
||||
},
|
||||
validator=validate_anthropic_messages_response
|
||||
))
|
||||
cases.append(TestCase(
|
||||
desc="assistant prefill（部分回复填充）",
|
||||
method="POST",
|
||||
url=messages_url,
|
||||
headers=headers,
|
||||
body={
|
||||
"model": model,
|
||||
"max_tokens": 1,
|
||||
"messages": [
|
||||
{"role": "user", "content": "What is latin for Ant? (A) Apoidea (B) Rhopalocera (C) Formicidae"},
|
||||
{"role": "assistant", "content": "The answer is ("}
|
||||
]
|
||||
},
|
||||
validator=validate_anthropic_messages_response
|
||||
))
|
||||
cases.append(TestCase(
|
||||
desc="content 数组格式（多个 text block）",
|
||||
method="POST",
|
||||
url=messages_url,
|
||||
headers=headers,
|
||||
body={
|
||||
"model": model,
|
||||
"max_tokens": 5,
|
||||
"messages": [{"role": "user", "content": [
|
||||
{"type": "text", "text": "Hello"},
|
||||
{"type": "text", "text": "1+1=?"}
|
||||
]}]
|
||||
},
|
||||
validator=validate_anthropic_messages_response
|
||||
))
|
||||
cases.append(TestCase(
|
||||
desc="temperature + top_p",
|
||||
method="POST",
|
||||
url=messages_url,
|
||||
headers=headers,
|
||||
body={
|
||||
"model": model,
|
||||
"max_tokens": 5,
|
||||
"temperature": 0.5,
|
||||
"top_p": 0.9,
|
||||
"messages": [{"role": "user", "content": "Hi"}]
|
||||
},
|
||||
validator=validate_anthropic_messages_response
|
||||
))
|
||||
cases.append(TestCase(
|
||||
desc="temperature = 0（类确定性输出）",
|
||||
method="POST",
|
||||
url=messages_url,
|
||||
headers=headers,
|
||||
body={
|
||||
"model": model,
|
||||
"max_tokens": 5,
|
||||
"temperature": 0,
|
||||
"messages": [{"role": "user", "content": "1+1="}]
|
||||
},
|
||||
validator=validate_anthropic_messages_response
|
||||
))
|
||||
cases.append(TestCase(
|
||||
desc="top_k 参数",
|
||||
method="POST",
|
||||
url=messages_url,
|
||||
headers=headers,
|
||||
body={
|
||||
"model": model,
|
||||
"max_tokens": 5,
|
||||
"top_k": 40,
|
||||
"messages": [{"role": "user", "content": "Hi"}]
|
||||
},
|
||||
validator=validate_anthropic_messages_response
|
||||
))
|
||||
cases.append(TestCase(
|
||||
desc="max_tokens 限制",
|
||||
method="POST",
|
||||
url=messages_url,
|
||||
headers=headers,
|
||||
body={
|
||||
"model": model,
|
||||
"max_tokens": 10,
|
||||
"messages": [{"role": "user", "content": "讲一个故事"}]
|
||||
},
|
||||
validator=validate_anthropic_messages_response
|
||||
))
|
||||
cases.append(TestCase(
|
||||
desc="stop_sequences",
|
||||
method="POST",
|
||||
url=messages_url,
|
||||
headers=headers,
|
||||
body={
|
||||
"model": model,
|
||||
"max_tokens": 20,
|
||||
"stop_sequences": ["5"],
|
||||
"messages": [{"role": "user", "content": "数数: 1,2,3,"}]
|
||||
},
|
||||
validator=validate_anthropic_messages_response
|
||||
))
|
||||
cases.append(TestCase(
|
||||
desc="metadata 参数（user_id）",
|
||||
method="POST",
|
||||
url=messages_url,
|
||||
headers=headers,
|
||||
body={
|
||||
"model": model,
|
||||
"max_tokens": 5,
|
||||
"metadata": {"user_id": "test-user-001"},
|
||||
"messages": [{"role": "user", "content": "Hi"}]
|
||||
},
|
||||
validator=validate_anthropic_messages_response
|
||||
))
|
||||
cases.append(TestCase(
|
||||
desc="assistant content 数组格式（text + tool_use 块）",
|
||||
method="POST",
|
||||
url=messages_url,
|
||||
headers=headers,
|
||||
body={
|
||||
"model": model,
|
||||
"max_tokens": 20,
|
||||
"messages": [
|
||||
{"role": "user", "content": "帮我查一下北京的天气"},
|
||||
{"role": "assistant", "content": [
|
||||
{"type": "text", "text": "好的，让我查一下。"},
|
||||
{"type": "tool_use", "id": "toolu_prev_001", "name": "get_weather", "input": {"location": "Beijing"}}
|
||||
]},
|
||||
{"role": "user", "content": [
|
||||
{"type": "tool_result", "tool_use_id": "toolu_prev_001", "content": [
|
||||
{"type": "text", "text": "{\"temperature\": 22, \"condition\": \"晴\"}"}
|
||||
]}
|
||||
]}
|
||||
]
|
||||
},
|
||||
validator=validate_anthropic_messages_response
|
||||
))
|
||||
|
||||
# ==== Count Tokens API ====
|
||||
cases.append(TestCase(
|
||||
desc="计数 Token (POST /v1/messages/count_tokens)",
|
||||
method="POST",
|
||||
url=count_tokens_url,
|
||||
headers=headers,
|
||||
body={
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": "Hello, how are you?"}]
|
||||
},
|
||||
validator=validate_anthropic_count_tokens_response
|
||||
))
|
||||
cases.append(TestCase(
|
||||
desc="计数 Token（带 system + tools）",
|
||||
method="POST",
|
||||
url=count_tokens_url,
|
||||
headers=headers,
|
||||
body={
|
||||
"model": model,
|
||||
"system": "You are a helpful assistant.",
|
||||
"messages": [{"role": "user", "content": "Hi"}],
|
||||
"tools": [{
|
||||
"name": "get_weather",
|
||||
"description": "获取天气",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {"location": {"type": "string"}},
|
||||
"required": ["location"]
|
||||
}
|
||||
}]
|
||||
}
|
||||
))
|
||||
cases.append(TestCase(
|
||||
desc="计数 Token 缺少 model（负面）",
|
||||
method="POST",
|
||||
url=count_tokens_url,
|
||||
headers=headers,
|
||||
body={
|
||||
"messages": [{"role": "user", "content": "Hi"}]
|
||||
}
|
||||
))
|
||||
|
||||
# ==== Messages API: 负面用例 ====
|
||||
cases.append(TestCase(
|
||||
desc="缺少 x-api-key header（无认证）",
|
||||
method="POST",
|
||||
url=messages_url,
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"anthropic-version": ANTHROPIC_VERSION,
|
||||
},
|
||||
body={
|
||||
"model": model,
|
||||
"max_tokens": 5,
|
||||
"messages": [{"role": "user", "content": "Hi"}]
|
||||
}
|
||||
))
|
||||
cases.append(TestCase(
|
||||
desc="错误的 anthropic-version header",
|
||||
method="POST",
|
||||
url=messages_url,
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"anthropic-version": "0000-00-00",
|
||||
"x-api-key": api_key,
|
||||
} if api_key else {
|
||||
"Content-Type": "application/json",
|
||||
"anthropic-version": "0000-00-00",
|
||||
},
|
||||
body={
|
||||
"model": model,
|
||||
"max_tokens": 5,
|
||||
"messages": [{"role": "user", "content": "Hi"}]
|
||||
}
|
||||
))
|
||||
cases.append(TestCase(
|
||||
desc="缺少 model 参数",
|
||||
method="POST",
|
||||
url=messages_url,
|
||||
headers=headers,
|
||||
body={
|
||||
"max_tokens": 5,
|
||||
"messages": [{"role": "user", "content": "Hi"}]
|
||||
}
|
||||
))
|
||||
cases.append(TestCase(
|
||||
desc="缺少 messages 参数",
|
||||
method="POST",
|
||||
url=messages_url,
|
||||
headers=headers,
|
||||
body={
|
||||
"model": model,
|
||||
"max_tokens": 5
|
||||
}
|
||||
))
|
||||
cases.append(TestCase(
|
||||
desc="缺少 max_tokens 参数",
|
||||
method="POST",
|
||||
url=messages_url,
|
||||
headers=headers,
|
||||
body={
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": "Hi"}]
|
||||
}
|
||||
))
|
||||
cases.append(TestCase(
|
||||
desc="messages 为空数组",
|
||||
method="POST",
|
||||
url=messages_url,
|
||||
headers=headers,
|
||||
body={
|
||||
"model": model,
|
||||
"max_tokens": 5,
|
||||
"messages": []
|
||||
}
|
||||
))
|
||||
cases.append(TestCase(
|
||||
desc="无效 API key",
|
||||
method="POST",
|
||||
url=messages_url,
|
||||
headers=headers_bad_auth,
|
||||
body={
|
||||
"model": model,
|
||||
"max_tokens": 5,
|
||||
"messages": [{"role": "user", "content": "Hi"}]
|
||||
}
|
||||
))
|
||||
cases.append(TestCase(
|
||||
desc="不存在的模型",
|
||||
method="POST",
|
||||
url=messages_url,
|
||||
headers=headers,
|
||||
body={
|
||||
"model": "nonexistent-model-xxx",
|
||||
"max_tokens": 5,
|
||||
"messages": [{"role": "user", "content": "Hi"}]
|
||||
}
|
||||
))
|
||||
cases.append(TestCase(
|
||||
desc="畸形 JSON body",
|
||||
method="POST",
|
||||
url=messages_url,
|
||||
headers=headers,
|
||||
body="invalid json{"
|
||||
))
|
||||
cases.append(TestCase(
|
||||
desc="无效 role（非法消息角色）",
|
||||
method="POST",
|
||||
url=messages_url,
|
||||
headers=headers,
|
||||
body={
|
||||
"model": model,
|
||||
"max_tokens": 5,
|
||||
"messages": [{"role": "system", "content": "You are helpful"}]
|
||||
}
|
||||
))
|
||||
cases.append(TestCase(
|
||||
desc="max_tokens 为负数",
|
||||
method="POST",
|
||||
url=messages_url,
|
||||
headers=headers,
|
||||
body={
|
||||
"model": model,
|
||||
"max_tokens": -1,
|
||||
"messages": [{"role": "user", "content": "Hi"}]
|
||||
}
|
||||
))
|
||||
cases.append(TestCase(
|
||||
desc="max_tokens = 0",
|
||||
method="POST",
|
||||
url=messages_url,
|
||||
headers=headers,
|
||||
body={
|
||||
"model": model,
|
||||
"max_tokens": 0,
|
||||
"messages": [{"role": "user", "content": "Hi"}]
|
||||
}
|
||||
))
|
||||
cases.append(TestCase(
|
||||
desc="temperature 超出范围 (2.0)",
|
||||
method="POST",
|
||||
url=messages_url,
|
||||
headers=headers,
|
||||
body={
|
||||
"model": model,
|
||||
"max_tokens": 5,
|
||||
"temperature": 2.0,
|
||||
"messages": [{"role": "user", "content": "Hi"}]
|
||||
}
|
||||
))
|
||||
|
||||
# ==== --vision ====
|
||||
if args.vision:
|
||||
cases.append(TestCase(
|
||||
desc="图片 URL 输入 (--vision)",
|
||||
method="POST",
|
||||
url=messages_url,
|
||||
headers=headers,
|
||||
body={
|
||||
"model": model,
|
||||
"max_tokens": 10,
|
||||
"messages": [{"role": "user", "content": [
|
||||
{"type": "text", "text": "用一个词描述这张图"},
|
||||
{"type": "image", "source": {
|
||||
"type": "url",
|
||||
"url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/"
|
||||
"Gfp-wisconsin-madison-the-nature-boardwalk.jpg/"
|
||||
"2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
|
||||
}}
|
||||
]}]
|
||||
},
|
||||
validator=validate_anthropic_messages_response
|
||||
))
|
||||
|
||||
# ==== --stream ====
|
||||
if args.stream:
|
||||
cases.append(TestCase(
|
||||
desc="基本流式 (--stream)",
|
||||
method="POST",
|
||||
url=messages_url,
|
||||
headers=headers,
|
||||
body={
|
||||
"model": model,
|
||||
"max_tokens": 5,
|
||||
"stream": True,
|
||||
"messages": [{"role": "user", "content": "Hi"}]
|
||||
},
|
||||
stream=True,
|
||||
validator=validate_anthropic_messages_response
|
||||
))
|
||||
cases.append(TestCase(
|
||||
desc="流式 + system prompt (--stream)",
|
||||
method="POST",
|
||||
url=messages_url,
|
||||
headers=headers,
|
||||
body={
|
||||
"model": model,
|
||||
"max_tokens": 5,
|
||||
"stream": True,
|
||||
"system": "Reply in one word.",
|
||||
"messages": [{"role": "user", "content": "1+1="}]
|
||||
},
|
||||
stream=True,
|
||||
validator=validate_anthropic_messages_response
|
||||
))
|
||||
cases.append(TestCase(
|
||||
desc="流式 + stop_sequences (--stream)",
|
||||
method="POST",
|
||||
url=messages_url,
|
||||
headers=headers,
|
||||
body={
|
||||
"model": model,
|
||||
"max_tokens": 20,
|
||||
"stream": True,
|
||||
"stop_sequences": ["5"],
|
||||
"messages": [{"role": "user", "content": "数数: 1,2,3,"}]
|
||||
},
|
||||
stream=True,
|
||||
validator=validate_anthropic_messages_response
|
||||
))
|
||||
|
||||
# ==== --tools ====
|
||||
if args.tools:
|
||||
tool_weather = {
|
||||
"name": "get_weather",
|
||||
"description": "获取指定城市的天气",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {"type": "string", "description": "城市名称"}
|
||||
},
|
||||
"required": ["location"]
|
||||
}
|
||||
}
|
||||
cases.append(TestCase(
|
||||
desc="工具调用 tool_choice: auto (--tools)",
|
||||
method="POST",
|
||||
url=messages_url,
|
||||
headers=headers,
|
||||
body={
|
||||
"model": model,
|
||||
"max_tokens": 50,
|
||||
"tools": [tool_weather],
|
||||
"tool_choice": {"type": "auto"},
|
||||
"messages": [{"role": "user", "content": "北京天气怎么样？"}]
|
||||
},
|
||||
validator=validate_anthropic_messages_response
|
||||
))
|
||||
cases.append(TestCase(
|
||||
desc="工具调用 tool_choice: any (--tools)",
|
||||
method="POST",
|
||||
url=messages_url,
|
||||
headers=headers,
|
||||
body={
|
||||
"model": model,
|
||||
"max_tokens": 50,
|
||||
"tools": [tool_weather],
|
||||
"tool_choice": {"type": "any"},
|
||||
"messages": [{"role": "user", "content": "北京天气怎么样？"}]
|
||||
},
|
||||
validator=validate_anthropic_messages_response
|
||||
))
|
||||
cases.append(TestCase(
|
||||
desc="指定工具调用 tool_choice: {name} (--tools)",
|
||||
method="POST",
|
||||
url=messages_url,
|
||||
headers=headers,
|
||||
body={
|
||||
"model": model,
|
||||
"max_tokens": 50,
|
||||
"tools": [tool_weather],
|
||||
"tool_choice": {"type": "tool", "name": "get_weather"},
|
||||
"messages": [{"role": "user", "content": "北京天气怎么样？"}]
|
||||
},
|
||||
validator=validate_anthropic_messages_response
|
||||
))
|
||||
cases.append(TestCase(
|
||||
desc="tool_choice: none (--tools)",
|
||||
method="POST",
|
||||
url=messages_url,
|
||||
headers=headers,
|
||||
body={
|
||||
"model": model,
|
||||
"max_tokens": 20,
|
||||
"tools": [tool_weather],
|
||||
"tool_choice": {"type": "none"},
|
||||
"messages": [{"role": "user", "content": "北京天气怎么样？"}]
|
||||
},
|
||||
validator=validate_anthropic_messages_response
|
||||
))
|
||||
cases.append(TestCase(
|
||||
desc="多轮工具调用（tool_result 返回）(--tools)",
|
||||
method="POST",
|
||||
url=messages_url,
|
||||
headers=headers,
|
||||
body={
|
||||
"model": model,
|
||||
"max_tokens": 20,
|
||||
"tools": [tool_weather],
|
||||
"messages": [
|
||||
{"role": "user", "content": "北京天气怎么样？"},
|
||||
{"role": "assistant", "content": [
|
||||
{"type": "text", "text": "让我查一下。"},
|
||||
{"type": "tool_use", "id": "toolu_001", "name": "get_weather", "input": {"location": "Beijing"}}
|
||||
]},
|
||||
{"role": "user", "content": [
|
||||
{"type": "tool_result", "tool_use_id": "toolu_001", "content": "{\"temperature\": 22, \"condition\": \"晴\"}"}
|
||||
]}
|
||||
]
|
||||
},
|
||||
validator=validate_anthropic_messages_response
|
||||
))
|
||||
cases.append(TestCase(
|
||||
desc="多轮工具调用（tool_result 带 is_error）(--tools)",
|
||||
method="POST",
|
||||
url=messages_url,
|
||||
headers=headers,
|
||||
body={
|
||||
"model": model,
|
||||
"max_tokens": 20,
|
||||
"tools": [tool_weather],
|
||||
"messages": [
|
||||
{"role": "user", "content": "北京天气怎么样？"},
|
||||
{"role": "assistant", "content": [
|
||||
{"type": "tool_use", "id": "toolu_002", "name": "get_weather", "input": {"location": "Beijing"}}
|
||||
]},
|
||||
{"role": "user", "content": [
|
||||
{"type": "tool_result", "tool_use_id": "toolu_002", "is_error": True, "content": "天气服务不可用"}
|
||||
]}
|
||||
]
|
||||
},
|
||||
validator=validate_anthropic_messages_response
|
||||
))
|
||||
cases.append(TestCase(
|
||||
desc="tool_choice 指向不存在的工具（负面）(--tools)",
|
||||
method="POST",
|
||||
url=messages_url,
|
||||
headers=headers,
|
||||
body={
|
||||
"model": model,
|
||||
"max_tokens": 50,
|
||||
"tools": [tool_weather],
|
||||
"tool_choice": {"type": "tool", "name": "nonexistent_tool_xxx"},
|
||||
"messages": [{"role": "user", "content": "北京天气怎么样？"}]
|
||||
}
|
||||
))
|
||||
cases.append(TestCase(
|
||||
desc="多工具定义 (--tools)",
|
||||
method="POST",
|
||||
url=messages_url,
|
||||
headers=headers,
|
||||
body={
|
||||
"model": model,
|
||||
"max_tokens": 50,
|
||||
"tools": [
|
||||
tool_weather,
|
||||
{
|
||||
"name": "get_time",
|
||||
"description": "获取指定城市的当前时间",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {"type": "string", "description": "城市名称"}
|
||||
},
|
||||
"required": ["location"]
|
||||
}
|
||||
}
|
||||
],
|
||||
"tool_choice": {"type": "auto"},
|
||||
"messages": [{"role": "user", "content": "北京现在几点了？天气怎么样？"}]
|
||||
},
|
||||
validator=validate_anthropic_messages_response
|
||||
))
|
||||
|
||||
# ==== --thinking ====
|
||||
if args.thinking:
|
||||
cases.append(TestCase(
|
||||
desc="扩展思维 enabled (--thinking)",
|
||||
method="POST",
|
||||
url=messages_url,
|
||||
headers=headers,
|
||||
body={
|
||||
"model": model,
|
||||
"max_tokens": 200,
|
||||
"thinking": {"type": "enabled", "budget_tokens": 100},
|
||||
"messages": [{"role": "user", "content": "1+1=?"}]
|
||||
},
|
||||
validator=validate_anthropic_messages_response
|
||||
))
|
||||
cases.append(TestCase(
|
||||
desc="扩展思维 adaptive (--thinking)",
|
||||
method="POST",
|
||||
url=messages_url,
|
||||
headers=headers,
|
||||
body={
|
||||
"model": model,
|
||||
"max_tokens": 200,
|
||||
"thinking": {"type": "adaptive", "budget_tokens": 100},
|
||||
"messages": [{"role": "user", "content": "1+1=?"}]
|
||||
},
|
||||
validator=validate_anthropic_messages_response
|
||||
))
|
||||
|
||||
# ==== --stream + --tools 组合 ====
|
||||
if args.stream and args.tools:
|
||||
tool_weather_stream = {
|
||||
"name": "get_weather",
|
||||
"description": "获取指定城市的天气",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {"type": "string", "description": "城市名称"}
|
||||
},
|
||||
"required": ["location"]
|
||||
}
|
||||
}
|
||||
cases.append(TestCase(
|
||||
desc="流式工具调用 (--stream --tools)",
|
||||
method="POST",
|
||||
url=messages_url,
|
||||
headers=headers,
|
||||
body={
|
||||
"model": model,
|
||||
"max_tokens": 50,
|
||||
"stream": True,
|
||||
"tools": [tool_weather_stream],
|
||||
"tool_choice": {"type": "auto"},
|
||||
"messages": [{"role": "user", "content": "北京天气怎么样？"}]
|
||||
},
|
||||
stream=True,
|
||||
validator=validate_anthropic_messages_response
|
||||
))
|
||||
|
||||
# ==== 高级参数测试 ====
|
||||
# cache_control: 缓存控制
|
||||
cases.append(TestCase(
|
||||
desc="cache_control 缓存控制",
|
||||
method="POST",
|
||||
url=messages_url,
|
||||
headers=headers,
|
||||
body={
|
||||
"model": model,
|
||||
"max_tokens": 10,
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
"messages": [{"role": "user", "content": "Hello"}]
|
||||
},
|
||||
validator=validate_anthropic_messages_response
|
||||
))
|
||||
|
||||
# output_config: 输出配置
|
||||
cases.append(TestCase(
|
||||
desc="output_config 输出配置",
|
||||
method="POST",
|
||||
url=messages_url,
|
||||
headers=headers,
|
||||
body={
|
||||
"model": model,
|
||||
"max_tokens": 10,
|
||||
"output_config": {"format": "text"},
|
||||
"messages": [{"role": "user", "content": "Hi"}]
|
||||
},
|
||||
validator=validate_anthropic_messages_response
|
||||
))
|
||||
|
||||
# service_tier: 服务层级
|
||||
cases.append(TestCase(
|
||||
desc="service_tier: auto",
|
||||
method="POST",
|
||||
url=messages_url,
|
||||
headers=headers,
|
||||
body={
|
||||
"model": model,
|
||||
"max_tokens": 5,
|
||||
"service_tier": "auto",
|
||||
"messages": [{"role": "user", "content": "Hello"}]
|
||||
},
|
||||
validator=validate_anthropic_messages_response
|
||||
))
|
||||
|
||||
# ==== Models API 分页测试 ====
|
||||
cases.append(TestCase(
|
||||
desc="Models API 分页 limit=5",
|
||||
method="GET",
|
||||
url=f"{models_url}?limit=5",
|
||||
headers=headers
|
||||
))
|
||||
|
||||
# ==== 执行测试 ====
|
||||
flags = []
|
||||
if args.vision:
|
||||
flags.append("vision")
|
||||
if args.stream:
|
||||
flags.append("stream")
|
||||
if args.tools:
|
||||
flags.append("tools")
|
||||
if args.thinking:
|
||||
flags.append("thinking")
|
||||
|
||||
run_test_suite(
|
||||
cases=cases,
|
||||
ssl_ctx=ssl_ctx,
|
||||
title="Anthropic 兼容性测试",
|
||||
base_url=base_url,
|
||||
model=model,
|
||||
flags=flags
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user