Add cases
curl --request POST \
--url https://api.zelto.ai/v1/benchmarks/{id}/cases \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"cases": [
{
"title": "<string>",
"tags": [
"<string>"
],
"checkGroupId": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"language": "<string>",
"agentVersionId": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"agentId": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"draft": true,
"turns": [
{
"text": "<string>",
"nodeId": "<string>"
}
],
"variables": {},
"expected": "<string>",
"tools": [
{
"name": "<string>",
"parameters": {
"type": "object",
"properties": {},
"required": [
"<string>"
],
"additionalProperties": true
},
"description": "<string>"
}
],
"toolMocks": [
{
"tool": "<string>",
"response": "<string>",
"match": {}
}
],
"referenceTranscript": "<string>",
"keyTerms": [
"<string>"
],
"referenceVerified": true,
"skippedJudgeIds": [
"3c90c3cc-0d44-4b50-8888-8dd25736052a"
],
"judgeParamOverrides": {}
}
]
}
'import requests
url = "https://api.zelto.ai/v1/benchmarks/{id}/cases"
payload = { "cases": [
{
"title": "<string>",
"tags": ["<string>"],
"checkGroupId": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"language": "<string>",
"agentVersionId": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"agentId": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"draft": True,
"turns": [
{
"text": "<string>",
"nodeId": "<string>"
}
],
"variables": {},
"expected": "<string>",
"tools": [
{
"name": "<string>",
"parameters": {
"type": "object",
"properties": {},
"required": ["<string>"],
"additionalProperties": True
},
"description": "<string>"
}
],
"toolMocks": [
{
"tool": "<string>",
"response": "<string>",
"match": {}
}
],
"referenceTranscript": "<string>",
"keyTerms": ["<string>"],
"referenceVerified": True,
"skippedJudgeIds": ["3c90c3cc-0d44-4b50-8888-8dd25736052a"],
"judgeParamOverrides": {}
}
] }
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
cases: [
{
title: '<string>',
tags: ['<string>'],
checkGroupId: '3c90c3cc-0d44-4b50-8888-8dd25736052a',
language: '<string>',
agentVersionId: '3c90c3cc-0d44-4b50-8888-8dd25736052a',
agentId: '3c90c3cc-0d44-4b50-8888-8dd25736052a',
draft: true,
turns: [{text: '<string>', nodeId: '<string>'}],
variables: {},
expected: '<string>',
tools: [
{
name: '<string>',
parameters: {
type: 'object',
properties: {},
required: ['<string>'],
additionalProperties: true
},
description: '<string>'
}
],
toolMocks: [{tool: '<string>', response: '<string>', match: {}}],
referenceTranscript: '<string>',
keyTerms: ['<string>'],
referenceVerified: true,
skippedJudgeIds: ['3c90c3cc-0d44-4b50-8888-8dd25736052a'],
judgeParamOverrides: {}
}
]
})
};
fetch('https://api.zelto.ai/v1/benchmarks/{id}/cases', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.zelto.ai/v1/benchmarks/{id}/cases",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'cases' => [
[
'title' => '<string>',
'tags' => [
'<string>'
],
'checkGroupId' => '3c90c3cc-0d44-4b50-8888-8dd25736052a',
'language' => '<string>',
'agentVersionId' => '3c90c3cc-0d44-4b50-8888-8dd25736052a',
'agentId' => '3c90c3cc-0d44-4b50-8888-8dd25736052a',
'draft' => true,
'turns' => [
[
'text' => '<string>',
'nodeId' => '<string>'
]
],
'variables' => [
],
'expected' => '<string>',
'tools' => [
[
'name' => '<string>',
'parameters' => [
'type' => 'object',
'properties' => [
],
'required' => [
'<string>'
],
'additionalProperties' => true
],
'description' => '<string>'
]
],
'toolMocks' => [
[
'tool' => '<string>',
'response' => '<string>',
'match' => [
]
]
],
'referenceTranscript' => '<string>',
'keyTerms' => [
'<string>'
],
'referenceVerified' => true,
'skippedJudgeIds' => [
'3c90c3cc-0d44-4b50-8888-8dd25736052a'
],
'judgeParamOverrides' => [
]
]
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.zelto.ai/v1/benchmarks/{id}/cases"
payload := strings.NewReader("{\n \"cases\": [\n {\n \"title\": \"<string>\",\n \"tags\": [\n \"<string>\"\n ],\n \"checkGroupId\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n \"language\": \"<string>\",\n \"agentVersionId\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n \"agentId\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n \"draft\": true,\n \"turns\": [\n {\n \"text\": \"<string>\",\n \"nodeId\": \"<string>\"\n }\n ],\n \"variables\": {},\n \"expected\": \"<string>\",\n \"tools\": [\n {\n \"name\": \"<string>\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {},\n \"required\": [\n \"<string>\"\n ],\n \"additionalProperties\": true\n },\n \"description\": \"<string>\"\n }\n ],\n \"toolMocks\": [\n {\n \"tool\": \"<string>\",\n \"response\": \"<string>\",\n \"match\": {}\n }\n ],\n \"referenceTranscript\": \"<string>\",\n \"keyTerms\": [\n \"<string>\"\n ],\n \"referenceVerified\": true,\n \"skippedJudgeIds\": [\n \"3c90c3cc-0d44-4b50-8888-8dd25736052a\"\n ],\n \"judgeParamOverrides\": {}\n }\n ]\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.zelto.ai/v1/benchmarks/{id}/cases")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"cases\": [\n {\n \"title\": \"<string>\",\n \"tags\": [\n \"<string>\"\n ],\n \"checkGroupId\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n \"language\": \"<string>\",\n \"agentVersionId\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n \"agentId\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n \"draft\": true,\n \"turns\": [\n {\n \"text\": \"<string>\",\n \"nodeId\": \"<string>\"\n }\n ],\n \"variables\": {},\n \"expected\": \"<string>\",\n \"tools\": [\n {\n \"name\": \"<string>\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {},\n \"required\": [\n \"<string>\"\n ],\n \"additionalProperties\": true\n },\n \"description\": \"<string>\"\n }\n ],\n \"toolMocks\": [\n {\n \"tool\": \"<string>\",\n \"response\": \"<string>\",\n \"match\": {}\n }\n ],\n \"referenceTranscript\": \"<string>\",\n \"keyTerms\": [\n \"<string>\"\n ],\n \"referenceVerified\": true,\n \"skippedJudgeIds\": [\n \"3c90c3cc-0d44-4b50-8888-8dd25736052a\"\n ],\n \"judgeParamOverrides\": {}\n }\n ]\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.zelto.ai/v1/benchmarks/{id}/cases")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"cases\": [\n {\n \"title\": \"<string>\",\n \"tags\": [\n \"<string>\"\n ],\n \"checkGroupId\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n \"language\": \"<string>\",\n \"agentVersionId\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n \"agentId\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n \"draft\": true,\n \"turns\": [\n {\n \"text\": \"<string>\",\n \"nodeId\": \"<string>\"\n }\n ],\n \"variables\": {},\n \"expected\": \"<string>\",\n \"tools\": [\n {\n \"name\": \"<string>\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {},\n \"required\": [\n \"<string>\"\n ],\n \"additionalProperties\": true\n },\n \"description\": \"<string>\"\n }\n ],\n \"toolMocks\": [\n {\n \"tool\": \"<string>\",\n \"response\": \"<string>\",\n \"match\": {}\n }\n ],\n \"referenceTranscript\": \"<string>\",\n \"keyTerms\": [\n \"<string>\"\n ],\n \"referenceVerified\": true,\n \"skippedJudgeIds\": [\n \"3c90c3cc-0d44-4b50-8888-8dd25736052a\"\n ],\n \"judgeParamOverrides\": {}\n }\n ]\n}"
response = http.request(request)
puts response.read_body{
"cases": [
{
"id": "<string>",
"revision": 123,
"type": "turn",
"title": "<string>",
"checkGroupId": "<string>",
"source": "production",
"sourceConversationId": "<string>",
"sourceAgent": {
"id": "<string>",
"name": "<string>"
},
"tags": [
"<string>"
],
"language": "<string>",
"retired": true,
"draft": true,
"preview": "<string>",
"sourceAgentVersionLabel": "<string>",
"referenceUnverified": true,
"unpublished": true,
"skippedJudgeIds": [
"<string>"
],
"judgeParamOverrides": {},
"addedInVersion": 123,
"retiredInVersion": 123,
"toolExpectation": {
"expect": "call",
"calls": [
{
"tool": "<string>",
"arguments": {}
}
],
"ordered": true,
"forbiddenTools": [
"<string>"
],
"noExtraArguments": true,
"replyAt": 5
},
"replyExpectation": {
"check": "reference",
"text": "<string>",
"idealReply": "<string>"
},
"traceSource": {
"kind": "trace",
"traceId": "<string>",
"spanId": "<string>",
"turns": "transcript"
},
"status": "ready",
"contentPurged": true,
"contentDeleted": true,
"createdAt": "<string>",
"updatedAt": "<string>"
}
],
"skipped": [
{
"reason": "<string>",
"conversationId": "<string>",
"traceId": "<string>"
}
],
"dryRun": true,
"warning": "<string>"
}{
"error": "<string>",
"message": "<string>",
"code": "<string>",
"hint": "<string>",
"parameter": "<string>",
"unknown": [
"<string>"
],
"valid": [
"<string>"
],
"details": {}
}{
"error": "<string>",
"message": "<string>",
"code": "<string>",
"hint": "<string>",
"parameter": "<string>",
"unknown": [
"<string>"
],
"valid": [
"<string>"
],
"details": {}
}{
"error": "validation_failed",
"code": "validation_failed",
"message": "<string>",
"hint": "<string>",
"details": {}
}{
"error": "<string>",
"message": "<string>",
"code": "<string>",
"hint": "<string>",
"parameter": "<string>",
"unknown": [
"<string>"
],
"valid": [
"<string>"
],
"details": {}
}{
"error": "validation_failed",
"code": "validation_failed",
"message": "<string>",
"hint": "<string>",
"details": {}
}Benchmarks
Add cases
Creates manual cases in the draft. Audio cases are created as drafts until their clip is uploaded (clip uploads ship with speech-to-text).
POST
/
v1
/
benchmarks
/
{id}
/
cases
Add cases
curl --request POST \
--url https://api.zelto.ai/v1/benchmarks/{id}/cases \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"cases": [
{
"title": "<string>",
"tags": [
"<string>"
],
"checkGroupId": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"language": "<string>",
"agentVersionId": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"agentId": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"draft": true,
"turns": [
{
"text": "<string>",
"nodeId": "<string>"
}
],
"variables": {},
"expected": "<string>",
"tools": [
{
"name": "<string>",
"parameters": {
"type": "object",
"properties": {},
"required": [
"<string>"
],
"additionalProperties": true
},
"description": "<string>"
}
],
"toolMocks": [
{
"tool": "<string>",
"response": "<string>",
"match": {}
}
],
"referenceTranscript": "<string>",
"keyTerms": [
"<string>"
],
"referenceVerified": true,
"skippedJudgeIds": [
"3c90c3cc-0d44-4b50-8888-8dd25736052a"
],
"judgeParamOverrides": {}
}
]
}
'import requests
url = "https://api.zelto.ai/v1/benchmarks/{id}/cases"
payload = { "cases": [
{
"title": "<string>",
"tags": ["<string>"],
"checkGroupId": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"language": "<string>",
"agentVersionId": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"agentId": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"draft": True,
"turns": [
{
"text": "<string>",
"nodeId": "<string>"
}
],
"variables": {},
"expected": "<string>",
"tools": [
{
"name": "<string>",
"parameters": {
"type": "object",
"properties": {},
"required": ["<string>"],
"additionalProperties": True
},
"description": "<string>"
}
],
"toolMocks": [
{
"tool": "<string>",
"response": "<string>",
"match": {}
}
],
"referenceTranscript": "<string>",
"keyTerms": ["<string>"],
"referenceVerified": True,
"skippedJudgeIds": ["3c90c3cc-0d44-4b50-8888-8dd25736052a"],
"judgeParamOverrides": {}
}
] }
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
cases: [
{
title: '<string>',
tags: ['<string>'],
checkGroupId: '3c90c3cc-0d44-4b50-8888-8dd25736052a',
language: '<string>',
agentVersionId: '3c90c3cc-0d44-4b50-8888-8dd25736052a',
agentId: '3c90c3cc-0d44-4b50-8888-8dd25736052a',
draft: true,
turns: [{text: '<string>', nodeId: '<string>'}],
variables: {},
expected: '<string>',
tools: [
{
name: '<string>',
parameters: {
type: 'object',
properties: {},
required: ['<string>'],
additionalProperties: true
},
description: '<string>'
}
],
toolMocks: [{tool: '<string>', response: '<string>', match: {}}],
referenceTranscript: '<string>',
keyTerms: ['<string>'],
referenceVerified: true,
skippedJudgeIds: ['3c90c3cc-0d44-4b50-8888-8dd25736052a'],
judgeParamOverrides: {}
}
]
})
};
fetch('https://api.zelto.ai/v1/benchmarks/{id}/cases', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.zelto.ai/v1/benchmarks/{id}/cases",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'cases' => [
[
'title' => '<string>',
'tags' => [
'<string>'
],
'checkGroupId' => '3c90c3cc-0d44-4b50-8888-8dd25736052a',
'language' => '<string>',
'agentVersionId' => '3c90c3cc-0d44-4b50-8888-8dd25736052a',
'agentId' => '3c90c3cc-0d44-4b50-8888-8dd25736052a',
'draft' => true,
'turns' => [
[
'text' => '<string>',
'nodeId' => '<string>'
]
],
'variables' => [
],
'expected' => '<string>',
'tools' => [
[
'name' => '<string>',
'parameters' => [
'type' => 'object',
'properties' => [
],
'required' => [
'<string>'
],
'additionalProperties' => true
],
'description' => '<string>'
]
],
'toolMocks' => [
[
'tool' => '<string>',
'response' => '<string>',
'match' => [
]
]
],
'referenceTranscript' => '<string>',
'keyTerms' => [
'<string>'
],
'referenceVerified' => true,
'skippedJudgeIds' => [
'3c90c3cc-0d44-4b50-8888-8dd25736052a'
],
'judgeParamOverrides' => [
]
]
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.zelto.ai/v1/benchmarks/{id}/cases"
payload := strings.NewReader("{\n \"cases\": [\n {\n \"title\": \"<string>\",\n \"tags\": [\n \"<string>\"\n ],\n \"checkGroupId\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n \"language\": \"<string>\",\n \"agentVersionId\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n \"agentId\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n \"draft\": true,\n \"turns\": [\n {\n \"text\": \"<string>\",\n \"nodeId\": \"<string>\"\n }\n ],\n \"variables\": {},\n \"expected\": \"<string>\",\n \"tools\": [\n {\n \"name\": \"<string>\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {},\n \"required\": [\n \"<string>\"\n ],\n \"additionalProperties\": true\n },\n \"description\": \"<string>\"\n }\n ],\n \"toolMocks\": [\n {\n \"tool\": \"<string>\",\n \"response\": \"<string>\",\n \"match\": {}\n }\n ],\n \"referenceTranscript\": \"<string>\",\n \"keyTerms\": [\n \"<string>\"\n ],\n \"referenceVerified\": true,\n \"skippedJudgeIds\": [\n \"3c90c3cc-0d44-4b50-8888-8dd25736052a\"\n ],\n \"judgeParamOverrides\": {}\n }\n ]\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.zelto.ai/v1/benchmarks/{id}/cases")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"cases\": [\n {\n \"title\": \"<string>\",\n \"tags\": [\n \"<string>\"\n ],\n \"checkGroupId\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n \"language\": \"<string>\",\n \"agentVersionId\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n \"agentId\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n \"draft\": true,\n \"turns\": [\n {\n \"text\": \"<string>\",\n \"nodeId\": \"<string>\"\n }\n ],\n \"variables\": {},\n \"expected\": \"<string>\",\n \"tools\": [\n {\n \"name\": \"<string>\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {},\n \"required\": [\n \"<string>\"\n ],\n \"additionalProperties\": true\n },\n \"description\": \"<string>\"\n }\n ],\n \"toolMocks\": [\n {\n \"tool\": \"<string>\",\n \"response\": \"<string>\",\n \"match\": {}\n }\n ],\n \"referenceTranscript\": \"<string>\",\n \"keyTerms\": [\n \"<string>\"\n ],\n \"referenceVerified\": true,\n \"skippedJudgeIds\": [\n \"3c90c3cc-0d44-4b50-8888-8dd25736052a\"\n ],\n \"judgeParamOverrides\": {}\n }\n ]\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.zelto.ai/v1/benchmarks/{id}/cases")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"cases\": [\n {\n \"title\": \"<string>\",\n \"tags\": [\n \"<string>\"\n ],\n \"checkGroupId\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n \"language\": \"<string>\",\n \"agentVersionId\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n \"agentId\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n \"draft\": true,\n \"turns\": [\n {\n \"text\": \"<string>\",\n \"nodeId\": \"<string>\"\n }\n ],\n \"variables\": {},\n \"expected\": \"<string>\",\n \"tools\": [\n {\n \"name\": \"<string>\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {},\n \"required\": [\n \"<string>\"\n ],\n \"additionalProperties\": true\n },\n \"description\": \"<string>\"\n }\n ],\n \"toolMocks\": [\n {\n \"tool\": \"<string>\",\n \"response\": \"<string>\",\n \"match\": {}\n }\n ],\n \"referenceTranscript\": \"<string>\",\n \"keyTerms\": [\n \"<string>\"\n ],\n \"referenceVerified\": true,\n \"skippedJudgeIds\": [\n \"3c90c3cc-0d44-4b50-8888-8dd25736052a\"\n ],\n \"judgeParamOverrides\": {}\n }\n ]\n}"
response = http.request(request)
puts response.read_body{
"cases": [
{
"id": "<string>",
"revision": 123,
"type": "turn",
"title": "<string>",
"checkGroupId": "<string>",
"source": "production",
"sourceConversationId": "<string>",
"sourceAgent": {
"id": "<string>",
"name": "<string>"
},
"tags": [
"<string>"
],
"language": "<string>",
"retired": true,
"draft": true,
"preview": "<string>",
"sourceAgentVersionLabel": "<string>",
"referenceUnverified": true,
"unpublished": true,
"skippedJudgeIds": [
"<string>"
],
"judgeParamOverrides": {},
"addedInVersion": 123,
"retiredInVersion": 123,
"toolExpectation": {
"expect": "call",
"calls": [
{
"tool": "<string>",
"arguments": {}
}
],
"ordered": true,
"forbiddenTools": [
"<string>"
],
"noExtraArguments": true,
"replyAt": 5
},
"replyExpectation": {
"check": "reference",
"text": "<string>",
"idealReply": "<string>"
},
"traceSource": {
"kind": "trace",
"traceId": "<string>",
"spanId": "<string>",
"turns": "transcript"
},
"status": "ready",
"contentPurged": true,
"contentDeleted": true,
"createdAt": "<string>",
"updatedAt": "<string>"
}
],
"skipped": [
{
"reason": "<string>",
"conversationId": "<string>",
"traceId": "<string>"
}
],
"dryRun": true,
"warning": "<string>"
}{
"error": "<string>",
"message": "<string>",
"code": "<string>",
"hint": "<string>",
"parameter": "<string>",
"unknown": [
"<string>"
],
"valid": [
"<string>"
],
"details": {}
}{
"error": "<string>",
"message": "<string>",
"code": "<string>",
"hint": "<string>",
"parameter": "<string>",
"unknown": [
"<string>"
],
"valid": [
"<string>"
],
"details": {}
}{
"error": "validation_failed",
"code": "validation_failed",
"message": "<string>",
"hint": "<string>",
"details": {}
}{
"error": "<string>",
"message": "<string>",
"code": "<string>",
"hint": "<string>",
"parameter": "<string>",
"unknown": [
"<string>"
],
"valid": [
"<string>"
],
"details": {}
}{
"error": "validation_failed",
"code": "validation_failed",
"message": "<string>",
"hint": "<string>",
"details": {}
}Authorizations
Send Authorization: Bearer YOUR_API_KEY. The key resolves to a single organization.
Headers
Optional key, unique per organization (1-255 printable ASCII characters, e.g. a UUID). Resending a request with the same key returns the original result instead of creating a second one; reusing it for a different request is a 409 idempotency_key_reused.
Required string length:
1 - 255Path Parameters
Benchmark id.
Body
application/json
Up to 100 cases per request.
Required array length:
1 - 100 elementsShow child attributes
Show child attributes