Try a Judge
curl --request POST \
--url https://api.zelto.ai/v1/benchmarks/{id}/judges/try \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"judge": {
"label": "<string>",
"config": {
"kind": "judge",
"rubric": "<string>",
"model": {
"modelId": "<string>",
"aiKeyId": "<string>"
},
"scale": {
"type": "pass_fail"
}
},
"labelTemplate": "<string>",
"required": true,
"weight": 1,
"checkGroupId": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"appliesTo": {},
"params": []
},
"caseIds": [
"3c90c3cc-0d44-4b50-8888-8dd25736052a"
],
"estimateOnly": true
}
'import requests
url = "https://api.zelto.ai/v1/benchmarks/{id}/judges/try"
payload = {
"judge": {
"label": "<string>",
"config": {
"kind": "judge",
"rubric": "<string>",
"model": {
"modelId": "<string>",
"aiKeyId": "<string>"
},
"scale": { "type": "pass_fail" }
},
"labelTemplate": "<string>",
"required": True,
"weight": 1,
"checkGroupId": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"appliesTo": {},
"params": []
},
"caseIds": ["3c90c3cc-0d44-4b50-8888-8dd25736052a"],
"estimateOnly": True
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
judge: {
label: '<string>',
config: {
kind: 'judge',
rubric: '<string>',
model: {modelId: '<string>', aiKeyId: '<string>'},
scale: {type: 'pass_fail'}
},
labelTemplate: '<string>',
required: true,
weight: 1,
checkGroupId: '3c90c3cc-0d44-4b50-8888-8dd25736052a',
appliesTo: {},
params: []
},
caseIds: ['3c90c3cc-0d44-4b50-8888-8dd25736052a'],
estimateOnly: true
})
};
fetch('https://api.zelto.ai/v1/benchmarks/{id}/judges/try', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.zelto.ai/v1/benchmarks/{id}/judges/try",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'judge' => [
'label' => '<string>',
'config' => [
'kind' => 'judge',
'rubric' => '<string>',
'model' => [
'modelId' => '<string>',
'aiKeyId' => '<string>'
],
'scale' => [
'type' => 'pass_fail'
]
],
'labelTemplate' => '<string>',
'required' => true,
'weight' => 1,
'checkGroupId' => '3c90c3cc-0d44-4b50-8888-8dd25736052a',
'appliesTo' => [
],
'params' => [
]
],
'caseIds' => [
'3c90c3cc-0d44-4b50-8888-8dd25736052a'
],
'estimateOnly' => true
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.zelto.ai/v1/benchmarks/{id}/judges/try"
payload := strings.NewReader("{\n \"judge\": {\n \"label\": \"<string>\",\n \"config\": {\n \"kind\": \"judge\",\n \"rubric\": \"<string>\",\n \"model\": {\n \"modelId\": \"<string>\",\n \"aiKeyId\": \"<string>\"\n },\n \"scale\": {\n \"type\": \"pass_fail\"\n }\n },\n \"labelTemplate\": \"<string>\",\n \"required\": true,\n \"weight\": 1,\n \"checkGroupId\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n \"appliesTo\": {},\n \"params\": []\n },\n \"caseIds\": [\n \"3c90c3cc-0d44-4b50-8888-8dd25736052a\"\n ],\n \"estimateOnly\": true\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.zelto.ai/v1/benchmarks/{id}/judges/try")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"judge\": {\n \"label\": \"<string>\",\n \"config\": {\n \"kind\": \"judge\",\n \"rubric\": \"<string>\",\n \"model\": {\n \"modelId\": \"<string>\",\n \"aiKeyId\": \"<string>\"\n },\n \"scale\": {\n \"type\": \"pass_fail\"\n }\n },\n \"labelTemplate\": \"<string>\",\n \"required\": true,\n \"weight\": 1,\n \"checkGroupId\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n \"appliesTo\": {},\n \"params\": []\n },\n \"caseIds\": [\n \"3c90c3cc-0d44-4b50-8888-8dd25736052a\"\n ],\n \"estimateOnly\": true\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.zelto.ai/v1/benchmarks/{id}/judges/try")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"judge\": {\n \"label\": \"<string>\",\n \"config\": {\n \"kind\": \"judge\",\n \"rubric\": \"<string>\",\n \"model\": {\n \"modelId\": \"<string>\",\n \"aiKeyId\": \"<string>\"\n },\n \"scale\": {\n \"type\": \"pass_fail\"\n }\n },\n \"labelTemplate\": \"<string>\",\n \"required\": true,\n \"weight\": 1,\n \"checkGroupId\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n \"appliesTo\": {},\n \"params\": []\n },\n \"caseIds\": [\n \"3c90c3cc-0d44-4b50-8888-8dd25736052a\"\n ],\n \"estimateOnly\": true\n}"
response = http.request(request)
puts response.read_body{
"results": [
{
"caseId": "<string>",
"passed": true,
"value": "<string>",
"reasoning": "<string>",
"error": "<string>",
"notApplicable": true
}
],
"simulated": true,
"estimate": {
"calls": 123,
"inputTokens": 123,
"outputTokens": 123,
"costUsd": 123,
"model": "<string>",
"unpriced": true
},
"costUsd": 123,
"listCostUsd": 123,
"tryId": "<string>"
}{
"error": "<string>",
"message": "<string>",
"code": "<string>",
"hint": "<string>",
"parameter": "<string>",
"unknown": [
"<string>"
],
"valid": [
"<string>"
],
"details": {}
}{
"error": "<string>",
"message": "<string>",
"code": "<string>",
"hint": "<string>",
"parameter": "<string>",
"unknown": [
"<string>"
],
"valid": [
"<string>"
],
"details": {}
}{
"error": "validation_failed",
"code": "validation_failed",
"message": "<string>",
"hint": "<string>",
"details": {}
}{
"error": "<string>",
"message": "<string>",
"code": "<string>",
"hint": "<string>",
"parameter": "<string>",
"unknown": [
"<string>"
],
"valid": [
"<string>"
],
"details": {}
}Benchmarks
Try a Judge
Scores up to 5 cases’ recorded replies with an unsaved Judge configuration. LLM Judges call their model (one call per case) and return the estimate and the actual cost; send estimateOnly: true to get the estimate first without calling. Nothing is saved, but an LLM Judge’s model calls are billed to benchmark spend like a tiny run and count toward month-to-date spend (costUsd is what Zelto billed for the try, listCostUsd the provider list price); built-in and code Judges are free.
POST
/
v1
/
benchmarks
/
{id}
/
judges
/
try
Try a Judge
curl --request POST \
--url https://api.zelto.ai/v1/benchmarks/{id}/judges/try \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"judge": {
"label": "<string>",
"config": {
"kind": "judge",
"rubric": "<string>",
"model": {
"modelId": "<string>",
"aiKeyId": "<string>"
},
"scale": {
"type": "pass_fail"
}
},
"labelTemplate": "<string>",
"required": true,
"weight": 1,
"checkGroupId": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"appliesTo": {},
"params": []
},
"caseIds": [
"3c90c3cc-0d44-4b50-8888-8dd25736052a"
],
"estimateOnly": true
}
'import requests
url = "https://api.zelto.ai/v1/benchmarks/{id}/judges/try"
payload = {
"judge": {
"label": "<string>",
"config": {
"kind": "judge",
"rubric": "<string>",
"model": {
"modelId": "<string>",
"aiKeyId": "<string>"
},
"scale": { "type": "pass_fail" }
},
"labelTemplate": "<string>",
"required": True,
"weight": 1,
"checkGroupId": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"appliesTo": {},
"params": []
},
"caseIds": ["3c90c3cc-0d44-4b50-8888-8dd25736052a"],
"estimateOnly": True
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
judge: {
label: '<string>',
config: {
kind: 'judge',
rubric: '<string>',
model: {modelId: '<string>', aiKeyId: '<string>'},
scale: {type: 'pass_fail'}
},
labelTemplate: '<string>',
required: true,
weight: 1,
checkGroupId: '3c90c3cc-0d44-4b50-8888-8dd25736052a',
appliesTo: {},
params: []
},
caseIds: ['3c90c3cc-0d44-4b50-8888-8dd25736052a'],
estimateOnly: true
})
};
fetch('https://api.zelto.ai/v1/benchmarks/{id}/judges/try', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.zelto.ai/v1/benchmarks/{id}/judges/try",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'judge' => [
'label' => '<string>',
'config' => [
'kind' => 'judge',
'rubric' => '<string>',
'model' => [
'modelId' => '<string>',
'aiKeyId' => '<string>'
],
'scale' => [
'type' => 'pass_fail'
]
],
'labelTemplate' => '<string>',
'required' => true,
'weight' => 1,
'checkGroupId' => '3c90c3cc-0d44-4b50-8888-8dd25736052a',
'appliesTo' => [
],
'params' => [
]
],
'caseIds' => [
'3c90c3cc-0d44-4b50-8888-8dd25736052a'
],
'estimateOnly' => true
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.zelto.ai/v1/benchmarks/{id}/judges/try"
payload := strings.NewReader("{\n \"judge\": {\n \"label\": \"<string>\",\n \"config\": {\n \"kind\": \"judge\",\n \"rubric\": \"<string>\",\n \"model\": {\n \"modelId\": \"<string>\",\n \"aiKeyId\": \"<string>\"\n },\n \"scale\": {\n \"type\": \"pass_fail\"\n }\n },\n \"labelTemplate\": \"<string>\",\n \"required\": true,\n \"weight\": 1,\n \"checkGroupId\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n \"appliesTo\": {},\n \"params\": []\n },\n \"caseIds\": [\n \"3c90c3cc-0d44-4b50-8888-8dd25736052a\"\n ],\n \"estimateOnly\": true\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.zelto.ai/v1/benchmarks/{id}/judges/try")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"judge\": {\n \"label\": \"<string>\",\n \"config\": {\n \"kind\": \"judge\",\n \"rubric\": \"<string>\",\n \"model\": {\n \"modelId\": \"<string>\",\n \"aiKeyId\": \"<string>\"\n },\n \"scale\": {\n \"type\": \"pass_fail\"\n }\n },\n \"labelTemplate\": \"<string>\",\n \"required\": true,\n \"weight\": 1,\n \"checkGroupId\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n \"appliesTo\": {},\n \"params\": []\n },\n \"caseIds\": [\n \"3c90c3cc-0d44-4b50-8888-8dd25736052a\"\n ],\n \"estimateOnly\": true\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.zelto.ai/v1/benchmarks/{id}/judges/try")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"judge\": {\n \"label\": \"<string>\",\n \"config\": {\n \"kind\": \"judge\",\n \"rubric\": \"<string>\",\n \"model\": {\n \"modelId\": \"<string>\",\n \"aiKeyId\": \"<string>\"\n },\n \"scale\": {\n \"type\": \"pass_fail\"\n }\n },\n \"labelTemplate\": \"<string>\",\n \"required\": true,\n \"weight\": 1,\n \"checkGroupId\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n \"appliesTo\": {},\n \"params\": []\n },\n \"caseIds\": [\n \"3c90c3cc-0d44-4b50-8888-8dd25736052a\"\n ],\n \"estimateOnly\": true\n}"
response = http.request(request)
puts response.read_body{
"results": [
{
"caseId": "<string>",
"passed": true,
"value": "<string>",
"reasoning": "<string>",
"error": "<string>",
"notApplicable": true
}
],
"simulated": true,
"estimate": {
"calls": 123,
"inputTokens": 123,
"outputTokens": 123,
"costUsd": 123,
"model": "<string>",
"unpriced": true
},
"costUsd": 123,
"listCostUsd": 123,
"tryId": "<string>"
}{
"error": "<string>",
"message": "<string>",
"code": "<string>",
"hint": "<string>",
"parameter": "<string>",
"unknown": [
"<string>"
],
"valid": [
"<string>"
],
"details": {}
}{
"error": "<string>",
"message": "<string>",
"code": "<string>",
"hint": "<string>",
"parameter": "<string>",
"unknown": [
"<string>"
],
"valid": [
"<string>"
],
"details": {}
}{
"error": "validation_failed",
"code": "validation_failed",
"message": "<string>",
"hint": "<string>",
"details": {}
}{
"error": "<string>",
"message": "<string>",
"code": "<string>",
"hint": "<string>",
"parameter": "<string>",
"unknown": [
"<string>"
],
"valid": [
"<string>"
],
"details": {}
}Authorizations
Send Authorization: Bearer YOUR_API_KEY. The key resolves to a single organization.
Path Parameters
Benchmark id.
Body
application/json
Response
Successful response.
Verdicts from trying a Judge on a few cases. Nothing is saved; an LLM Judge's model calls are billed to benchmark spend like a tiny run.
Show child attributes
Show child attributes
True when LLM Judge verdicts were simulated because the Judge's model was unavailable.
LLM Judges: the estimated model calls, tokens and list-price cost (USD) before calling. Null for built-in and code Judges.
Show child attributes
Show child attributes
LLM Judges: what Zelto billed for the try (USD), like a tiny run: list cost plus margin, or the bring-your-own-key fee. Counts toward month-to-date benchmark spend. Null for built-in and code Judges.
LLM Judges: the provider list price (USD) of the calls made.
LLM Judges: the try's billing id (one benchmark_spend charge per try).