Probar un Judge
curl --request POST \
--url https://api.zelto.ai/v1/benchmarks/{id}/judges/try \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"judge": {
"label": "<string>",
"config": {
"kind": "judge",
"rubric": "<string>",
"model": {
"modelId": "<string>",
"aiKeyId": "<string>"
},
"scale": {
"type": "pass_fail"
}
},
"labelTemplate": "<string>",
"required": true,
"weight": 1,
"checkGroupId": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"appliesTo": {},
"params": []
},
"caseIds": [
"3c90c3cc-0d44-4b50-8888-8dd25736052a"
],
"estimateOnly": true
}
'import requests
url = "https://api.zelto.ai/v1/benchmarks/{id}/judges/try"
payload = {
"judge": {
"label": "<string>",
"config": {
"kind": "judge",
"rubric": "<string>",
"model": {
"modelId": "<string>",
"aiKeyId": "<string>"
},
"scale": { "type": "pass_fail" }
},
"labelTemplate": "<string>",
"required": True,
"weight": 1,
"checkGroupId": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"appliesTo": {},
"params": []
},
"caseIds": ["3c90c3cc-0d44-4b50-8888-8dd25736052a"],
"estimateOnly": True
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
judge: {
label: '<string>',
config: {
kind: 'judge',
rubric: '<string>',
model: {modelId: '<string>', aiKeyId: '<string>'},
scale: {type: 'pass_fail'}
},
labelTemplate: '<string>',
required: true,
weight: 1,
checkGroupId: '3c90c3cc-0d44-4b50-8888-8dd25736052a',
appliesTo: {},
params: []
},
caseIds: ['3c90c3cc-0d44-4b50-8888-8dd25736052a'],
estimateOnly: true
})
};
fetch('https://api.zelto.ai/v1/benchmarks/{id}/judges/try', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.zelto.ai/v1/benchmarks/{id}/judges/try",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'judge' => [
'label' => '<string>',
'config' => [
'kind' => 'judge',
'rubric' => '<string>',
'model' => [
'modelId' => '<string>',
'aiKeyId' => '<string>'
],
'scale' => [
'type' => 'pass_fail'
]
],
'labelTemplate' => '<string>',
'required' => true,
'weight' => 1,
'checkGroupId' => '3c90c3cc-0d44-4b50-8888-8dd25736052a',
'appliesTo' => [
],
'params' => [
]
],
'caseIds' => [
'3c90c3cc-0d44-4b50-8888-8dd25736052a'
],
'estimateOnly' => true
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.zelto.ai/v1/benchmarks/{id}/judges/try"
payload := strings.NewReader("{\n \"judge\": {\n \"label\": \"<string>\",\n \"config\": {\n \"kind\": \"judge\",\n \"rubric\": \"<string>\",\n \"model\": {\n \"modelId\": \"<string>\",\n \"aiKeyId\": \"<string>\"\n },\n \"scale\": {\n \"type\": \"pass_fail\"\n }\n },\n \"labelTemplate\": \"<string>\",\n \"required\": true,\n \"weight\": 1,\n \"checkGroupId\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n \"appliesTo\": {},\n \"params\": []\n },\n \"caseIds\": [\n \"3c90c3cc-0d44-4b50-8888-8dd25736052a\"\n ],\n \"estimateOnly\": true\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.zelto.ai/v1/benchmarks/{id}/judges/try")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"judge\": {\n \"label\": \"<string>\",\n \"config\": {\n \"kind\": \"judge\",\n \"rubric\": \"<string>\",\n \"model\": {\n \"modelId\": \"<string>\",\n \"aiKeyId\": \"<string>\"\n },\n \"scale\": {\n \"type\": \"pass_fail\"\n }\n },\n \"labelTemplate\": \"<string>\",\n \"required\": true,\n \"weight\": 1,\n \"checkGroupId\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n \"appliesTo\": {},\n \"params\": []\n },\n \"caseIds\": [\n \"3c90c3cc-0d44-4b50-8888-8dd25736052a\"\n ],\n \"estimateOnly\": true\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.zelto.ai/v1/benchmarks/{id}/judges/try")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"judge\": {\n \"label\": \"<string>\",\n \"config\": {\n \"kind\": \"judge\",\n \"rubric\": \"<string>\",\n \"model\": {\n \"modelId\": \"<string>\",\n \"aiKeyId\": \"<string>\"\n },\n \"scale\": {\n \"type\": \"pass_fail\"\n }\n },\n \"labelTemplate\": \"<string>\",\n \"required\": true,\n \"weight\": 1,\n \"checkGroupId\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n \"appliesTo\": {},\n \"params\": []\n },\n \"caseIds\": [\n \"3c90c3cc-0d44-4b50-8888-8dd25736052a\"\n ],\n \"estimateOnly\": true\n}"
response = http.request(request)
puts response.read_body{
"results": [
{
"caseId": "<string>",
"passed": true,
"value": "<string>",
"reasoning": "<string>",
"error": "<string>",
"notApplicable": true
}
],
"simulated": true,
"estimate": {
"calls": 123,
"inputTokens": 123,
"outputTokens": 123,
"costUsd": 123,
"model": "<string>",
"unpriced": true
},
"costUsd": 123,
"listCostUsd": 123,
"tryId": "<string>"
}{
"error": "<string>",
"message": "<string>",
"code": "<string>",
"hint": "<string>",
"parameter": "<string>",
"unknown": [
"<string>"
],
"valid": [
"<string>"
],
"details": {}
}{
"error": "<string>",
"message": "<string>",
"code": "<string>",
"hint": "<string>",
"parameter": "<string>",
"unknown": [
"<string>"
],
"valid": [
"<string>"
],
"details": {}
}{
"error": "validation_failed",
"code": "validation_failed",
"message": "<string>",
"hint": "<string>",
"details": {}
}{
"error": "<string>",
"message": "<string>",
"code": "<string>",
"hint": "<string>",
"parameter": "<string>",
"unknown": [
"<string>"
],
"valid": [
"<string>"
],
"details": {}
}Benchmarks
Probar un Judge
Puntúa las respuestas grabadas de hasta 5 casos con una configuración de Judge sin guardar. Los Judges LLM llaman a su modelo (una llamada por caso) y devuelven la estimación y el costo real; envía estimateOnly: true para obtener primero la estimación sin llamar al modelo. No se guarda nada, pero las llamadas al modelo de un Judge LLM se cobran al gasto de benchmarks como una ejecución pequeña y cuentan para el gasto del mes en curso (costUsd es lo que Zelto cobró por la prueba y listCostUsd, el precio de lista del proveedor); los Judges integrados y de código son gratuitos.
POST
/
v1
/
benchmarks
/
{id}
/
judges
/
try
Probar un Judge
curl --request POST \
--url https://api.zelto.ai/v1/benchmarks/{id}/judges/try \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"judge": {
"label": "<string>",
"config": {
"kind": "judge",
"rubric": "<string>",
"model": {
"modelId": "<string>",
"aiKeyId": "<string>"
},
"scale": {
"type": "pass_fail"
}
},
"labelTemplate": "<string>",
"required": true,
"weight": 1,
"checkGroupId": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"appliesTo": {},
"params": []
},
"caseIds": [
"3c90c3cc-0d44-4b50-8888-8dd25736052a"
],
"estimateOnly": true
}
'import requests
url = "https://api.zelto.ai/v1/benchmarks/{id}/judges/try"
payload = {
"judge": {
"label": "<string>",
"config": {
"kind": "judge",
"rubric": "<string>",
"model": {
"modelId": "<string>",
"aiKeyId": "<string>"
},
"scale": { "type": "pass_fail" }
},
"labelTemplate": "<string>",
"required": True,
"weight": 1,
"checkGroupId": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"appliesTo": {},
"params": []
},
"caseIds": ["3c90c3cc-0d44-4b50-8888-8dd25736052a"],
"estimateOnly": True
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
judge: {
label: '<string>',
config: {
kind: 'judge',
rubric: '<string>',
model: {modelId: '<string>', aiKeyId: '<string>'},
scale: {type: 'pass_fail'}
},
labelTemplate: '<string>',
required: true,
weight: 1,
checkGroupId: '3c90c3cc-0d44-4b50-8888-8dd25736052a',
appliesTo: {},
params: []
},
caseIds: ['3c90c3cc-0d44-4b50-8888-8dd25736052a'],
estimateOnly: true
})
};
fetch('https://api.zelto.ai/v1/benchmarks/{id}/judges/try', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.zelto.ai/v1/benchmarks/{id}/judges/try",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'judge' => [
'label' => '<string>',
'config' => [
'kind' => 'judge',
'rubric' => '<string>',
'model' => [
'modelId' => '<string>',
'aiKeyId' => '<string>'
],
'scale' => [
'type' => 'pass_fail'
]
],
'labelTemplate' => '<string>',
'required' => true,
'weight' => 1,
'checkGroupId' => '3c90c3cc-0d44-4b50-8888-8dd25736052a',
'appliesTo' => [
],
'params' => [
]
],
'caseIds' => [
'3c90c3cc-0d44-4b50-8888-8dd25736052a'
],
'estimateOnly' => true
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.zelto.ai/v1/benchmarks/{id}/judges/try"
payload := strings.NewReader("{\n \"judge\": {\n \"label\": \"<string>\",\n \"config\": {\n \"kind\": \"judge\",\n \"rubric\": \"<string>\",\n \"model\": {\n \"modelId\": \"<string>\",\n \"aiKeyId\": \"<string>\"\n },\n \"scale\": {\n \"type\": \"pass_fail\"\n }\n },\n \"labelTemplate\": \"<string>\",\n \"required\": true,\n \"weight\": 1,\n \"checkGroupId\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n \"appliesTo\": {},\n \"params\": []\n },\n \"caseIds\": [\n \"3c90c3cc-0d44-4b50-8888-8dd25736052a\"\n ],\n \"estimateOnly\": true\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.zelto.ai/v1/benchmarks/{id}/judges/try")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"judge\": {\n \"label\": \"<string>\",\n \"config\": {\n \"kind\": \"judge\",\n \"rubric\": \"<string>\",\n \"model\": {\n \"modelId\": \"<string>\",\n \"aiKeyId\": \"<string>\"\n },\n \"scale\": {\n \"type\": \"pass_fail\"\n }\n },\n \"labelTemplate\": \"<string>\",\n \"required\": true,\n \"weight\": 1,\n \"checkGroupId\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n \"appliesTo\": {},\n \"params\": []\n },\n \"caseIds\": [\n \"3c90c3cc-0d44-4b50-8888-8dd25736052a\"\n ],\n \"estimateOnly\": true\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.zelto.ai/v1/benchmarks/{id}/judges/try")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"judge\": {\n \"label\": \"<string>\",\n \"config\": {\n \"kind\": \"judge\",\n \"rubric\": \"<string>\",\n \"model\": {\n \"modelId\": \"<string>\",\n \"aiKeyId\": \"<string>\"\n },\n \"scale\": {\n \"type\": \"pass_fail\"\n }\n },\n \"labelTemplate\": \"<string>\",\n \"required\": true,\n \"weight\": 1,\n \"checkGroupId\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n \"appliesTo\": {},\n \"params\": []\n },\n \"caseIds\": [\n \"3c90c3cc-0d44-4b50-8888-8dd25736052a\"\n ],\n \"estimateOnly\": true\n}"
response = http.request(request)
puts response.read_body{
"results": [
{
"caseId": "<string>",
"passed": true,
"value": "<string>",
"reasoning": "<string>",
"error": "<string>",
"notApplicable": true
}
],
"simulated": true,
"estimate": {
"calls": 123,
"inputTokens": 123,
"outputTokens": 123,
"costUsd": 123,
"model": "<string>",
"unpriced": true
},
"costUsd": 123,
"listCostUsd": 123,
"tryId": "<string>"
}{
"error": "<string>",
"message": "<string>",
"code": "<string>",
"hint": "<string>",
"parameter": "<string>",
"unknown": [
"<string>"
],
"valid": [
"<string>"
],
"details": {}
}{
"error": "<string>",
"message": "<string>",
"code": "<string>",
"hint": "<string>",
"parameter": "<string>",
"unknown": [
"<string>"
],
"valid": [
"<string>"
],
"details": {}
}{
"error": "validation_failed",
"code": "validation_failed",
"message": "<string>",
"hint": "<string>",
"details": {}
}{
"error": "<string>",
"message": "<string>",
"code": "<string>",
"hint": "<string>",
"parameter": "<string>",
"unknown": [
"<string>"
],
"valid": [
"<string>"
],
"details": {}
}Autorizaciones
Envía Authorization: Bearer YOUR_API_KEY. La clave corresponde a una sola organización.
Parámetros de ruta
Id del benchmark.
Cuerpo
application/json
Respuesta
Respuesta correcta.
Veredictos de probar un Judge en algunos casos. No se guarda nada; las llamadas al modelo de un Judge LLM se cobran al gasto de benchmarks como una ejecución pequeña.
Show child attributes
Show child attributes
Verdadero cuando los veredictos del Judge LLM se simularon porque su modelo no estaba disponible.
Judges LLM: las llamadas al modelo, los tokens y el costo a precio de lista (USD) estimados antes de llamar. Nulo para Judges integrados y de código.
Show child attributes
Show child attributes
Judges LLM: lo que Zelto facturó por la prueba (USD), como una ejecución pequeña: el costo de lista más el margen, o la tarifa de clave propia. Cuenta para el gasto en benchmarks del mes en curso. Null para los Judges integrados y de código.
Judges LLM: el precio de lista del proveedor (USD) de las llamadas realizadas.
Judges LLM: el id de facturación de la prueba (un cargo de benchmark_spend por prueba).