curl --request POST \
--url https://api.lightsage.com/v2/evals \
--header 'Content-Type: application/json' \
--header 'X-Lightsage-Api-Key: <api-key>' \
--data '
{
"agent": [
"codex:gpt-5-4"
],
"cleanup_commands": [
"npm run db:cleanup"
],
"clis": [
"brew install lightsagehq/tools/lightsage"
],
"judges": [
"The SDK is installed and the request succeeds."
],
"name": "SDK installation",
"prompts": [
"Install the SDK and send a successful request."
],
"runs": 2,
"search_grounding": "disabled",
"setup_commands": [
"npm install",
"npm run db:migrate"
],
"tags": [
"sdk",
"installation"
]
}
'import requests
url = "https://api.lightsage.com/v2/evals"
payload = {
"agent": ["codex:gpt-5-4"],
"cleanup_commands": ["npm run db:cleanup"],
"clis": ["brew install lightsagehq/tools/lightsage"],
"judges": ["The SDK is installed and the request succeeds."],
"name": "SDK installation",
"prompts": ["Install the SDK and send a successful request."],
"runs": 2,
"search_grounding": "disabled",
"setup_commands": ["npm install", "npm run db:migrate"],
"tags": ["sdk", "installation"]
}
headers = {
"X-Lightsage-Api-Key": "<api-key>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {'X-Lightsage-Api-Key': '<api-key>', 'Content-Type': 'application/json'},
body: JSON.stringify({
agent: ['codex:gpt-5-4'],
cleanup_commands: ['npm run db:cleanup'],
clis: ['brew install lightsagehq/tools/lightsage'],
judges: ['The SDK is installed and the request succeeds.'],
name: 'SDK installation',
prompts: ['Install the SDK and send a successful request.'],
runs: 2,
search_grounding: 'disabled',
setup_commands: ['npm install', 'npm run db:migrate'],
tags: ['sdk', 'installation']
})
};
fetch('https://api.lightsage.com/v2/evals', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.lightsage.com/v2/evals",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'agent' => [
'codex:gpt-5-4'
],
'cleanup_commands' => [
'npm run db:cleanup'
],
'clis' => [
'brew install lightsagehq/tools/lightsage'
],
'judges' => [
'The SDK is installed and the request succeeds.'
],
'name' => 'SDK installation',
'prompts' => [
'Install the SDK and send a successful request.'
],
'runs' => 2,
'search_grounding' => 'disabled',
'setup_commands' => [
'npm install',
'npm run db:migrate'
],
'tags' => [
'sdk',
'installation'
]
]),
CURLOPT_HTTPHEADER => [
"Content-Type: application/json",
"X-Lightsage-Api-Key: <api-key>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.lightsage.com/v2/evals"
payload := strings.NewReader("{\n \"agent\": [\n \"codex:gpt-5-4\"\n ],\n \"cleanup_commands\": [\n \"npm run db:cleanup\"\n ],\n \"clis\": [\n \"brew install lightsagehq/tools/lightsage\"\n ],\n \"judges\": [\n \"The SDK is installed and the request succeeds.\"\n ],\n \"name\": \"SDK installation\",\n \"prompts\": [\n \"Install the SDK and send a successful request.\"\n ],\n \"runs\": 2,\n \"search_grounding\": \"disabled\",\n \"setup_commands\": [\n \"npm install\",\n \"npm run db:migrate\"\n ],\n \"tags\": [\n \"sdk\",\n \"installation\"\n ]\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("X-Lightsage-Api-Key", "<api-key>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.lightsage.com/v2/evals")
.header("X-Lightsage-Api-Key", "<api-key>")
.header("Content-Type", "application/json")
.body("{\n \"agent\": [\n \"codex:gpt-5-4\"\n ],\n \"cleanup_commands\": [\n \"npm run db:cleanup\"\n ],\n \"clis\": [\n \"brew install lightsagehq/tools/lightsage\"\n ],\n \"judges\": [\n \"The SDK is installed and the request succeeds.\"\n ],\n \"name\": \"SDK installation\",\n \"prompts\": [\n \"Install the SDK and send a successful request.\"\n ],\n \"runs\": 2,\n \"search_grounding\": \"disabled\",\n \"setup_commands\": [\n \"npm install\",\n \"npm run db:migrate\"\n ],\n \"tags\": [\n \"sdk\",\n \"installation\"\n ]\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.lightsage.com/v2/evals")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["X-Lightsage-Api-Key"] = '<api-key>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"agent\": [\n \"codex:gpt-5-4\"\n ],\n \"cleanup_commands\": [\n \"npm run db:cleanup\"\n ],\n \"clis\": [\n \"brew install lightsagehq/tools/lightsage\"\n ],\n \"judges\": [\n \"The SDK is installed and the request succeeds.\"\n ],\n \"name\": \"SDK installation\",\n \"prompts\": [\n \"Install the SDK and send a successful request.\"\n ],\n \"runs\": 2,\n \"search_grounding\": \"disabled\",\n \"setup_commands\": [\n \"npm install\",\n \"npm run db:migrate\"\n ],\n \"tags\": [\n \"sdk\",\n \"installation\"\n ]\n}"
response = http.request(request)
puts response.read_body{
"id": "<string>",
"name": "<string>",
"prompts": [
"<string>"
],
"judges": [
"<string>"
],
"agent": [
"<string>"
],
"persona": "<string>",
"clis": [
{
"id": "<string>"
}
],
"mcps": [
{
"id": "<string>"
}
],
"setup_commands": [
"<string>"
],
"cleanup_commands": [
"<string>"
],
"search_grounding": "default",
"runs": 1,
"tags": [
"<string>"
],
"created_at": "<string>",
"updated_at": "<string>"
}{
"detail": [
{
"loc": [
"<string>"
],
"msg": "<string>",
"type": "<string>",
"input": "<unknown>",
"ctx": {}
}
]
}Create Eval
Creates a reusable simulation definition with one or more task prompts and judges. Optional agents, persona, CLI installations, MCP servers, setup commands, cleanup commands, grounding policy, repeat count, and tags are saved with the eval. search_grounding supports disabled, enabled, and default; default preserves normal platform behavior. Creating the eval does not start a run and does not accept secret environment values.
curl --request POST \
--url https://api.lightsage.com/v2/evals \
--header 'Content-Type: application/json' \
--header 'X-Lightsage-Api-Key: <api-key>' \
--data '
{
"agent": [
"codex:gpt-5-4"
],
"cleanup_commands": [
"npm run db:cleanup"
],
"clis": [
"brew install lightsagehq/tools/lightsage"
],
"judges": [
"The SDK is installed and the request succeeds."
],
"name": "SDK installation",
"prompts": [
"Install the SDK and send a successful request."
],
"runs": 2,
"search_grounding": "disabled",
"setup_commands": [
"npm install",
"npm run db:migrate"
],
"tags": [
"sdk",
"installation"
]
}
'import requests
url = "https://api.lightsage.com/v2/evals"
payload = {
"agent": ["codex:gpt-5-4"],
"cleanup_commands": ["npm run db:cleanup"],
"clis": ["brew install lightsagehq/tools/lightsage"],
"judges": ["The SDK is installed and the request succeeds."],
"name": "SDK installation",
"prompts": ["Install the SDK and send a successful request."],
"runs": 2,
"search_grounding": "disabled",
"setup_commands": ["npm install", "npm run db:migrate"],
"tags": ["sdk", "installation"]
}
headers = {
"X-Lightsage-Api-Key": "<api-key>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {'X-Lightsage-Api-Key': '<api-key>', 'Content-Type': 'application/json'},
body: JSON.stringify({
agent: ['codex:gpt-5-4'],
cleanup_commands: ['npm run db:cleanup'],
clis: ['brew install lightsagehq/tools/lightsage'],
judges: ['The SDK is installed and the request succeeds.'],
name: 'SDK installation',
prompts: ['Install the SDK and send a successful request.'],
runs: 2,
search_grounding: 'disabled',
setup_commands: ['npm install', 'npm run db:migrate'],
tags: ['sdk', 'installation']
})
};
fetch('https://api.lightsage.com/v2/evals', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.lightsage.com/v2/evals",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'agent' => [
'codex:gpt-5-4'
],
'cleanup_commands' => [
'npm run db:cleanup'
],
'clis' => [
'brew install lightsagehq/tools/lightsage'
],
'judges' => [
'The SDK is installed and the request succeeds.'
],
'name' => 'SDK installation',
'prompts' => [
'Install the SDK and send a successful request.'
],
'runs' => 2,
'search_grounding' => 'disabled',
'setup_commands' => [
'npm install',
'npm run db:migrate'
],
'tags' => [
'sdk',
'installation'
]
]),
CURLOPT_HTTPHEADER => [
"Content-Type: application/json",
"X-Lightsage-Api-Key: <api-key>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.lightsage.com/v2/evals"
payload := strings.NewReader("{\n \"agent\": [\n \"codex:gpt-5-4\"\n ],\n \"cleanup_commands\": [\n \"npm run db:cleanup\"\n ],\n \"clis\": [\n \"brew install lightsagehq/tools/lightsage\"\n ],\n \"judges\": [\n \"The SDK is installed and the request succeeds.\"\n ],\n \"name\": \"SDK installation\",\n \"prompts\": [\n \"Install the SDK and send a successful request.\"\n ],\n \"runs\": 2,\n \"search_grounding\": \"disabled\",\n \"setup_commands\": [\n \"npm install\",\n \"npm run db:migrate\"\n ],\n \"tags\": [\n \"sdk\",\n \"installation\"\n ]\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("X-Lightsage-Api-Key", "<api-key>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.lightsage.com/v2/evals")
.header("X-Lightsage-Api-Key", "<api-key>")
.header("Content-Type", "application/json")
.body("{\n \"agent\": [\n \"codex:gpt-5-4\"\n ],\n \"cleanup_commands\": [\n \"npm run db:cleanup\"\n ],\n \"clis\": [\n \"brew install lightsagehq/tools/lightsage\"\n ],\n \"judges\": [\n \"The SDK is installed and the request succeeds.\"\n ],\n \"name\": \"SDK installation\",\n \"prompts\": [\n \"Install the SDK and send a successful request.\"\n ],\n \"runs\": 2,\n \"search_grounding\": \"disabled\",\n \"setup_commands\": [\n \"npm install\",\n \"npm run db:migrate\"\n ],\n \"tags\": [\n \"sdk\",\n \"installation\"\n ]\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.lightsage.com/v2/evals")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["X-Lightsage-Api-Key"] = '<api-key>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"agent\": [\n \"codex:gpt-5-4\"\n ],\n \"cleanup_commands\": [\n \"npm run db:cleanup\"\n ],\n \"clis\": [\n \"brew install lightsagehq/tools/lightsage\"\n ],\n \"judges\": [\n \"The SDK is installed and the request succeeds.\"\n ],\n \"name\": \"SDK installation\",\n \"prompts\": [\n \"Install the SDK and send a successful request.\"\n ],\n \"runs\": 2,\n \"search_grounding\": \"disabled\",\n \"setup_commands\": [\n \"npm install\",\n \"npm run db:migrate\"\n ],\n \"tags\": [\n \"sdk\",\n \"installation\"\n ]\n}"
response = http.request(request)
puts response.read_body{
"id": "<string>",
"name": "<string>",
"prompts": [
"<string>"
],
"judges": [
"<string>"
],
"agent": [
"<string>"
],
"persona": "<string>",
"clis": [
{
"id": "<string>"
}
],
"mcps": [
{
"id": "<string>"
}
],
"setup_commands": [
"<string>"
],
"cleanup_commands": [
"<string>"
],
"search_grounding": "default",
"runs": 1,
"tags": [
"<string>"
],
"created_at": "<string>",
"updated_at": "<string>"
}{
"detail": [
{
"loc": [
"<string>"
],
"msg": "<string>",
"type": "<string>",
"input": "<unknown>",
"ctx": {}
}
]
}Authorizations
Body
1 - 160Task prompts. CLI clients expose this as repeatable --prompt.
1 - 20 elementsPlain-text success criteria. CLI clients expose this as repeatable --judge.
1 - 20 elementsRunnable agents from GET /v2/agents. An empty array saves the eval without default agents. CLI clients expose repeatable --agent.
10Optional perspective applied to the whole eval. It is stored as metadata and does not change prompts or multiply attempts.
160Saved CLI IDs or shell installation commands. UUID values resolve saved CLIs from GET /v2/clis; every other value is run as an installation command. CLI clients expose repeatable --cli.
20Saved MCP references or inline portable MCP definitions.
20- V2SavedMcpReference
- V2InlineMcpDefinition
Show child attributes
Show child attributes
Shell commands run sequentially in the prepared repository before the agent starts. CLI clients expose repeatable --setup-command.
20Best-effort shell commands run sequentially after agent execution and result capture, before sandbox deletion. CLI clients expose repeatable --cleanup-command.
20Web-retrieval policy for every run. disabled hides native search and fetch tools and requests network isolation; enabled exposes retrieval tools, requires live retrieval in the runtime prompt, and qualifies the run only when retrieval is observed; default exposes the platform's normal tools without requiring retrieval.
disabled, enabled, default Independent attempts for every prompt-agent pair.
1 <= x <= 20Optional labels for organizing the eval. CLI clients may expose this string array as a repeatable --tag flag.
20Response
Successful Response
- V2SavedCliReference
- V2InlineCliReference
Show child attributes
Show child attributes
- V2SavedMcpReference
- V2InlineMcpDefinition
Show child attributes
Show child attributes
Saved web-retrieval policy: disabled, enabled, or default.
disabled, enabled, default 1 <= x <= 20