Create an Eval template
curl --request POST \
--url https://api.lightsage.com/v2/eval-templates \
--header 'Content-Type: application/json' \
--header 'X-Lightsage-Api-Key: <api-key>' \
--data '
{
"description": "Reusable policy for crypto recommendation evals.",
"eval_type": "custom_eval",
"included_components": [
"judges",
"targets",
"schedule"
],
"judges": [
{
"blocking": false,
"criteria": "Pass only if CoinMarketCap is recommended as the first and main choice. Fail otherwise.",
"name": "CoinMarketCap first",
"type": "judge",
"weight": 1
}
],
"name": "Crypto recommendation",
"schedule": {
"enabled": true,
"frequency": "daily"
},
"target_keys": [
"agent:codex:gpt-5.6-terra",
"agent:claude-code:global.anthropic.claude-sonnet-5"
]
}
'import requests
url = "https://api.lightsage.com/v2/eval-templates"
payload = {
"description": "Reusable policy for crypto recommendation evals.",
"eval_type": "custom_eval",
"included_components": ["judges", "targets", "schedule"],
"judges": [
{
"blocking": False,
"criteria": "Pass only if CoinMarketCap is recommended as the first and main choice. Fail otherwise.",
"name": "CoinMarketCap first",
"type": "judge",
"weight": 1
}
],
"name": "Crypto recommendation",
"schedule": {
"enabled": True,
"frequency": "daily"
},
"target_keys": ["agent:codex:gpt-5.6-terra", "agent:claude-code:global.anthropic.claude-sonnet-5"]
}
headers = {
"X-Lightsage-Api-Key": "<api-key>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {'X-Lightsage-Api-Key': '<api-key>', 'Content-Type': 'application/json'},
body: JSON.stringify({
description: 'Reusable policy for crypto recommendation evals.',
eval_type: 'custom_eval',
included_components: ['judges', 'targets', 'schedule'],
judges: [
{
blocking: false,
criteria: 'Pass only if CoinMarketCap is recommended as the first and main choice. Fail otherwise.',
name: 'CoinMarketCap first',
type: 'judge',
weight: 1
}
],
name: 'Crypto recommendation',
schedule: {enabled: true, frequency: 'daily'},
target_keys: [
'agent:codex:gpt-5.6-terra',
'agent:claude-code:global.anthropic.claude-sonnet-5'
]
})
};
fetch('https://api.lightsage.com/v2/eval-templates', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.lightsage.com/v2/eval-templates",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'description' => 'Reusable policy for crypto recommendation evals.',
'eval_type' => 'custom_eval',
'included_components' => [
'judges',
'targets',
'schedule'
],
'judges' => [
[
'blocking' => false,
'criteria' => 'Pass only if CoinMarketCap is recommended as the first and main choice. Fail otherwise.',
'name' => 'CoinMarketCap first',
'type' => 'judge',
'weight' => 1
]
],
'name' => 'Crypto recommendation',
'schedule' => [
'enabled' => true,
'frequency' => 'daily'
],
'target_keys' => [
'agent:codex:gpt-5.6-terra',
'agent:claude-code:global.anthropic.claude-sonnet-5'
]
]),
CURLOPT_HTTPHEADER => [
"Content-Type: application/json",
"X-Lightsage-Api-Key: <api-key>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.lightsage.com/v2/eval-templates"
payload := strings.NewReader("{\n \"description\": \"Reusable policy for crypto recommendation evals.\",\n \"eval_type\": \"custom_eval\",\n \"included_components\": [\n \"judges\",\n \"targets\",\n \"schedule\"\n ],\n \"judges\": [\n {\n \"blocking\": false,\n \"criteria\": \"Pass only if CoinMarketCap is recommended as the first and main choice. Fail otherwise.\",\n \"name\": \"CoinMarketCap first\",\n \"type\": \"judge\",\n \"weight\": 1\n }\n ],\n \"name\": \"Crypto recommendation\",\n \"schedule\": {\n \"enabled\": true,\n \"frequency\": \"daily\"\n },\n \"target_keys\": [\n \"agent:codex:gpt-5.6-terra\",\n \"agent:claude-code:global.anthropic.claude-sonnet-5\"\n ]\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("X-Lightsage-Api-Key", "<api-key>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.lightsage.com/v2/eval-templates")
.header("X-Lightsage-Api-Key", "<api-key>")
.header("Content-Type", "application/json")
.body("{\n \"description\": \"Reusable policy for crypto recommendation evals.\",\n \"eval_type\": \"custom_eval\",\n \"included_components\": [\n \"judges\",\n \"targets\",\n \"schedule\"\n ],\n \"judges\": [\n {\n \"blocking\": false,\n \"criteria\": \"Pass only if CoinMarketCap is recommended as the first and main choice. Fail otherwise.\",\n \"name\": \"CoinMarketCap first\",\n \"type\": \"judge\",\n \"weight\": 1\n }\n ],\n \"name\": \"Crypto recommendation\",\n \"schedule\": {\n \"enabled\": true,\n \"frequency\": \"daily\"\n },\n \"target_keys\": [\n \"agent:codex:gpt-5.6-terra\",\n \"agent:claude-code:global.anthropic.claude-sonnet-5\"\n ]\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.lightsage.com/v2/eval-templates")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["X-Lightsage-Api-Key"] = '<api-key>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"description\": \"Reusable policy for crypto recommendation evals.\",\n \"eval_type\": \"custom_eval\",\n \"included_components\": [\n \"judges\",\n \"targets\",\n \"schedule\"\n ],\n \"judges\": [\n {\n \"blocking\": false,\n \"criteria\": \"Pass only if CoinMarketCap is recommended as the first and main choice. Fail otherwise.\",\n \"name\": \"CoinMarketCap first\",\n \"type\": \"judge\",\n \"weight\": 1\n }\n ],\n \"name\": \"Crypto recommendation\",\n \"schedule\": {\n \"enabled\": true,\n \"frequency\": \"daily\"\n },\n \"target_keys\": [\n \"agent:codex:gpt-5.6-terra\",\n \"agent:claude-code:global.anthropic.claude-sonnet-5\"\n ]\n}"
response = http.request(request)
puts response.read_body{
"created_at": "2026-09-24T18:00:00Z",
"created_by": "0b6d843f-cd4e-4820-8999-5c6dac8ca6f2",
"description": "Reusable policy for crypto recommendation evals.",
"eval_type": "custom_eval",
"id": "58f3604e-d58e-4eb8-b1fe-3478be875c36",
"included_components": [
"judges",
"targets",
"schedule"
],
"judges": [
{
"blocking": false,
"criteria": "Pass only if CoinMarketCap is recommended as the first and main choice. Fail otherwise.",
"name": "CoinMarketCap first",
"type": "judge",
"weight": 1
}
],
"name": "Crypto recommendation",
"org_id": "ab9fe05c-5224-4d9b-a382-a23834ec1947",
"schedule": {
"enabled": true,
"frequency": "daily"
},
"target_keys": [
"agent:codex:gpt-5.6-terra",
"agent:claude-code:global.anthropic.claude-sonnet-5"
],
"updated_at": "2026-09-24T18:00:00Z",
"version": 1
}{
"detail": "Unknown or inactive competitor ID"
}{
"detail": [
{
"loc": [
"<string>"
],
"msg": "<string>",
"type": "<string>",
"input": "<unknown>",
"ctx": {}
}
]
}Eval Templates
Create an Eval template
Step 1 of the reusable template workflow. Creates an organization-owned Eval template containing only the configuration components selected in included_components. Do not include task prompts in the template. The response returns the template id and version; pass them as template_id and expected_template_version to POST /v2/evals/batch with each eval’s name and prompts. Applying a template copies its values into the new evals without creating a live link.
POST
/
v2
/
eval-templates
Create an Eval template
curl --request POST \
--url https://api.lightsage.com/v2/eval-templates \
--header 'Content-Type: application/json' \
--header 'X-Lightsage-Api-Key: <api-key>' \
--data '
{
"description": "Reusable policy for crypto recommendation evals.",
"eval_type": "custom_eval",
"included_components": [
"judges",
"targets",
"schedule"
],
"judges": [
{
"blocking": false,
"criteria": "Pass only if CoinMarketCap is recommended as the first and main choice. Fail otherwise.",
"name": "CoinMarketCap first",
"type": "judge",
"weight": 1
}
],
"name": "Crypto recommendation",
"schedule": {
"enabled": true,
"frequency": "daily"
},
"target_keys": [
"agent:codex:gpt-5.6-terra",
"agent:claude-code:global.anthropic.claude-sonnet-5"
]
}
'import requests
url = "https://api.lightsage.com/v2/eval-templates"
payload = {
"description": "Reusable policy for crypto recommendation evals.",
"eval_type": "custom_eval",
"included_components": ["judges", "targets", "schedule"],
"judges": [
{
"blocking": False,
"criteria": "Pass only if CoinMarketCap is recommended as the first and main choice. Fail otherwise.",
"name": "CoinMarketCap first",
"type": "judge",
"weight": 1
}
],
"name": "Crypto recommendation",
"schedule": {
"enabled": True,
"frequency": "daily"
},
"target_keys": ["agent:codex:gpt-5.6-terra", "agent:claude-code:global.anthropic.claude-sonnet-5"]
}
headers = {
"X-Lightsage-Api-Key": "<api-key>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {'X-Lightsage-Api-Key': '<api-key>', 'Content-Type': 'application/json'},
body: JSON.stringify({
description: 'Reusable policy for crypto recommendation evals.',
eval_type: 'custom_eval',
included_components: ['judges', 'targets', 'schedule'],
judges: [
{
blocking: false,
criteria: 'Pass only if CoinMarketCap is recommended as the first and main choice. Fail otherwise.',
name: 'CoinMarketCap first',
type: 'judge',
weight: 1
}
],
name: 'Crypto recommendation',
schedule: {enabled: true, frequency: 'daily'},
target_keys: [
'agent:codex:gpt-5.6-terra',
'agent:claude-code:global.anthropic.claude-sonnet-5'
]
})
};
fetch('https://api.lightsage.com/v2/eval-templates', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.lightsage.com/v2/eval-templates",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'description' => 'Reusable policy for crypto recommendation evals.',
'eval_type' => 'custom_eval',
'included_components' => [
'judges',
'targets',
'schedule'
],
'judges' => [
[
'blocking' => false,
'criteria' => 'Pass only if CoinMarketCap is recommended as the first and main choice. Fail otherwise.',
'name' => 'CoinMarketCap first',
'type' => 'judge',
'weight' => 1
]
],
'name' => 'Crypto recommendation',
'schedule' => [
'enabled' => true,
'frequency' => 'daily'
],
'target_keys' => [
'agent:codex:gpt-5.6-terra',
'agent:claude-code:global.anthropic.claude-sonnet-5'
]
]),
CURLOPT_HTTPHEADER => [
"Content-Type: application/json",
"X-Lightsage-Api-Key: <api-key>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.lightsage.com/v2/eval-templates"
payload := strings.NewReader("{\n \"description\": \"Reusable policy for crypto recommendation evals.\",\n \"eval_type\": \"custom_eval\",\n \"included_components\": [\n \"judges\",\n \"targets\",\n \"schedule\"\n ],\n \"judges\": [\n {\n \"blocking\": false,\n \"criteria\": \"Pass only if CoinMarketCap is recommended as the first and main choice. Fail otherwise.\",\n \"name\": \"CoinMarketCap first\",\n \"type\": \"judge\",\n \"weight\": 1\n }\n ],\n \"name\": \"Crypto recommendation\",\n \"schedule\": {\n \"enabled\": true,\n \"frequency\": \"daily\"\n },\n \"target_keys\": [\n \"agent:codex:gpt-5.6-terra\",\n \"agent:claude-code:global.anthropic.claude-sonnet-5\"\n ]\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("X-Lightsage-Api-Key", "<api-key>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.lightsage.com/v2/eval-templates")
.header("X-Lightsage-Api-Key", "<api-key>")
.header("Content-Type", "application/json")
.body("{\n \"description\": \"Reusable policy for crypto recommendation evals.\",\n \"eval_type\": \"custom_eval\",\n \"included_components\": [\n \"judges\",\n \"targets\",\n \"schedule\"\n ],\n \"judges\": [\n {\n \"blocking\": false,\n \"criteria\": \"Pass only if CoinMarketCap is recommended as the first and main choice. Fail otherwise.\",\n \"name\": \"CoinMarketCap first\",\n \"type\": \"judge\",\n \"weight\": 1\n }\n ],\n \"name\": \"Crypto recommendation\",\n \"schedule\": {\n \"enabled\": true,\n \"frequency\": \"daily\"\n },\n \"target_keys\": [\n \"agent:codex:gpt-5.6-terra\",\n \"agent:claude-code:global.anthropic.claude-sonnet-5\"\n ]\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.lightsage.com/v2/eval-templates")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["X-Lightsage-Api-Key"] = '<api-key>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"description\": \"Reusable policy for crypto recommendation evals.\",\n \"eval_type\": \"custom_eval\",\n \"included_components\": [\n \"judges\",\n \"targets\",\n \"schedule\"\n ],\n \"judges\": [\n {\n \"blocking\": false,\n \"criteria\": \"Pass only if CoinMarketCap is recommended as the first and main choice. Fail otherwise.\",\n \"name\": \"CoinMarketCap first\",\n \"type\": \"judge\",\n \"weight\": 1\n }\n ],\n \"name\": \"Crypto recommendation\",\n \"schedule\": {\n \"enabled\": true,\n \"frequency\": \"daily\"\n },\n \"target_keys\": [\n \"agent:codex:gpt-5.6-terra\",\n \"agent:claude-code:global.anthropic.claude-sonnet-5\"\n ]\n}"
response = http.request(request)
puts response.read_body{
"created_at": "2026-09-24T18:00:00Z",
"created_by": "0b6d843f-cd4e-4820-8999-5c6dac8ca6f2",
"description": "Reusable policy for crypto recommendation evals.",
"eval_type": "custom_eval",
"id": "58f3604e-d58e-4eb8-b1fe-3478be875c36",
"included_components": [
"judges",
"targets",
"schedule"
],
"judges": [
{
"blocking": false,
"criteria": "Pass only if CoinMarketCap is recommended as the first and main choice. Fail otherwise.",
"name": "CoinMarketCap first",
"type": "judge",
"weight": 1
}
],
"name": "Crypto recommendation",
"org_id": "ab9fe05c-5224-4d9b-a382-a23834ec1947",
"schedule": {
"enabled": true,
"frequency": "daily"
},
"target_keys": [
"agent:codex:gpt-5.6-terra",
"agent:claude-code:global.anthropic.claude-sonnet-5"
],
"updated_at": "2026-09-24T18:00:00Z",
"version": 1
}{
"detail": "Unknown or inactive competitor ID"
}{
"detail": [
{
"loc": [
"<string>"
],
"msg": "<string>",
"type": "<string>",
"input": "<unknown>",
"ctx": {}
}
]
}Authorizations
Body
application/json
Create a reusable eval that can be applied to many brands or prompts.
Organization-visible name for this reusable template.
Required string length:
1 - 160Settings copied when the template is applied. Omitted components do not override the new eval; included components copy their saved value, including an empty value.
Minimum array length:
1Available options:
judges, targets, persona, competitors, tag, category, runs, schedule, repository, environment, search_grounding, skills, mcp_servers, clis, brand_docs, github_pr, runtime_commands Optional guidance describing when to use the template.
Maximum string length:
1000Prompt filled into the new eval form when the template is applied, or the scenario for Agent Preference, where {{competitor_name}} names the brand each comparison is against. Requests that create evals from a template still supply their own prompts.
Maximum string length:
20000Kind of eval the template creates. A template only applies to evals of the same type.
Available options:
custom_eval, agent_preference Optional persona copied when persona is included.
Maximum string length:
500Optional category copied when category is included.
Maximum string length:
160Optional tag copied when tag is included.
Maximum string length:
100Competitor IDs that create additional variants when competitors is included.
Judge definitions copied when judges is included. May be empty to explicitly create the eval without template judges.
Maximum array length:
20Show child attributes
Show child attributes
Runnable target keys copied when targets is included, such as agent:codex:gpt-5.6-terra.
Schedule copied when schedule is included.
Show child attributes
Show child attributes
Attempt count copied when runs is included.
Required range:
1 <= x <= 20Saved runtime values. Each value is copied only when its matching component appears in included_components.
Show child attributes
Show child attributes
Response
Successful Response
Organization-visible name for this reusable template.
Required string length:
1 - 160Settings copied when the template is applied. Omitted components do not override the new eval; included components copy their saved value, including an empty value.
Minimum array length:
1Available options:
judges, targets, persona, competitors, tag, category, runs, schedule, repository, environment, search_grounding, skills, mcp_servers, clis, brand_docs, github_pr, runtime_commands Optional guidance describing when to use the template.
Maximum string length:
1000Prompt filled into the new eval form when the template is applied, or the scenario for Agent Preference, where {{competitor_name}} names the brand each comparison is against. Requests that create evals from a template still supply their own prompts.
Maximum string length:
20000Kind of eval the template creates. A template only applies to evals of the same type.
Available options:
custom_eval, agent_preference Optional persona copied when persona is included.
Maximum string length:
500Optional category copied when category is included.
Maximum string length:
160Optional tag copied when tag is included.
Maximum string length:
100Competitor IDs that create additional variants when competitors is included.
Judge definitions copied when judges is included. May be empty to explicitly create the eval without template judges.
Maximum array length:
20Show child attributes
Show child attributes
Runnable target keys copied when targets is included, such as agent:codex:gpt-5.6-terra.
Schedule copied when schedule is included.
Show child attributes
Show child attributes
Attempt count copied when runs is included.
Required range:
1 <= x <= 20Saved runtime values. Each value is copied only when its matching component appears in included_components.
Show child attributes
Show child attributes