{"openapi":"3.1.0","info":{"title":"JustEvals API","version":"1.0.0","description":"Upload eval suites, runs and outputs, and read human feedback. Guide: https://justevals.com/docs/api","contact":{"email":"contact@justevals.com","url":"https://justevals.com/docs/api"}},"servers":[{"url":"https://justevals.com/api/v1"}],"security":[{"apiKey":[]}],"components":{"securitySchemes":{"apiKey":{"type":"http","scheme":"bearer","description":"Project API key (je_…), created in JustEvals under Settings → API keys."}},"responses":{"Invalid":{"description":"The request body is not valid.","content":{"application/json":{"schema":{"type":"object","properties":{"error":{"type":"string"},"details":{"type":"array"}}}}}},"Unauthorized":{"description":"The API key is missing, wrong or revoked."},"NotFound":{"description":"The object does not exist in this project."}}},"paths":{"/project":{"get":{"summary":"Check the key and show the project","responses":{"200":{"description":"The project of this key","content":{"application/json":{"schema":{"type":"object"}}}},"400":{"$ref":"#/components/responses/Invalid"},"401":{"$ref":"#/components/responses/Unauthorized"},"404":{"$ref":"#/components/responses/NotFound"}}}},"/files":{"post":{"summary":"Declare a file by hash and get an upload URL","requestBody":{"required":true,"content":{"application/json":{"schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"sha256":{"type":"string","pattern":"^[a-f0-9]{64}$"},"size":{"type":"integer","exclusiveMinimum":0,"maximum":209715200},"content_type":{"type":"string","enum":["image/jpeg","image/png","image/webp","image/gif","image/avif","video/mp4","video/webm","application/json","text/plain","application/pdf"]},"filename":{"type":"string","maxLength":255},"width":{"type":"integer","exclusiveMinimum":0,"maximum":9007199254740991},"height":{"type":"integer","exclusiveMinimum":0,"maximum":9007199254740991}},"required":["sha256","size","content_type"]}}}},"responses":{"200":{"description":"File ID and, if needed, a URL to PUT the bytes to","content":{"application/json":{"schema":{"type":"object"}}}},"400":{"$ref":"#/components/responses/Invalid"},"401":{"$ref":"#/components/responses/Unauthorized"},"404":{"$ref":"#/components/responses/NotFound"}}}},"/files/{id}":{"get":{"summary":"File details and a download link","parameters":[{"name":"id","in":"path","required":true,"description":"File ID","schema":{"type":"string"}}],"responses":{"200":{"description":"The file","content":{"application/json":{"schema":{"type":"object"}}}},"400":{"$ref":"#/components/responses/Invalid"},"401":{"$ref":"#/components/responses/Unauthorized"},"404":{"$ref":"#/components/responses/NotFound"}}}},"/files/{id}/complete":{"post":{"summary":"Confirm an upload (checks size and SHA-256)","parameters":[{"name":"id","in":"path","required":true,"description":"File ID","schema":{"type":"string"}}],"responses":{"200":{"description":"The file is ready","content":{"application/json":{"schema":{"type":"object"}}}},"400":{"$ref":"#/components/responses/Invalid"},"401":{"$ref":"#/components/responses/Unauthorized"},"404":{"$ref":"#/components/responses/NotFound"},"409":{"description":"Not uploaded yet, or the content does not match the hash."}}}},"/suites":{"get":{"summary":"List suites","responses":{"200":{"description":"Suites with their latest version","content":{"application/json":{"schema":{"type":"object"}}}},"400":{"$ref":"#/components/responses/Invalid"},"401":{"$ref":"#/components/responses/Unauthorized"},"404":{"$ref":"#/components/responses/NotFound"}}}},"/suites/{slug}":{"get":{"summary":"Cases, labels and review settings of a suite version","parameters":[{"name":"slug","in":"path","required":true,"description":"Suite slug","schema":{"type":"string"}},{"name":"version","in":"query","required":false,"description":"Version number (default: latest)","schema":{"type":"string"}}],"responses":{"200":{"description":"The suite version","content":{"application/json":{"schema":{"type":"object"}}}},"400":{"$ref":"#/components/responses/Invalid"},"401":{"$ref":"#/components/responses/Unauthorized"},"404":{"$ref":"#/components/responses/NotFound"}}},"put":{"summary":"Create or update a suite from its full case list","parameters":[{"name":"slug","in":"path","required":true,"description":"Suite slug","schema":{"type":"string"}}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"name":{"type":"string","minLength":1,"maxLength":200},"description":{"type":"string","maxLength":5000},"review_config":{"anyOf":[{"type":"string","enum":["best","good_bad","score_1_5","video","prompt_harness"]},{"type":"object","properties":{"compare":{"default":{"enabled":true,"allowTie":true,"allowAllBad":true},"type":"object","properties":{"enabled":{"default":true,"type":"boolean"},"allowTie":{"default":true,"type":"boolean"},"allowAllBad":{"default":true,"type":"boolean"}}},"dimensions":{"default":[],"maxItems":12,"type":"array","items":{"oneOf":[{"type":"object","properties":{"key":{"type":"string","pattern":"^[a-z][a-z0-9_]{0,39}$"},"label":{"type":"string","minLength":1,"maxLength":60},"type":{"type":"string","const":"choice"},"options":{"minItems":2,"maxItems":8,"type":"array","items":{"type":"object","properties":{"value":{"type":"string","minLength":1,"maxLength":40},"label":{"type":"string","minLength":1,"maxLength":60},"tone":{"default":"neutral","type":"string","enum":["good","bad","neutral"]}},"required":["value","label"]}}},"required":["key","label","type","options"]},{"type":"object","properties":{"key":{"type":"string","pattern":"^[a-z][a-z0-9_]{0,39}$"},"label":{"type":"string","minLength":1,"maxLength":60},"type":{"type":"string","const":"scale"},"min":{"default":1,"type":"integer","minimum":0,"maximum":10},"max":{"default":5,"type":"integer","minimum":1,"maximum":10},"minLabel":{"type":"string","maxLength":40},"maxLabel":{"type":"string","maxLength":40}},"required":["key","label","type"]},{"type":"object","properties":{"key":{"type":"string","pattern":"^[a-z][a-z0-9_]{0,39}$"},"label":{"type":"string","minLength":1,"maxLength":60},"type":{"type":"string","const":"boolean"}},"required":["key","label","type"]},{"type":"object","properties":{"key":{"type":"string","pattern":"^[a-z][a-z0-9_]{0,39}$"},"label":{"type":"string","minLength":1,"maxLength":60},"type":{"type":"string","const":"tags"},"options":{"default":[],"maxItems":30,"type":"array","items":{"type":"string","minLength":1,"maxLength":40}}},"required":["key","label","type"]}]}},"notes":{"default":true,"type":"boolean"},"blind":{"default":true,"type":"boolean"}}}]},"note":{"type":"string","maxLength":2000},"cases":{"minItems":1,"maxItems":5000,"type":"array","items":{"type":"object","properties":{"key":{"type":"string","minLength":1,"maxLength":100},"title":{"type":"string","maxLength":300},"group":{"type":"string","maxLength":100},"instructions":{"type":"string","maxLength":20000},"expected":{"type":"string","maxLength":20000},"failure_conditions":{"type":"string","maxLength":20000},"inputs":{"default":[],"maxItems":20,"type":"array","items":{"type":"object","properties":{"file_id":{"type":"string"},"sha256":{"type":"string","pattern":"^[a-f0-9]{64}$"},"role":{"type":"string","pattern":"^[a-z][a-z0-9_-]{0,29}$"},"label":{"type":"string","maxLength":120}}}},"params":{"default":{},"type":"object","propertyNames":{"type":"string"},"additionalProperties":{}},"tags":{"default":[],"maxItems":30,"type":"array","items":{"type":"string","maxLength":60}},"meta":{"default":{},"type":"object","propertyNames":{"type":"string"},"additionalProperties":{}},"status":{"type":"string","enum":["active","irrelevant","needs_fixing"]},"status_note":{"type":"string","maxLength":2000}},"required":["key"]}}},"required":["cases"]}}}},"responses":{"200":{"description":"The version that holds these cases","content":{"application/json":{"schema":{"type":"object"}}}},"400":{"$ref":"#/components/responses/Invalid"},"401":{"$ref":"#/components/responses/Unauthorized"},"404":{"$ref":"#/components/responses/NotFound"}}}},"/runs":{"get":{"summary":"List runs","parameters":[{"name":"suite","in":"query","required":false,"description":"Suite slug","schema":{"type":"string"}},{"name":"batch","in":"query","required":false,"description":"Batch","schema":{"type":"string"}},{"name":"limit","in":"query","required":false,"description":"Maximum number of runs (default 100, max 500)","schema":{"type":"string"}}],"responses":{"200":{"description":"Runs, newest first","content":{"application/json":{"schema":{"type":"object"}}}},"400":{"$ref":"#/components/responses/Invalid"},"401":{"$ref":"#/components/responses/Unauthorized"},"404":{"$ref":"#/components/responses/NotFound"}}},"post":{"summary":"Create a run, or update it when external_id exists","requestBody":{"required":true,"content":{"application/json":{"schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"suite":{"type":"string","pattern":"^[a-z0-9][a-z0-9._-]{0,79}$"},"suite_version":{"type":"integer","exclusiveMinimum":0,"maximum":9007199254740991},"name":{"type":"string","minLength":1,"maxLength":200},"model":{"type":"string","maxLength":200},"batch":{"type":"string","maxLength":200},"is_baseline":{"type":"boolean"},"config":{"type":"object","propertyNames":{"type":"string"},"additionalProperties":{}},"external_id":{"type":"string","minLength":1,"maxLength":200},"notes":{"type":"string","maxLength":10000}},"required":["suite","name"]}}}},"responses":{"200":{"description":"The run, with created and url","content":{"application/json":{"schema":{"type":"object"}}}},"400":{"$ref":"#/components/responses/Invalid"},"401":{"$ref":"#/components/responses/Unauthorized"},"404":{"$ref":"#/components/responses/NotFound"}}}},"/runs/{id}":{"get":{"summary":"A run with its outputs","parameters":[{"name":"id","in":"path","required":true,"description":"Run ID or external_id","schema":{"type":"string"}}],"responses":{"200":{"description":"The run","content":{"application/json":{"schema":{"type":"object"}}}},"400":{"$ref":"#/components/responses/Invalid"},"401":{"$ref":"#/components/responses/Unauthorized"},"404":{"$ref":"#/components/responses/NotFound"}}}},"/runs/{id}/outputs":{"post":{"summary":"Add or replace up to 500 outputs","parameters":[{"name":"id","in":"path","required":true,"description":"Run ID or external_id","schema":{"type":"string"}}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"outputs":{"minItems":1,"maxItems":500,"type":"array","items":{"type":"object","properties":{"case":{"type":"string","minLength":1,"maxLength":100},"sample":{"default":0,"type":"integer","minimum":0,"maximum":99},"status":{"default":"ok","type":"string","enum":["ok","error"]},"error":{"type":"string","maxLength":10000},"cost_usd":{"type":"number","minimum":0},"latency_ms":{"type":"integer","minimum":0,"maximum":9007199254740991},"files":{"default":[],"maxItems":30,"type":"array","items":{"type":"object","properties":{"file_id":{"type":"string"},"sha256":{"type":"string","pattern":"^[a-f0-9]{64}$"},"role":{"type":"string","pattern":"^[a-z][a-z0-9_-]{0,29}$"},"label":{"type":"string","maxLength":120}}}},"text":{"type":"string","maxLength":200000},"scores":{"default":{},"type":"object","propertyNames":{"type":"string"},"additionalProperties":{"type":["number","string","boolean"]}},"meta":{"default":{},"type":"object","propertyNames":{"type":"string"},"additionalProperties":{}}},"required":["case"]}}},"required":["outputs"]}}}},"responses":{"200":{"description":"Number of saved outputs and their IDs","content":{"application/json":{"schema":{"type":"object"}}}},"400":{"$ref":"#/components/responses/Invalid"},"401":{"$ref":"#/components/responses/Unauthorized"},"404":{"$ref":"#/components/responses/NotFound"}}}},"/runs/{id}/complete":{"post":{"summary":"Mark a run complete or failed","parameters":[{"name":"id","in":"path","required":true,"description":"Run ID or external_id","schema":{"type":"string"}}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"status":{"default":"complete","type":"string","enum":["complete","failed"]},"notes":{"type":"string","maxLength":10000}}}}}},"responses":{"200":{"description":"The run","content":{"application/json":{"schema":{"type":"object"}}}},"400":{"$ref":"#/components/responses/Invalid"},"401":{"$ref":"#/components/responses/Unauthorized"},"404":{"$ref":"#/components/responses/NotFound"}}}},"/feedback":{"get":{"summary":"Verdicts, ratings, notes and case labels for a suite","parameters":[{"name":"suite","in":"query","required":true,"description":"Suite slug","schema":{"type":"string"}},{"name":"runs","in":"query","required":false,"description":"Run IDs or external IDs, comma-separated","schema":{"type":"string"}},{"name":"batch","in":"query","required":false,"description":"Batch","schema":{"type":"string"}},{"name":"since","in":"query","required":false,"description":"ISO date; only judgments updated since then","schema":{"type":"string"}}],"responses":{"200":{"description":"Feedback for the selected runs","content":{"application/json":{"schema":{"type":"object"}}}},"400":{"$ref":"#/components/responses/Invalid"},"401":{"$ref":"#/components/responses/Unauthorized"},"404":{"$ref":"#/components/responses/NotFound"}}}},"/judgments":{"post":{"summary":"Import judgments or record an automatic judge","requestBody":{"required":true,"content":{"application/json":{"schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"comparisons":{"default":[],"maxItems":1000,"type":"array","items":{"type":"object","properties":{"outputs":{"minItems":2,"maxItems":8,"type":"array","items":{"anyOf":[{"type":"string"},{"type":"object","properties":{"run":{"type":"string"},"case":{"type":"string"},"sample":{"default":0,"type":"integer","minimum":0,"maximum":9007199254740991}},"required":["run","case"]}]}},"verdict":{"type":"string","enum":["winner","tie","all_bad"]},"winner":{"anyOf":[{"type":"string"},{"type":"object","properties":{"run":{"type":"string"},"case":{"type":"string"},"sample":{"default":0,"type":"integer","minimum":0,"maximum":9007199254740991}},"required":["run","case"]}]},"note":{"type":"string","maxLength":10000},"source":{"type":"string","enum":["import","auto"]},"reviewer":{"type":"string","minLength":1,"maxLength":100,"pattern":"^[^:]+$"},"created_at":{"type":"string","format":"date-time","pattern":"^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$"}},"required":["outputs","verdict","source","reviewer"]}},"reviews":{"default":[],"maxItems":5000,"type":"array","items":{"type":"object","properties":{"output":{"anyOf":[{"type":"string"},{"type":"object","properties":{"run":{"type":"string"},"case":{"type":"string"},"sample":{"default":0,"type":"integer","minimum":0,"maximum":9007199254740991}},"required":["run","case"]}]},"values":{"default":{},"type":"object","propertyNames":{"type":"string"},"additionalProperties":{}},"note":{"type":"string","maxLength":10000},"source":{"type":"string","enum":["import","auto"]},"reviewer":{"type":"string","minLength":1,"maxLength":100,"pattern":"^[^:]+$"},"created_at":{"type":"string","format":"date-time","pattern":"^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$"}},"required":["output","source","reviewer"]}}}}}}},"responses":{"200":{"description":"Counts of saved comparisons and reviews","content":{"application/json":{"schema":{"type":"object"}}}},"400":{"$ref":"#/components/responses/Invalid"},"401":{"$ref":"#/components/responses/Unauthorized"},"404":{"$ref":"#/components/responses/NotFound"}}},"delete":{"summary":"Remove imported or automatic judgments","requestBody":{"required":true,"content":{"application/json":{"schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"comparisons":{"default":[],"maxItems":1000,"type":"array","items":{"type":"object","properties":{"outputs":{"minItems":2,"maxItems":8,"type":"array","items":{"anyOf":[{"type":"string"},{"type":"object","properties":{"run":{"type":"string"},"case":{"type":"string"},"sample":{"default":0,"type":"integer","minimum":0,"maximum":9007199254740991}},"required":["run","case"]}]}},"source":{"type":"string","enum":["import","auto"]},"reviewer":{"type":"string","minLength":1,"maxLength":100,"pattern":"^[^:]+$"}},"required":["outputs","source","reviewer"]}},"reviews":{"default":[],"maxItems":5000,"type":"array","items":{"type":"object","properties":{"output":{"anyOf":[{"type":"string"},{"type":"object","properties":{"run":{"type":"string"},"case":{"type":"string"},"sample":{"default":0,"type":"integer","minimum":0,"maximum":9007199254740991}},"required":["run","case"]}]},"source":{"type":"string","enum":["import","auto"]},"reviewer":{"type":"string","minLength":1,"maxLength":100,"pattern":"^[^:]+$"}},"required":["output","source","reviewer"]}}}}}}},"responses":{"200":{"description":"Counts of removed comparisons and reviews","content":{"application/json":{"schema":{"type":"object"}}}},"400":{"$ref":"#/components/responses/Invalid"},"401":{"$ref":"#/components/responses/Unauthorized"},"404":{"$ref":"#/components/responses/NotFound"}}}}}}