Files
archipelago/sdk/systemone/systemone.go
T

477 lines
15 KiB
Go

// Package systemone speaks the System One classifier contract: a request carries
// JSON state and typed questions, the response carries typed answers.
//
// It is the same contract llama.cpp decision models serve at `/v1/systemone`, the
// one pi's provider catalogs use for hosted classifiers, and therefore one client
// covers both the local (`clef-flash` on a llama.cpp router) and hosted
// (`typesafe/jev-router` via OpenRouter) cases. Only the base URL and model id
// change.
//
// Wire shape, which is deliberately mirrored rather than invented:
//
// POST {base}/systemone
// {"model":"<id>","state":{...},"questions":{"<id>":{"type":"choice|score|noul",
// "instructions":"...","criteria":{...}|[...]}}}
//
// A boolean question is named "noul" on the wire.
package systemone
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"time"
)
// Question types. QuestionBool marshals as "noul", which is the wire name.
const (
QuestionChoice = "choice"
QuestionScore = "score"
QuestionBool = "bool"
)
// Stop reasons reported by the service.
const (
StopReasonStop = "stop"
StopReasonError = "error"
StopReasonAborted = "aborted"
)
// Question is one typed question about the state.
type Question struct {
Type string
Instructions string
// Criteria is a map[string]string for choice and bool questions, and a
// []string of levels (lowest first) for a score question.
Criteria any
}
// Choice asks for one label, with the meaning of each label spelled out.
func Choice(instructions string, criteria map[string]string) Question {
return Question{Type: QuestionChoice, Instructions: instructions, Criteria: criteria}
}
// Score asks for a level on an ordered scale, lowest first.
func Score(instructions string, criteria []string) Question {
return Question{Type: QuestionScore, Instructions: instructions, Criteria: criteria}
}
// Bool asks a yes/no question. The two criteria state what counts as each.
func Bool(instructions, whenTrue, whenFalse string) Question {
return Question{
Type: QuestionBool,
Instructions: instructions,
Criteria: map[string]string{"true": whenTrue, "false": whenFalse},
}
}
// wire renders the question in the request shape.
func (q Question) Wire() map[string]any {
name := q.Type
if name == QuestionBool {
name = "noul"
}
return map[string]any{"type": name, "instructions": q.Instructions, "criteria": q.Criteria}
}
// validate rejects a question the service could not answer meaningfully.
func (q Question) validate() error {
if strings.TrimSpace(q.Instructions) == "" {
return errors.New("instructions are required")
}
switch q.Type {
case QuestionChoice:
criteria, ok := q.Criteria.(map[string]string)
if !ok || len(criteria) < 2 {
return errors.New("a choice question needs at least two criteria")
}
case QuestionScore:
criteria, ok := q.Criteria.([]string)
if !ok || len(criteria) < 2 {
return errors.New("a score question needs at least two criteria levels")
}
case QuestionBool:
criteria, ok := q.Criteria.(map[string]string)
if !ok || criteria["true"] == "" || criteria["false"] == "" {
return errors.New("a bool question needs both criteria")
}
default:
return fmt.Errorf("unknown question type %q", q.Type)
}
return nil
}
// Answer is the response to one question. Only the fields belonging to the
// answer's type are set.
//
// A bool answer's probability is named "noul" on the wire, after the question
// type; Probability is accepted as an alias for deployments that name it that
// way.
type Answer struct {
Type string `json:"type"`
// Choice answers.
Choice string `json:"choice,omitempty"`
Probabilities map[string]float64 `json:"probabilities,omitempty"`
Confidence float64 `json:"confidence,omitempty"`
// Score answers: the expected level index, and its confidence.
Score *float64 `json:"score,omitempty"`
// Bool answers: the probability of true.
Noul *float64 `json:"noul,omitempty"`
Probability *float64 `json:"probability,omitempty"`
}
// BoolProbability returns a bool answer's probability of true, and reports
// whether this answer carries one.
func (a Answer) BoolProbability() (float64, bool) {
switch {
case a.Noul != nil:
return *a.Noul, true
case a.Probability != nil:
return *a.Probability, true
default:
return 0, false
}
}
// Label returns the chosen label for a choice answer, or "" for another type.
func (a Answer) Label() string {
return a.Choice
}
// IsTrue reports a bool answer's decision at the given probability threshold.
// It returns ok=false when the answer is not a bool answer.
func (a Answer) IsTrue(threshold float64) (value, ok bool) {
p, ok := a.BoolProbability()
if !ok {
return false, false
}
return p >= threshold, true
}
// Level returns a score answer's level, fine-grained value included.
func (a Answer) Level() (rounded int, exact float64, ok bool) {
if a.Score == nil {
return 0, 0, false
}
return int(*a.Score + 0.5), *a.Score, true
}
// Usage is token accounting, when the service reports it.
type Usage struct {
Input int
Output int
TotalTokens int
CostUSD float64
}
// Usage wire shapes differ between deployments, so parsing is lenient: this
// field is reported, never decided on. System One reports input_tokens,
// output_tokens and a flat cost.
func (u *Usage) UnmarshalJSON(data []byte) error {
var raw struct {
Input *int `json:"input"`
Output *int `json:"output"`
Total *int `json:"total"`
InputTokens *int `json:"input_tokens"`
OutputTokens *int `json:"output_tokens"`
PromptTokens *int `json:"prompt_tokens"`
CompletionTokens *int `json:"completion_tokens"`
TotalTokens *int `json:"total_tokens"`
Cost json.RawMessage `json:"cost"`
}
if err := json.Unmarshal(data, &raw); err != nil {
return err
}
pick := func(vals ...*int) int {
for _, v := range vals {
if v != nil {
return *v
}
}
return 0
}
u.Input = pick(raw.Input, raw.InputTokens, raw.PromptTokens)
u.Output = pick(raw.Output, raw.OutputTokens, raw.CompletionTokens)
u.TotalTokens = pick(raw.Total, raw.TotalTokens)
if sum := u.Input + u.Output; u.TotalTokens < sum {
u.TotalTokens = sum
}
if len(raw.Cost) > 0 {
var nested struct {
Total float64 `json:"total"`
}
if err := json.Unmarshal(raw.Cost, &nested); err == nil && nested.Total != 0 {
u.CostUSD = nested.Total
} else {
var flat float64
if err := json.Unmarshal(raw.Cost, &flat); err == nil {
u.CostUSD = flat
}
}
}
return nil
}
// Result is one classification response.
type Result struct {
ID string `json:"id,omitempty"`
Provider string `json:"provider,omitempty"`
Model string `json:"model,omitempty"`
Answers map[string]Answer `json:"answers"`
Usage *Usage `json:"usage,omitempty"`
StopReason string `json:"stopReason,omitempty"`
ErrorMessage string `json:"errorMessage,omitempty"`
// Request is the exact body sent to the endpoint, and Response the exact
// body it returned. Both are kept even when the call fails, so a failed
// judge call still records what was asked and whatever came back.
Request []byte `json:"-"`
Response []byte `json:"-"`
EndpointPath string `json:"-"`
HTTPStatus int `json:"-"`
// Latency is measured by this client, not reported by the service.
Latency time.Duration `json:"-"`
}
// AnswersJSON is every answer as JSON, for a record that keeps what the judge
// said whole: choices, probabilities and confidences.
func (r Result) AnswersJSON() json.RawMessage {
out, err := json.Marshal(r.Answers)
if err != nil {
return nil
}
return out
}
// Client is a System One endpoint.
type Client struct {
// BaseURL is the API root including its version segment. OpenRouter's System
// One API is https://openrouter.ai/api/v1, which resolves to
// https://openrouter.ai/api/v1/systemone.
BaseURL string
// APIKey is sent as a bearer token when set.
APIKey string
// Model is the classifier model id, for example typesafe/jev-router or
// clef-flash. Decision models are named, not pathed.
Model string
// Label names the deployment in reports, for example "openrouter".
Label string
// HTTP is optional; a 30s-timeout client is used when nil.
HTTP *http.Client
}
// BuildRequest renders the exact JSON body a classify call would send. It is
// exported so the audit log can store what the judge actually received, which is
// what makes a later fine-tune match production byte for byte.
func (c Client) BuildRequest(state any, questions map[string]any) ([]byte, error) {
body, err := json.Marshal(map[string]any{"model": c.Model, "state": state, "questions": questions})
if err != nil {
return nil, fmt.Errorf("systemone: encode request: %w", err)
}
return body, nil
}
// QuestionWires renders a question set for BuildRequest.
func QuestionWires(questions map[string]Question) map[string]any {
out := make(map[string]any, len(questions))
for id, q := range questions {
out[id] = q.Wire()
}
return out
}
// CheckAnswers validates a response against the questions that were asked, and is
// exported so a stored log entry can be checked for completeness on read.
func (r Result) CheckAnswers(questions map[string]Question) error {
return r.checkAnswers(questions)
}
// Endpoint resolves the classifier URL the way pi does: "systemone" is appended
// to the base URL, so .../v1 becomes .../v1/systemone.
func (c Client) Endpoint() string {
return strings.TrimRight(c.BaseURL, "/") + "/systemone"
}
// ModelID is the model this client asks.
func (c Client) ModelID() string { return c.Model }
// Deployment names the endpoint in reports.
func (c Client) Deployment() string { return c.Label }
// Classify asks every question about state in one request.
//
// State is any JSON value: the API accepts an object of named fields, and the
// TypeSafe SDKs also pass a bare string. A response that is not a 2xx, carries no
// answers, or is missing any requested question returns an error. Callers must
// treat that as "the classifier did not answer" and keep their own verdict: a
// judge that fails must not silently approve anything.
func (c Client) Classify(ctx context.Context, state any, questions map[string]Question) (Result, error) {
if c.BaseURL == "" {
return Result{}, errors.New("systemone: base URL is required")
}
if c.Model == "" {
return Result{}, errors.New("systemone: model is required")
}
if len(questions) == 0 {
return Result{}, errors.New("systemone: at least one question is required")
}
wire := make(map[string]any, len(questions))
for id, q := range questions {
if id == "" {
return Result{}, errors.New("systemone: question ids must not be empty")
}
if err := q.validate(); err != nil {
return Result{}, fmt.Errorf("systemone: question %q: %w", id, err)
}
wire[id] = q.Wire()
}
body, err := c.BuildRequest(state, wire)
if err != nil {
return Result{}, err
}
endpoint := c.Endpoint()
result := Result{Request: body, EndpointPath: endpointPath(endpoint), Model: c.Model, Provider: c.Label}
ctx, cancel := context.WithTimeout(ctx, 60*time.Second)
defer cancel()
req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(body))
if err != nil {
return result, fmt.Errorf("systemone: build request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
if c.APIKey != "" {
req.Header.Set("Authorization", "Bearer "+c.APIKey)
}
httpClient := c.HTTP
if httpClient == nil {
httpClient = &http.Client{Timeout: 30 * time.Second}
}
start := time.Now()
resp, err := httpClient.Do(req)
if err != nil {
result.Latency = time.Since(start)
return result, fmt.Errorf("systemone: %s: %w", c.Label, err)
}
defer func() { _ = resp.Body.Close() }()
result.Latency = time.Since(start)
result.HTTPStatus = resp.StatusCode
payload, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
result.Response = payload
if err != nil {
return result, fmt.Errorf("systemone: read response: %w", err)
}
if resp.StatusCode < 200 || resp.StatusCode > 299 {
return result, fmt.Errorf("systemone: %s returned HTTP %d%s: %s",
c.Label, resp.StatusCode, statusHint(resp.StatusCode), truncate(string(payload), 300))
}
if err := json.Unmarshal(payload, &result); err != nil {
return result, fmt.Errorf("systemone: decode response: %w", err)
}
if result.Provider == "" {
result.Provider = c.Label
}
if result.Model == "" {
result.Model = c.Model
}
if result.StopReason == StopReasonError || result.StopReason == StopReasonAborted {
return result, fmt.Errorf("systemone: %s reported stopReason %q: %s",
c.Label, result.StopReason, truncate(result.ErrorMessage, 200))
}
if err := result.checkAnswers(questions); err != nil {
return result, err
}
return result, nil
}
// WireType is the type name used on the wire, where a bool question and its
// answer are both called "noul". Exported so stored answers and training rows
// carry the documented name rather than the Go constant.
func WireType(name string) string {
if name == QuestionBool {
return "noul"
}
return name
}
// wireType is the internal alias kept for readability at the comparison site.
func wireType(name string) string { return WireType(name) }
// checkAnswers fails closed on an incomplete answer set, so a partial response is
// never read as an approval.
func (r Result) checkAnswers(questions map[string]Question) error {
if len(r.Answers) == 0 {
return fmt.Errorf("systemone: %s returned no answers", r.Provider)
}
for id, q := range questions {
answer, ok := r.Answers[id]
if !ok {
return fmt.Errorf("systemone: no answer for question %q", id)
}
if answer.Type != "" && answer.Type != wireType(q.Type) && answer.Type != q.Type {
return fmt.Errorf("systemone: answer for %q has type %q, want %q", id, answer.Type, wireType(q.Type))
}
switch q.Type {
case QuestionBool:
if _, ok := answer.BoolProbability(); !ok {
return fmt.Errorf("systemone: answer for %q has no noul probability", id)
}
case QuestionChoice:
if answer.Choice == "" {
return fmt.Errorf("systemone: answer for %q has no choice", id)
}
case QuestionScore:
if answer.Score == nil {
return fmt.Errorf("systemone: answer for %q has no score", id)
}
}
}
return nil
}
// statusHint names the thing an operator has to fix, for the statuses that
// actually happen in practice.
func statusHint(status int) string {
switch status {
case 401, 403:
return " (the API key was rejected; OpenRouter reports this as \"User not found\")"
case 402:
return " (the account has insufficient credit)"
case 413:
return " (the state is too large)"
case 429:
return " (rate limited)"
case 524, 529, 502, 503:
return " (provider overloaded, timed out, or unavailable)"
default:
return ""
}
}
// endpointPath is the request path without its host, which is what a stored
// record keeps.
func endpointPath(endpoint string) string {
u, err := url.Parse(endpoint)
if err != nil || u.Path == "" {
return endpoint
}
return u.Path
}
func truncate(s string, n int) string {
s = strings.TrimSpace(s)
if len(s) <= n {
return s
}
return s[:n] + "..."
}