diff --git a/modules/judgement/internal/store/store.go b/modules/judgement/internal/store/store.go index abd9407d3..b4782291e 100644 --- a/modules/judgement/internal/store/store.go +++ b/modules/judgement/internal/store/store.go @@ -74,6 +74,20 @@ func (s *Store) AppendCall(ctx context.Context, orgID string, id uuid.UUID, call return nil } +// RecentBodies returns the newest decisions' bodies and when each was made. +// The judgement control page reads them to show the last standing a guard's +// model judge recorded. +func (s *Store) RecentBodies(ctx context.Context, orgID string, limit int) ([]Decision, error) { + var out []Decision + if err := s.db.SelectContext(ctx, &out, ` + SELECT id, org_id, run_id, key, connector, action, target_kind, environment, level, body, created_at + FROM judgement.decisions WHERE org_id = $1 + ORDER BY created_at DESC LIMIT $2`, orgID, limit); err != nil { + return nil, fmt.Errorf("list recent decisions: %w", err) + } + return out, nil +} + // ForRun returns a run's decisions in the order they were made. func (s *Store) ForRun(ctx context.Context, orgID string, runID uuid.UUID) ([]Decision, error) { var out []Decision diff --git a/modules/judgement/judgement.go b/modules/judgement/judgement.go index 5b089b5aa..d162ed8f1 100644 --- a/modules/judgement/judgement.go +++ b/modules/judgement/judgement.go @@ -245,7 +245,9 @@ type Module struct { rules []HardRule guards []Guard store recorder - failed FailureRecorder + // queries reads the decisions a guard's last judgement is found in. + queries *store.Store + failed FailureRecorder // changeRules decides whether approval goes through a change request. changeRules ChangeRules // extensions are the guard extensions asked alongside the built-in guards. @@ -267,7 +269,8 @@ func (m *Module) UseExtensions(e GuardExtensions) { m.extensions = e } // NewModule returns the judgement module for orgID, with its hard rules and // guards. An action of a class that fails is reported to failed. func NewModule(db *sqlx.DB, log *slog.Logger, orgID string, tools Tools, rules []HardRule, guards []Guard, failed FailureRecorder) *Module { - return &Module{orgID: orgID, tools: tools, rules: rules, guards: guards, store: store.New(db), failed: failed, log: log, now: time.Now} + s := store.New(db) + return &Module{orgID: orgID, tools: tools, rules: rules, guards: guards, store: s, queries: s, failed: failed, log: log, now: time.Now} } // Assess judges one action and records the decision. Judging the same key diff --git a/modules/judgement/judges.go b/modules/judgement/judges.go new file mode 100644 index 000000000..2729dc7df --- /dev/null +++ b/modules/judgement/judges.go @@ -0,0 +1,102 @@ +package judgement + +import ( + "context" + json "encoding/json/v2" + "net/http" + "time" + + "github.com/samcharles93/archie-core/internal/platform/httpjson" + "github.com/samcharles93/archie-core/modules/connectors" +) + +// judgeScanLimit bounds the decisions the control page reads to find a +// guard's last recorded standing. +const judgeScanLimit = 200 + +// JudgeControl is one running guard as the Controls page shows the judge it +// asks: what it was configured with, and the last standing a model judge +// recorded. +type JudgeControl struct { + Name string `json:"name"` + Version string `json:"version,omitempty"` + Digest string `json:"digest,omitempty"` + Advisory bool `json:"advisory"` + Settings map[string]string `json:"settings"` + SettingsSchema []connectors.Setting `json:"settings_schema"` + Latest *JudgeStanding `json:"latest,omitempty"` +} + +// JudgeStanding is the last model judgement a guard recorded: what the judge +// answered, whether its eval counts it, and the date that eval is from. +type JudgeStanding struct { + Model string `json:"model,omitempty"` + Trusted bool `json:"trusted"` + AdvisoryReason string `json:"advisory_reason,omitempty"` + EvalDate string `json:"eval_date,omitempty"` + Answer string `json:"answer,omitempty"` + At time.Time `json:"at"` +} + +// Judges returns each running guard with its judge configuration and the last +// model judgement recorded for it. +func (m *Module) Judges(ctx context.Context) ([]JudgeControl, error) { + if m.extensions == nil { + return []JudgeControl{}, nil + } + guards := m.extensions.Guards() + out := make([]JudgeControl, 0, len(guards)) + for _, g := range guards { + control := JudgeControl{ + Name: g.Name(), Version: g.Version(), Digest: g.Digest(), Advisory: g.Advisory(), + Settings: g.Settings(), SettingsSchema: g.SettingsSchema(), + } + latest, err := m.latestJudge(ctx, g.Name()) + if err != nil { + return nil, err + } + control.Latest = latest + out = append(out, control) + } + return out, nil +} + +// latestJudge finds the newest verdict a guard recorded with a model judge. +func (m *Module) latestJudge(ctx context.Context, guard string) (*JudgeStanding, error) { + recent, err := m.queries.RecentBodies(ctx, m.orgID, judgeScanLimit) + if err != nil { + return nil, err + } + for _, row := range recent { + var body Decision + if err := json.Unmarshal(row.Body, &body); err != nil { + continue + } + for _, v := range body.Verdicts { + if v.Guard != guard || len(v.Judges) == 0 { + continue + } + j := v.Judges[len(v.Judges)-1] + return &JudgeStanding{ + Model: j.Model, Trusted: j.Trusted, AdvisoryReason: j.AdvisoryReason, + EvalDate: j.EvalDate, Answer: j.Answer, At: row.CreatedAt, + }, nil + } + } + return nil, nil +} + +// Routes adds the judgement controls API to mux behind guard. +func (m *Module) Routes(mux *http.ServeMux, guard func(http.Handler) http.Handler) { + mux.Handle("GET /api/controls/judges", guard(http.HandlerFunc(m.listJudges))) +} + +func (m *Module) listJudges(w http.ResponseWriter, r *http.Request) { + out, err := m.Judges(r.Context()) + if err != nil { + m.log.ErrorContext(r.Context(), "list judges", "error", err) + httpjson.Error(w, http.StatusInternalServerError, "couldn't read the judges") + return + } + httpjson.Write(w, http.StatusOK, out) +}