From 470015483e93e238ad01c6ca505fee7109526650 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Mon, 16 Nov 2026 10:04:44 +1100 Subject: [PATCH] feat(judgement): a dry decision is recorded apart, counts toward nothing and never executes a changing action, and a draft's policy can overlay the set in force (#2008) --- modules/judgement/extensions.go | 2 + modules/judgement/internal/store/classes.go | 4 +- modules/judgement/internal/store/store.go | 9 +++-- modules/judgement/judgement.go | 27 ++++++++++--- modules/judgement/judgement_test.go | 10 ++++- modules/judgement/policies.go | 45 ++++++++++++++++++++- 6 files changed, 84 insertions(+), 13 deletions(-) diff --git a/modules/judgement/extensions.go b/modules/judgement/extensions.go index de2392e79..1bc08d5f3 100644 --- a/modules/judgement/extensions.go +++ b/modules/judgement/extensions.go @@ -29,6 +29,8 @@ type RunContext struct { // before proposing the action. Diagnosis string Evidence []Evidence + // Overlay is policy text judged as if published, for a dry run. + Overlay *PolicyOverlay } // Evidence is one tool result the agent read, as the tool returned it. diff --git a/modules/judgement/internal/store/classes.go b/modules/judgement/internal/store/classes.go index c04771b9a..13cf519fa 100644 --- a/modules/judgement/internal/store/classes.go +++ b/modules/judgement/internal/store/classes.go @@ -77,7 +77,7 @@ func (s *Store) Classes(ctx context.Context, orgID string) ([]ClassSummary, erro (SELECT count(*) FROM judgement.class_approvals a WHERE a.class_id = c.id AND a.counted) AS streak FROM judgement.classes c LEFT JOIN judgement.decisions d ON d.org_id = c.org_id AND d.connector = c.connector - AND d.action = c.action AND d.target_kind = c.target_kind AND d.environment = c.environment + AND d.action = c.action AND d.target_kind = c.target_kind AND d.environment = c.environment AND NOT d.dry WHERE c.org_id = $1 GROUP BY c.id ORDER BY coalesce(max(d.created_at), c.created_at) DESC, c.id`, orgID) @@ -106,7 +106,7 @@ func (s *Store) ClassDecisions(ctx context.Context, c Class, limit int) ([]Decis err := s.db.SelectContext(ctx, &out, ` SELECT id, org_id, run_id, key, connector, action, target_kind, environment, level, body, created_at FROM judgement.decisions - WHERE org_id = $1 AND connector = $2 AND action = $3 AND target_kind = $4 AND environment = $5 + WHERE org_id = $1 AND connector = $2 AND action = $3 AND target_kind = $4 AND environment = $5 AND NOT dry ORDER BY created_at DESC, id LIMIT $6`, c.OrgID, c.Connector, c.Action, c.TargetKind, c.Environment, limit) if err != nil { diff --git a/modules/judgement/internal/store/store.go b/modules/judgement/internal/store/store.go index d868690ed..c6468170a 100644 --- a/modules/judgement/internal/store/store.go +++ b/modules/judgement/internal/store/store.go @@ -24,6 +24,7 @@ type Decision struct { TargetKind string `db:"target_kind"` Environment string `db:"environment"` Level string `db:"level"` + Dry bool `db:"dry"` Body []byte `db:"body"` // CreatedAt is set by the database and read only by the class history. CreatedAt time.Time `db:"created_at"` @@ -43,11 +44,11 @@ func (s *Store) Record(ctx context.Context, d Decision) (Decision, error) { var kept Decision err := s.db.GetContext(ctx, &kept, ` INSERT INTO judgement.decisions - (id, org_id, run_id, key, connector, action, target_kind, environment, level, body) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + (id, org_id, run_id, key, connector, action, target_kind, environment, level, dry, body) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) ON CONFLICT (org_id, key) DO NOTHING RETURNING id, org_id, run_id, key, connector, action, target_kind, environment, level, body`, - d.ID, d.OrgID, d.RunID, d.Key, d.Connector, d.Action, d.TargetKind, d.Environment, d.Level, d.Body) + d.ID, d.OrgID, d.RunID, d.Key, d.Connector, d.Action, d.TargetKind, d.Environment, d.Level, d.Dry, d.Body) if errors.Is(err, sql.ErrNoRows) { err = s.db.GetContext(ctx, &kept, ` SELECT id, org_id, run_id, key, connector, action, target_kind, environment, level, body @@ -83,7 +84,7 @@ func (s *Store) CountActions(ctx context.Context, orgID string, runID uuid.UUID, SELECT count(*) FILTER (WHERE run_id = $2) AS in_run, count(*) FILTER (WHERE created_at >= $4) AS in_day FROM judgement.decisions - WHERE org_id = $1 AND key <> $3 AND level <> 'never' + WHERE org_id = $1 AND key <> $3 AND level <> 'never' AND NOT dry AND COALESCE((body->'action'->'effect'->>'ReadOnly')::boolean, false) = false`, orgID, runID, key, since) if err != nil { return 0, 0, fmt.Errorf("count actions: %w", err) diff --git a/modules/judgement/judgement.go b/modules/judgement/judgement.go index de481805a..8e3d049d5 100644 --- a/modules/judgement/judgement.go +++ b/modules/judgement/judgement.go @@ -32,6 +32,9 @@ var ( ErrHeld = errors.New("held for approval") // ErrAltered means the action to run isn't the one that was judged. ErrAltered = errors.New("action differs from the one judged") + // ErrDryRun means the decision was made for a dry run and the action + // changes something, so it is never executed. + ErrDryRun = errors.New("dry run: not executed") // ErrUnapproved means the action needs an approval it doesn't have, or the // approval offered was given for a different action. ErrUnapproved = errors.New("not approved") @@ -121,6 +124,9 @@ type Decision struct { ChangeRequestRule string `json:"change_request_rule,omitzero"` Reason string `json:"reason,omitzero"` At time.Time `json:"at"` + // Dry marks a decision made for a dry run. It is recorded for the run's + // trail and counts toward nothing: not spend, not a class's history. + Dry bool `json:"dry,omitzero"` } // Owner is the id of the person who answers for the decision's target, or empty @@ -150,6 +156,12 @@ type Request struct { // for guards that weigh the reasoning behind an action. Both may be empty. Diagnosis string Evidence []Evidence + // Dry judges the action for a dry run: the decision is recorded apart from + // the history autonomy and budgets count, and it is never executed unless + // the action only reads. + Dry bool + // Overlay is policy text a dry run judges under as if it were published. + Overlay *PolicyOverlay } // Tools finds the tool that resolves and runs an action. @@ -215,8 +227,8 @@ func (m *Module) Assess(ctx context.Context, req Request) (Decision, error) { Connector: req.Connector, Name: req.Action, Input: req.Input, Digest: connectors.Digest(req.Action, req.Input), Target: res.Target, Effect: res.Effect, } - d := m.decide(withRun(ctx, RunContext{RunID: req.RunID, Key: req.Key, Workflow: req.Workflow, Diagnosis: req.Diagnosis, Evidence: req.Evidence}), a) - d.ID, d.RunID, d.Key, d.At = uuid.New(), req.RunID, req.Key, m.now().UTC() + d := m.decide(withRun(ctx, RunContext{RunID: req.RunID, Key: req.Key, Workflow: req.Workflow, Diagnosis: req.Diagnosis, Evidence: req.Evidence, Overlay: req.Overlay}), a) + d.ID, d.RunID, d.Key, d.At, d.Dry = uuid.New(), req.RunID, req.Key, m.now().UTC(), req.Dry return m.record(ctx, d) } @@ -306,6 +318,8 @@ func (m *Module) Execute(ctx context.Context, d Decision, input jsontext.Value, return "", fmt.Errorf("%w at %s", ErrHeld, d.Level) case connectors.Digest(d.Action.Name, input) != d.Action.Digest: return "", ErrAltered + case d.Dry && !d.Action.Effect.ReadOnly: + return "", ErrDryRun } return m.invoke(ctx, d, input, key) } @@ -318,8 +332,9 @@ func (m *Module) invoke(ctx context.Context, d Decision, input jsontext.Value, k return "", fmt.Errorf("%q isn't running", d.Action.Connector) } out, err := tool.Invoke(ctx, connectors.Invocation{Action: d.Action.Name, Input: input, IdempotencyKey: key}) - // A shutdown cancels the call without the action having gone wrong. - if err != nil && m.failed != nil && !errors.Is(err, context.Canceled) { + // A shutdown cancels the call without the action having gone wrong, and a + // dry run's failure is not evidence about the class. + if err != nil && m.failed != nil && !d.Dry && !errors.Is(err, context.Canceled) { if ferr := m.failed.Failed(ctx, d.Action.Class()); ferr != nil { return out, errors.Join(err, fmt.Errorf("tighten class: %w", ferr)) } @@ -353,6 +368,8 @@ func (m *Module) ExecuteApproved(ctx context.Context, d Decision, a Approval, in return "", fmt.Errorf("%w: the approval names another action", ErrUnapproved) case connectors.Digest(d.Action.Name, input) != d.Action.Digest: return "", ErrAltered + case d.Dry && !d.Action.Effect.ReadOnly: + return "", ErrDryRun } return m.invoke(ctx, d, input, key) } @@ -383,7 +400,7 @@ func (m *Module) record(ctx context.Context, d Decision) (Decision, error) { row, err := m.store.Record(ctx, store.Decision{ ID: d.ID, OrgID: m.orgID, RunID: d.RunID, Key: d.Key, Connector: c.Connector, Action: c.Action, TargetKind: c.TargetKind, Environment: c.Environment, - Level: d.Level.String(), Body: body, + Level: d.Level.String(), Dry: d.Dry, Body: body, }) if err != nil { return Decision{}, err diff --git a/modules/judgement/judgement_test.go b/modules/judgement/judgement_test.go index 51aaa344a..1c7ee74d5 100644 --- a/modules/judgement/judgement_test.go +++ b/modules/judgement/judgement_test.go @@ -147,12 +147,17 @@ func (o oneTool) Tool(string) (connectors.Tool, bool) { return o.tool, true } // TestExecuteRunsOnlyTheJudgedAction guards spec section 5 rule 6 and // decision tree 8.5: an action runs only at a level that runs without // approval, and only with exactly the input that was judged. A refused, held -// or altered action never reaches the connector. +// or altered action never reaches the connector, and neither does a changing +// action judged for a dry run, whatever level it reached. func TestExecuteRunsOnlyTheJudgedAction(t *testing.T) { judged := jsontext.Value(`{"path":"/tmp/runner"}`) decided := func(l Level) Decision { return Decision{Level: l, Action: Action{Connector: "c", Name: "delete", Input: judged, Digest: connectors.Digest("delete", judged)}} } + dry := func(d Decision, readOnly bool) Decision { + d.Dry, d.Action.Effect.ReadOnly = true, readOnly + return d + } tests := []struct { name string d Decision @@ -165,6 +170,9 @@ func TestExecuteRunsOnlyTheJudgedAction(t *testing.T) { {name: "escalate is held", d: decided(Escalate), input: judged, wantErr: ErrHeld}, {name: "never is refused", d: decided(Never), input: judged, wantErr: ErrRefused}, {name: "altered input is refused", d: decided(ActQuietly), input: jsontext.Value(`{"path":"/"}`), wantErr: ErrAltered}, + {name: "a dry run's changing action is never executed at act quietly", d: dry(decided(ActQuietly), false), input: judged, wantErr: ErrDryRun}, + {name: "a dry run's changing action is never executed at act and tell", d: dry(decided(ActAndTell), false), input: judged, wantErr: ErrDryRun}, + {name: "a dry run's read-only action runs for real", d: dry(decided(ActQuietly), true), input: judged}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/modules/judgement/policies.go b/modules/judgement/policies.go index 793c77c64..5932d9554 100644 --- a/modules/judgement/policies.go +++ b/modules/judgement/policies.go @@ -12,6 +12,7 @@ import ( "math" "net/http" "strconv" + "strings" "time" cedar "github.com/cedar-policy/cedar-go" @@ -166,12 +167,54 @@ func (p *Policies) Requires(ctx context.Context, a Action, level Level, verdicts return "", false } +// PolicyOverlay is policy text of one kind added to the set in force, for a +// dry run judging a draft before it is published. +type PolicyOverlay struct { + Kind string `json:"kind"` + Body string `json:"body"` +} + func (p *Policies) compiled(ctx context.Context, kind string) (*cedar.PolicySet, error) { row, err := p.current(ctx, kind) if err != nil { return nil, err } - return parse(kind, row.Body) + body := row.Body + if o := runFrom(ctx).Overlay; o != nil && o.Kind == kind { + body = joined(body, o.Body) + } + return parse(kind, body) +} + +func joined(current, extra string) string { return current + "\n\n" + extra } + +// Check reports whether extra, added to the set of kind in force, is a valid +// policy set: it compiles, fits the kind and reuses no policy id. +func (p *Policies) Check(ctx context.Context, kind, extra string) error { + row, err := p.current(ctx, kind) + if err != nil { + return err + } + if strings.Contains(row.Body, strings.TrimSpace(extra)) { + return nil + } + if _, err := parse(kind, joined(row.Body, extra)); err != nil { + return fmt.Errorf("%w: %w", ErrInvalidPolicy, err) + } + return nil +} + +// Append stores the set in force with extra added as the next version. Text +// the set already contains is not added again, so a retried publish is safe. +func (p *Policies) Append(ctx context.Context, kind, extra, by string) (PolicySetView, error) { + row, err := p.current(ctx, kind) + if err != nil { + return PolicySetView{}, err + } + if strings.Contains(row.Body, strings.TrimSpace(extra)) { + return p.View(ctx, kind, 0) + } + return p.Put(ctx, kind, joined(row.Body, extra), by) } func entities(ctx context.Context, a Action) types.EntityMap {