feat(approvals,runs): an unanswered approval escalates step by step and then expires, a discarded approval never counts, and a change-request seam routes and withdraws (#1985)

This commit is contained in:
sam committed 2026-11-16 09:42:23 +11:00
1 parent 4b1ae4babf
commit 28733c8522
7 files changed
+689 -160

No files matched your search

+136 -45
View File
@@ -10,6 +10,7 @@ package approvals
import (
"context"
"database/sql"
"encoding/json/jsontext"
json "encoding/json/v2"
"errors"
@@ -31,12 +32,14 @@ import (
"github.com/samcharles93/archie-core/modules/judgement"
)
// The states a request can be in. A settled request never changes again.
// The states a request can be in. A settled request never changes again,
// except that an approval the action no longer matched is discarded.
const (
StatePending = "pending"
StateApproved = "approved"
StateDenied = "denied"
StateExpired = "expired"
StatePending = "pending"
StateApproved = "approved"
StateDenied = "denied"
StateExpired = "expired"
StateDiscarded = "discarded"
)
var (
@@ -50,6 +53,12 @@ var (
ErrUnverified = errors.New("that identity isn't held by any person, so its answer is refused")
// ErrNotApprover means the person may not answer approvals for actions.
ErrNotApprover = errors.New("that person isn't the default owner, on the escalation path or an authority, so their answer is refused")
// ErrNotAsked means the request hasn't yet reached that person on its
// escalation path.
ErrNotAsked = errors.New("that person hasn't been asked yet, so their answer is refused")
// ErrNeedsAuthority means the action needs a person with authority, and
// the answer came from one without.
ErrNeedsAuthority = errors.New("that action needs a person with authority, so their answer is refused")
// ErrNoAuthority means the answer to a raise came from a person without
// authority over policy and ceilings.
ErrNoAuthority = errors.New("that person holds no authority over policy and ceilings, so their answer to a raise is refused")
@@ -79,10 +88,19 @@ type Request struct {
Level string
Summary string
State string
Deadline time.Time
AnsweredAt time.Time
AnsweredBy string
AnsweredVia string
// Deadline is when the request runs out for good: the end of the last
// escalation step.
Deadline time.Time
// Step is the escalation step the request is at, and StepDeadline when it
// moves on.
Step int
StepDeadline time.Time
// ChangeRequest is whether the request went through a change request
// rather than the approvers' channels.
ChangeRequest bool
AnsweredAt time.Time
AnsweredBy string
AnsweredVia string
}
// Answer is what the run wakes with.
@@ -110,6 +128,10 @@ type View struct {
Summary string `json:"summary"`
State string `json:"state"`
Deadline time.Time `json:"deadline"`
// Step is the escalation step the request is at, and StepDeadline when it
// moves to the next.
Step int `json:"step"`
StepDeadline time.Time `json:"step_deadline,omitzero"`
// DecisionID is the judged action the request is about.
DecisionID uuid.UUID `json:"decision_id"`
RequestedAt time.Time `json:"requested_at"`
@@ -123,7 +145,7 @@ func (r Request) View() View {
return View{
ID: r.ID, Kind: r.Kind, RunID: r.RunID, Action: r.Connector + " " + r.Action,
Target: r.TargetKind + ":" + r.TargetID, Level: r.Level,
Summary: r.Summary, State: r.State, Deadline: r.Deadline,
Summary: r.Summary, State: r.State, Deadline: r.Deadline, Step: r.Step, StepDeadline: r.StepDeadline,
DecisionID: r.DecisionID, RequestedAt: r.RequestedAt,
AnsweredBy: r.AnsweredBy, AnsweredVia: r.AnsweredVia, AnsweredAt: r.AnsweredAt,
}
@@ -172,6 +194,7 @@ type Module struct {
classes Classes
policy Policy
people People
changes ChangeRequests
log *slog.Logger
now func() time.Time
}
@@ -183,20 +206,44 @@ type Module struct {
func NewModule(db *sqlx.DB, log *slog.Logger, orgID string, tell Delivery, wake Signaller, classes Classes, policy Policy, people People) *Module {
return &Module{
orgID: orgID, store: store.New(db), tell: tell, wake: wake, classes: classes, policy: policy, people: people,
log: log, now: time.Now,
changes: NoChangeRequests{}, log: log, now: time.Now,
}
}
// Ask records a request for d and tells people about it. Asking again for the
// same run and key returns the request that stands, so a replayed step asks
// once.
func (m *Module) Ask(ctx context.Context, runID uuid.UUID, key string, d judgement.Decision, deadline time.Time) (Request, error) {
// Ask records a request for d and puts it to the first step of path. It runs
// out for good when the last step does. Asking again for the same run and key
// returns the request that stands, so a replayed step asks once. An action the
// change-request rules pick goes through a change request, and every other one
// through the approvers' channels.
func (m *Module) Ask(ctx context.Context, runID uuid.UUID, key string, d judgement.Decision, path []governance.PathStep) (Request, error) {
if len(path) == 0 {
return Request{}, ErrNoPath
}
changeRequest, err := m.changes.Needed(ctx, d)
if err != nil {
return Request{}, fmt.Errorf("route approval: %w", err)
}
now := m.now().UTC()
var total time.Duration
for _, st := range path {
total += st.Deadline()
}
asked := []string{}
if path[0].Person != "" {
asked = append(asked, path[0].Person)
}
askedJSON, err := json.Marshal(asked)
if err != nil {
return Request{}, fmt.Errorf("encode asked: %w", err)
}
row, err := m.store.Put(ctx, store.Request{
ID: uuid.New(), OrgID: m.orgID, RunID: runID, Key: key,
DecisionID: d.ID, Kind: KindAction, Digest: d.Action.Digest, Level: d.Level.String(),
Connector: d.Action.Connector, Action: d.Action.Name,
TargetKind: d.Action.Target.Kind, TargetID: d.Action.Target.ID,
Summary: Summary(d), State: StatePending, Deadline: deadline.UTC(),
Summary: Summary(d), State: StatePending, Deadline: now.Add(total),
StepDeadline: sql.NullTime{Time: now.Add(path[0].Deadline()), Valid: true}, Asked: string(askedJSON),
ChangeRequest: changeRequest,
})
if err != nil {
return Request{}, err
@@ -205,10 +252,16 @@ func (m *Module) Ask(ctx context.Context, runID uuid.UUID, key string, d judgeme
if row.State != StatePending {
return r, nil
}
if row.ChangeRequest {
if err := m.changes.Open(ctx, r); err != nil {
return Request{}, fmt.Errorf("open change request: %w", err)
}
return r, nil
}
if err := m.tell.Ask(ctx, deliveries.ApprovalRequest{
RunID: r.RunID, RequestID: r.ID, Key: r.Key + ":ask",
Title: "Approval needed: " + d.Action.Connector + " " + d.Action.Name,
Body: r.Summary, ExpiresAt: r.Deadline,
Body: r.Summary, ExpiresAt: r.StepDeadline,
}); err != nil {
return Request{}, fmt.Errorf("send approval request: %w", err)
}
@@ -244,20 +297,20 @@ func (m *Module) answerAs(ctx context.Context, id uuid.UUID, by, via string, app
if err := m.wakeWith(ctx, row, answer); err != nil {
return Request{}, err
}
// The run has what it waited for; the answer's bearing on the class comes
// after, so a failure here is reported without holding the run.
if err := m.learn(ctx, row, answer); err != nil {
return Request{}, err
// The run has what it waited for. A denial tightens the class now. An
// approval counts only once the action is about to run on it, so one that
// is discarded first counts for nothing.
if !answer.Approved {
if err := m.classes.Denied(ctx, row.DecisionID); err != nil {
return Request{}, err
}
}
return requestOf(row, judgement.Decision{}), nil
}
// learn counts an answer to an action into its class. An approval that
// learn counts an approval of an action into its class. An approval that
// completes a run of them earns a raise, which is put to a person.
func (m *Module) learn(ctx context.Context, row store.Request, a Answer) error {
if !a.Approved {
return m.classes.Denied(ctx, row.DecisionID)
}
need, err := m.policy.RaiseAfter(ctx)
if err != nil {
return fmt.Errorf("read raise count: %w", err)
@@ -342,7 +395,7 @@ func (m *Module) Settle(ctx context.Context, id uuid.UUID, identity, via string,
if err != nil {
return false, err
}
who, err := m.verify(ctx, identity, row.Kind)
who, err := m.verify(ctx, identity, row)
if refusal(err) {
m.log.WarnContext(ctx, "approval answer refused", "request", id, "identity", identity, "reason", err.Error())
return false, m.store.Refuse(ctx, store.Refusal{
@@ -362,31 +415,57 @@ func (m *Module) Settle(ctx context.Context, id uuid.UUID, identity, via string,
return false, err
}
// verify says whether identity may answer a request of kind. An action is
// answered by anyone who answers for actions; a raise changes policy, so only
// a person with authority answers it.
func (m *Module) verify(ctx context.Context, identity, kind string) (governance.Verified, error) {
// verify says whether identity may answer request row. An action is answered
// by someone who answers for actions and whom the request has reached on its
// escalation path, or who holds authority, and an action at the escalate level
// needs authority. A raise changes policy, so only a person with authority
// answers it.
func (m *Module) verify(ctx context.Context, identity string, row store.Request) (governance.Verified, error) {
who, ok, err := m.people.Verify(ctx, identity)
switch {
case err != nil:
if err != nil {
return governance.Verified{}, fmt.Errorf("verify %s: %w", identity, err)
case !ok:
return governance.Verified{}, ErrUnverified
case kind == KindRaise && !who.Authority:
return governance.Verified{}, ErrNoAuthority
case kind != KindRaise && !who.Approver:
return governance.Verified{}, ErrNotApprover
}
return who, nil
if !ok {
return governance.Verified{}, ErrUnverified
}
return who, mayAnswer(who, row)
}
func mayAnswer(who governance.Verified, row store.Request) error {
switch {
case row.Kind == KindRaise && !who.Authority:
return ErrNoAuthority
case row.Kind == KindRaise:
return nil
case !who.Approver:
return ErrNotApprover
case row.Level == judgement.Escalate.String() && !who.Authority:
return ErrNeedsAuthority
case !who.Authority && !slices.Contains(askedOf(row), who.ID):
return ErrNotAsked
}
return nil
}
// askedOf is the ids of the people the request has reached.
func askedOf(row store.Request) []string {
var ids []string
if err := json.Unmarshal([]byte(row.Asked), &ids); err != nil {
return nil
}
return ids
}
// refusal reports whether err is a refused answer rather than a failure.
func refusal(err error) bool {
return errors.Is(err, ErrUnverified) || errors.Is(err, ErrNotApprover) || errors.Is(err, ErrNoAuthority)
return errors.Is(err, ErrUnverified) || errors.Is(err, ErrNotApprover) || errors.Is(err, ErrNoAuthority) ||
errors.Is(err, ErrNotAsked) || errors.Is(err, ErrNeedsAuthority)
}
// Expire settles a request whose deadline passed, so the trail shows why it
// never ran.
// Expire settles a request whose last escalation step ran out, so the trail
// shows why it never ran. Nothing is acted on, an open change request is
// withdrawn, and the report goes to every channel. Expiry says nothing about
// the action's class, so it is not tightened.
func (m *Module) Expire(ctx context.Context, id uuid.UUID) error {
row, err := m.store.Answer(ctx, m.orgID, id, StateExpired, "deadline", "clock", m.now().UTC())
if err != nil {
@@ -399,18 +478,26 @@ func (m *Module) Expire(ctx context.Context, id uuid.UUID) error {
return nil
}
m.log.InfoContext(ctx, "approval request expired", "request", row.ID, "run", row.RunID)
withdrawn := ""
if row.ChangeRequest {
if err := m.changes.Withdraw(ctx, requestOf(row, judgement.Decision{})); err != nil {
return fmt.Errorf("withdraw change request: %w", err)
}
withdrawn = " Its change request was withdrawn."
}
if err := m.tell.Told(ctx, deliveries.ApprovalRequest{
RunID: row.RunID, RequestID: row.ID, Key: row.Key + ":ask",
Title: "Approval expired", Body: "The request ran out of time, so nothing was run.",
Title: "Approval expired", Body: "The request ran out of time, so nothing was run." + withdrawn,
}, "Expired without an answer."); err != nil {
return err
}
// The escalation goes to every channel. Its key is stable, so an Expire
// The report goes to every channel. Its key is stable, so an Expire
// retried after a failed delivery sends it once.
return m.tell.Notify(ctx, deliveries.Notification{
RunID: row.RunID, Key: row.Key + ":expired",
Title: "Approval timed out: " + row.Connector + " " + row.Action,
Body: "Nobody answered before the deadline, so " + row.Connector + " " + row.Action + " did not run. " + row.Summary,
Body: "Nobody answered before the last escalation step ran out, so " + row.Connector + " " + row.Action +
" did not run." + withdrawn + " " + row.Summary,
})
}
@@ -619,6 +706,10 @@ func requestOf(row store.Request, d judgement.Decision) Request {
ID: row.ID, Kind: row.Kind, RunID: row.RunID, Key: row.Key, Decision: d, DecisionID: row.DecisionID, RequestedAt: row.CreatedAt,
Connector: row.Connector, Action: row.Action, TargetKind: row.TargetKind, TargetID: row.TargetID,
Level: row.Level, Summary: row.Summary, State: row.State, Deadline: row.Deadline,
Step: row.Step, ChangeRequest: row.ChangeRequest,
}
if row.StepDeadline.Valid {
r.StepDeadline = row.StepDeadline.Time
}
if row.AnsweredAt.Valid {
r.AnsweredAt = row.AnsweredAt.Time
+26 -13
View File
@@ -3,11 +3,13 @@ package approvals
import (
"context"
"errors"
"strings"
"testing"
"time"
"github.com/samcharles93/archie-core/modules/approvals/internal/store"
"github.com/samcharles93/archie-core/modules/governance"
"github.com/samcharles93/archie-core/modules/judgement"
)
type fakePeople map[string]governance.Verified
@@ -19,31 +21,42 @@ func (f fakePeople) Verify(_ context.Context, identity string) (governance.Verif
// TestVerifyAcceptsAnAnswerOnlyFromAPersonWhoMayGiveIt guards the rule that an
// approval counts only from a verified identity, on every channel: an identity
// no person holds, or a person without the standing for that kind of request,
// is refused rather than settling it.
// no person holds, a person without the standing for that kind of request, a
// person the request hasn't reached on its escalation path, or one without the
// authority the action's level needs, is refused rather than settling it.
func TestVerifyAcceptsAnAnswerOnlyFromAPersonWhoMayGiveIt(t *testing.T) {
people := fakePeople{
"telegram:1001": {Name: "Owner", Approver: true},
"telegram:1002": {Name: "Authority", Approver: true, Authority: true},
"telegram:1003": {Name: "Bystander"},
"telegram:1001": {ID: "owner", Name: "Owner", Approver: true},
"telegram:1002": {ID: "lead", Name: "Authority", Approver: true, Authority: true},
"telegram:1003": {ID: "bystander", Name: "Bystander"},
"telegram:1004": {ID: "second", Name: "Second", Approver: true},
}
action := func(level string, asked ...string) store.Request {
return store.Request{Kind: KindAction, Level: level, Asked: `["` + strings.Join(asked, `","`) + `"]`}
}
propose, escalate := judgement.ProposeAndWait.String(), judgement.Escalate.String()
tests := []struct {
name string
identity string
kind string
row store.Request
wantErr error
}{
{name: "an approver answers an action", identity: "telegram:1001", kind: KindAction},
{name: "an unmapped identity is refused", identity: "telegram:9999", kind: KindAction, wantErr: ErrUnverified},
{name: "an unmapped identity can't answer a raise", identity: "telegram:9999", kind: KindRaise, wantErr: ErrUnverified},
{name: "a person who isn't an approver is refused an action", identity: "telegram:1003", kind: KindAction, wantErr: ErrNotApprover},
{name: "an approver without authority is refused a raise", identity: "telegram:1001", kind: KindRaise, wantErr: ErrNoAuthority},
{name: "authority answers a raise", identity: "telegram:1002", kind: KindRaise},
{name: "an approver the request has reached answers", identity: "telegram:1001", row: action(propose, "owner")},
{name: "an unmapped identity is refused", identity: "telegram:9999", row: action(propose, "owner"), wantErr: ErrUnverified},
{name: "an unmapped identity can't answer a raise", identity: "telegram:9999", row: store.Request{Kind: KindRaise}, wantErr: ErrUnverified},
{name: "a person who isn't an approver is refused an action", identity: "telegram:1003", row: action(propose, "owner"), wantErr: ErrNotApprover},
{name: "an approver without authority is refused a raise", identity: "telegram:1001", row: store.Request{Kind: KindRaise}, wantErr: ErrNoAuthority},
{name: "authority answers a raise", identity: "telegram:1002", row: store.Request{Kind: KindRaise}},
{name: "a person on the path is refused until the request reaches them", identity: "telegram:1004", row: action(propose, "owner"), wantErr: ErrNotAsked},
{name: "a person on the path answers once the request reaches them", identity: "telegram:1004", row: action(propose, "owner", "second")},
{name: "authority answers before it is asked", identity: "telegram:1002", row: action(propose, "owner")},
{name: "an action at escalate needs authority even from someone asked", identity: "telegram:1001", row: action(escalate, "owner"), wantErr: ErrNeedsAuthority},
{name: "authority answers an action at escalate", identity: "telegram:1002", row: action(escalate, "owner")},
}
m := &Module{people: people}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, err := m.verify(context.Background(), tt.identity, tt.kind)
_, err := m.verify(context.Background(), tt.identity, tt.row)
if !errors.Is(err, tt.wantErr) {
t.Fatalf("err = %v, want %v", err, tt.wantErr)
}
+110
View File
@@ -0,0 +1,110 @@
package approvals
import (
"context"
"errors"
"fmt"
"strconv"
"github.com/google/uuid"
"github.com/samcharles93/archie-core/modules/deliveries"
"github.com/samcharles93/archie-core/modules/governance"
"github.com/samcharles93/archie-core/modules/judgement"
)
// ChangeRequests is the seam between approval routing and a change-request
// connector. Routing asks one question of an action, whether it needs a change
// request; every action that doesn't goes to the approvers' channels. The
// rules that answer it are deterministic and run after the guards.
type ChangeRequests interface {
// Needed reports whether d goes through a change request.
Needed(ctx context.Context, d judgement.Decision) (bool, error)
// Open raises the change request for r. It may be called again for the same
// request and must not open a second one.
Open(ctx context.Context, r Request) error
// Withdraw withdraws r's change request if it is still open.
Withdraw(ctx context.Context, r Request) error
}
// NoChangeRequests is the routing while no change-request connector exists:
// no action needs one, so there is nothing to open or withdraw.
type NoChangeRequests struct{}
// Needed reports false.
func (NoChangeRequests) Needed(context.Context, judgement.Decision) (bool, error) { return false, nil }
// Open does nothing.
func (NoChangeRequests) Open(context.Context, Request) error { return nil }
// Withdraw does nothing.
func (NoChangeRequests) Withdraw(context.Context, Request) error { return nil }
// UseChangeRequests makes the module route approvals through c.
func (m *Module) UseChangeRequests(c ChangeRequests) { m.changes = c }
// Escalate passes a pending request to step n of its path, telling every
// channel who it has reached and until when. The step's deadline runs from
// now. A request that has since been answered is left alone.
func (m *Module) Escalate(ctx context.Context, id uuid.UUID, n int, to governance.PathStep) error {
row, err := m.store.ByID(ctx, m.orgID, id)
if err != nil {
return err
}
if row.State != StatePending {
return nil
}
until := m.now().UTC().Add(to.Deadline())
who := "the next person on the escalation path"
if to.Name != "" {
who = to.Name
}
// The notice goes first and is idempotent by key, so an escalation retried
// after the move failed still tells people once.
if err := m.tell.Notify(ctx, deliveries.Notification{
RunID: row.RunID, Key: row.Key + ":escalated:" + strconv.Itoa(n),
Title: "Approval escalated: " + row.Connector + " " + row.Action,
Body: "Nobody answered in time, so it now goes to " + who + " and waits until " + until.Format("2006-01-02 15:04 UTC") + ". " + row.Summary,
}); err != nil {
return err
}
if _, _, err := m.store.Escalate(ctx, m.orgID, id, n, to.Person, until); err != nil {
return err
}
m.log.InfoContext(ctx, "approval request escalated", "request", id, "step", n)
return nil
}
// Accept counts an approval toward raising its action class, once, when the
// action it covers is about to run. An approval that is discarded first never
// counts.
func (m *Module) Accept(ctx context.Context, id uuid.UUID) error {
row, counted, err := m.store.Count(ctx, m.orgID, id)
if err != nil || !counted {
return err
}
return m.learn(ctx, row, Answer{
RequestID: row.ID, Approved: true, By: row.AnsweredBy.String, Via: row.AnsweredVia.String, At: row.AnsweredAt.Time,
})
}
// Discard withdraws an approval whose action changed before it could run, and
// says why. A discarded approval doesn't count toward a raise, and a request
// for the changed action is a new one.
func (m *Module) Discard(ctx context.Context, id uuid.UUID, because string) error {
row, discarded, err := m.store.Discard(ctx, m.orgID, id, because)
if err != nil || !discarded {
return err
}
m.log.InfoContext(ctx, "approval discarded", "request", id, "because", because)
if err := m.tell.Told(ctx, deliveries.ApprovalRequest{
RunID: row.RunID, RequestID: row.ID, Key: row.Key + ":ask",
Title: "Approval discarded", Body: because,
}, "Discarded: "+because); err != nil {
return fmt.Errorf("tell discard: %w", err)
}
return nil
}
// ErrNoPath means a request was asked without anyone to put it to.
var ErrNoPath = errors.New("an approval request needs an escalation path")
+79 -4
View File
@@ -38,6 +38,19 @@ type Request struct {
AnsweredAt sql.NullTime `db:"answered_at"`
AnsweredBy sql.NullString `db:"answered_by"`
AnsweredVia sql.NullString `db:"answered_via"`
// Step is the escalation step the request is at, StepDeadline when that
// step runs out, and Asked a JSON array of the ids of the people it has
// reached so far.
Step int `db:"step"`
StepDeadline sql.NullTime `db:"step_deadline"`
Asked string `db:"asked"`
// ChangeRequest is whether the request went through a change request
// rather than the approvers' channels.
ChangeRequest bool `db:"change_request"`
// Counted is whether the approval was counted toward raising its class.
Counted bool `db:"counted"`
// DiscardedBecause is why an approval no longer stood.
DiscardedBecause sql.NullString `db:"discarded_because"`
}
// Store is the approvals module's Postgres store.
@@ -49,7 +62,8 @@ type Store struct {
func New(db *sqlx.DB) *Store { return &Store{db: db} }
const requestColumns = `id, org_id, run_id, key, decision_id, kind, digest, connector, action, target_kind,
target_id, level, summary, state, deadline, created_at, answered_at, answered_by, answered_via`
target_id, level, summary, state, deadline, created_at, answered_at, answered_by, answered_via,
step, step_deadline, asked::text AS asked, change_request, counted, discarded_because`
// Put records r unless a request already exists for its run and key, and
// returns the request that stands either way.
@@ -57,12 +71,12 @@ func (s *Store) Put(ctx context.Context, r Request) (Request, error) {
var out Request
err := s.db.GetContext(ctx, &out, `
INSERT INTO approvals.requests (id, org_id, run_id, key, decision_id, kind, digest, connector, action,
target_kind, target_id, level, summary, state, deadline)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)
target_kind, target_id, level, summary, state, deadline, step_deadline, asked, change_request)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, CAST($17 AS jsonb), $18)
ON CONFLICT (org_id, run_id, key) DO NOTHING
RETURNING `+requestColumns,
r.ID, r.OrgID, r.RunID, r.Key, r.DecisionID, r.Kind, r.Digest, r.Connector, r.Action, r.TargetKind,
r.TargetID, r.Level, r.Summary, r.State, r.Deadline)
r.TargetID, r.Level, r.Summary, r.State, r.Deadline, r.StepDeadline, r.askedOrEmpty(), r.ChangeRequest)
if errors.Is(err, sql.ErrNoRows) {
return s.Get(ctx, r.OrgID, r.RunID, r.Key)
}
@@ -72,6 +86,13 @@ func (s *Store) Put(ctx context.Context, r Request) (Request, error) {
return out, nil
}
func (r Request) askedOrEmpty() string {
if r.Asked == "" {
return "[]"
}
return r.Asked
}
// Get returns the request for one run's key.
func (s *Store) Get(ctx context.Context, orgID string, runID uuid.UUID, key string) (Request, error) {
var out Request
@@ -104,6 +125,60 @@ func (s *Store) Answer(ctx context.Context, orgID string, id uuid.UUID, state, b
return out, nil
}
// Escalate moves a pending request to step, which has reached person and runs
// out at until. It returns false when the request is no longer pending, or has
// already moved that far, so a replayed escalation changes nothing.
func (s *Store) Escalate(ctx context.Context, orgID string, id uuid.UUID, step int, person string, until time.Time) (Request, bool, error) {
var out Request
err := s.db.GetContext(ctx, &out, `
UPDATE approvals.requests
SET step = $3, step_deadline = $4,
asked = CASE WHEN $5::text = '' OR asked @> to_jsonb($5::text) THEN asked ELSE asked || to_jsonb($5::text) END
WHERE org_id = $1 AND id = $2 AND state = 'pending' AND step < $3
RETURNING `+requestColumns, orgID, id, step, until, person)
if errors.Is(err, sql.ErrNoRows) {
return Request{}, false, nil
}
if err != nil {
return Request{}, false, fmt.Errorf("escalate request: %w", err)
}
return out, true, nil
}
// Discard withdraws an approval that no longer stands, and only while it has
// not been counted. It returns false when there was nothing to discard.
func (s *Store) Discard(ctx context.Context, orgID string, id uuid.UUID, because string) (Request, bool, error) {
var out Request
err := s.db.GetContext(ctx, &out, `
UPDATE approvals.requests SET state = 'discarded', discarded_because = $3
WHERE org_id = $1 AND id = $2 AND state = 'approved' AND NOT counted
RETURNING `+requestColumns, orgID, id, because)
if errors.Is(err, sql.ErrNoRows) {
return Request{}, false, nil
}
if err != nil {
return Request{}, false, fmt.Errorf("discard approval: %w", err)
}
return out, true, nil
}
// Count marks an approval as counted, once. It returns false when the approval
// was already counted or no longer stands.
func (s *Store) Count(ctx context.Context, orgID string, id uuid.UUID) (Request, bool, error) {
var out Request
err := s.db.GetContext(ctx, &out, `
UPDATE approvals.requests SET counted = true
WHERE org_id = $1 AND id = $2 AND state = 'approved' AND NOT counted AND kind = 'action'
RETURNING `+requestColumns, orgID, id)
if errors.Is(err, sql.ErrNoRows) {
return Request{}, false, nil
}
if err != nil {
return Request{}, false, fmt.Errorf("count approval: %w", err)
}
return out, true, nil
}
// ByID returns one request.
func (s *Store) ByID(ctx context.Context, orgID string, id uuid.UUID) (Request, error) {
var out Request
+140
View File
@@ -0,0 +1,140 @@
package runs
import (
"context"
"encoding/json/jsontext"
"slices"
"testing"
"github.com/google/uuid"
"github.com/samcharles93/archie-core/modules/approvals"
"github.com/samcharles93/archie-core/modules/connectors"
"github.com/samcharles93/archie-core/modules/governance"
"github.com/samcharles93/archie-core/modules/judgement"
"github.com/samcharles93/archie-core/modules/runs/internal/store"
)
// TestAfterApprovalDiscardsAnApprovalThatNoLongerMatchesTheAction guards the
// rule that an approval covers the action as it was shown: when something else
// changed the target or effect, or judgement got worse, the approval is
// discarded and the action is put to people again, never run on the old one.
func TestAfterApprovalDiscardsAnApprovalThatNoLongerMatchesTheAction(t *testing.T) {
decided := func(level judgement.Level, mutate func(*judgement.Decision)) judgement.Decision {
d := judgement.Decision{Level: level, Action: judgement.Action{
Target: connectors.Target{Kind: "service", ID: "api", Environment: "prod"},
Effect: connectors.Effect{Summary: "restart"},
}}
if mutate != nil {
mutate(&d)
}
return d
}
approved := decided(judgement.ProposeAndWait, nil)
tests := []struct {
name string
fresh judgement.Decision
want afterwards
}{
{name: "an unchanged action runs on its approval", fresh: decided(judgement.ProposeAndWait, nil), want: runApproval},
{name: "a worse verdict is put to people again", fresh: decided(judgement.Escalate, nil), want: reaskPerson},
{name: "a changed target is put to people again", fresh: decided(judgement.ProposeAndWait, func(d *judgement.Decision) {
d.Action.Target.ID = "db"
}), want: reaskPerson},
{name: "a changed environment is put to people again", fresh: decided(judgement.ProposeAndWait, func(d *judgement.Decision) {
d.Action.Target.Environment = "staging"
}), want: reaskPerson},
{name: "a changed effect is put to people again", fresh: decided(judgement.ProposeAndWait, func(d *judgement.Decision) {
d.Action.Effect.Summary = "delete"
}), want: reaskPerson},
{name: "a refusal discards the approval", fresh: decided(judgement.Never, nil), want: refuseAction},
{name: "a verdict that lets it run needs no approval", fresh: decided(judgement.ActAndTell, nil), want: runFresh},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, why := afterApproval(approved, tt.fresh)
if got != tt.want {
t.Fatalf("afterApproval = %v (%s), want %v", got, why, tt.want)
}
if (got == reaskPerson || got == refuseAction) && why == "" {
t.Fatal("a discarded approval gives no reason")
}
})
}
}
// signalAfter is a worker store whose answer arrives once the run has waited
// that many times, as when a person answers during a later escalation step.
type signalAfter struct {
fakeWork
waits int
arrives int
}
func (s *signalAfter) Signalled(context.Context, uuid.UUID, string) (jsontext.Value, bool, error) {
s.waits++
if s.arrives > 0 && s.waits >= s.arrives {
return jsontext.Value(`{"approved":true,"by":"Second"}`), true, nil
}
return nil, false, nil
}
// escalations records what the run asked of approvals.
type escalations struct {
steps []int
expired int
}
func (e *escalations) Ask(context.Context, uuid.UUID, string, judgement.Decision, []governance.PathStep) (approvals.Request, error) {
return approvals.Request{}, nil
}
func (e *escalations) Escalate(_ context.Context, _ uuid.UUID, step int, _ governance.PathStep) error {
e.steps = append(e.steps, step)
return nil
}
func (e *escalations) Expire(context.Context, uuid.UUID) error { e.expired++; return nil }
func (e *escalations) Accept(context.Context, uuid.UUID) error { return nil }
func (e *escalations) Discard(context.Context, uuid.UUID, string) error { return nil }
// TestWaitThroughPathMovesOnAtEachDeadlineAndExpiresAtTheEnd guards the timed
// escalation path: an unanswered request passes to each next step in order,
// the last step's expiry settles it as expired with nothing acted on, and an
// answer at any step stops the escalation there.
func TestWaitThroughPathMovesOnAtEachDeadlineAndExpiresAtTheEnd(t *testing.T) {
// A zero deadline has passed by the time it is waited on.
step := func(person string) governance.PathStep { return governance.PathStep{Person: person} }
tests := []struct {
name string
path []governance.PathStep
arrives int
wantSteps []int
wantExpired int
wantAnswered bool
}{
{name: "one step with no answer expires without escalating", path: []governance.PathStep{step("a")}, wantExpired: 1},
{name: "every step runs out and then the request expires", path: []governance.PathStep{step("a"), step("b"), step("c")}, wantSteps: []int{1, 2}, wantExpired: 1},
{name: "an answer at the second step stops the escalation", path: []governance.PathStep{step("a"), step("b"), step("c")}, arrives: 2, wantSteps: []int{1}, wantAnswered: true},
{name: "an answer at the first step escalates nowhere", path: []governance.PathStep{step("a"), step("b")}, arrives: 1, wantAnswered: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
work := &signalAfter{arrives: tt.arrives}
rc := newContext(context.Background(), work, store.Run{ID: uuid.New()}, uuid.New(), nil)
ask := &escalations{}
answer, expired, err := waitThroughPath(rc, ask, "k", approvals.Request{ID: uuid.New()}, tt.path)
if err != nil {
t.Fatalf("waitThroughPath: %v", err)
}
if !slices.Equal(ask.steps, tt.wantSteps) || ask.expired != tt.wantExpired {
t.Fatalf("escalated %v and expired %d times, want %v and %d", ask.steps, ask.expired, tt.wantSteps, tt.wantExpired)
}
if expired != (tt.wantExpired > 0) || answer.Approved != tt.wantAnswered {
t.Fatalf("expired = %v, approved = %v", expired, answer.Approved)
}
})
}
}
+197 -97
View File
@@ -8,6 +8,7 @@ import (
"fmt"
"regexp"
"slices"
"strconv"
"strings"
"time"
@@ -25,10 +26,10 @@ import (
)
// Settings is where a workflow finds what the organisation decided: the agent
// model it runs, and how long an approval request waits for an answer.
// model it runs, and the escalation path an approval request takes.
type Settings interface {
AgentModel(ctx context.Context) (governance.Model, error)
ApprovalDeadline(ctx context.Context) (time.Duration, error)
EscalationPath(ctx context.Context, needAuthority bool) ([]governance.PathStep, error)
RecurrenceThreshold(ctx context.Context) (int, error)
AttemptsPerRun(ctx context.Context) (int, error)
}
@@ -49,11 +50,16 @@ type Judge interface {
ForRun(ctx context.Context, runID uuid.UUID) ([]judgement.Decision, error)
}
// Approver asks people to allow an action judgement held, and settles a request
// whose deadline passed.
// Approver asks people to allow an action judgement held, moves a request
// that nobody answered to the next step of its path, and settles one whose last
// step ran out. An approval is counted when the action is about to run on it,
// and discarded when the action no longer matches it.
type Approver interface {
Ask(ctx context.Context, runID uuid.UUID, key string, d judgement.Decision, deadline time.Time) (approvals.Request, error)
Ask(ctx context.Context, runID uuid.UUID, key string, d judgement.Decision, path []governance.PathStep) (approvals.Request, error)
Escalate(ctx context.Context, id uuid.UUID, step int, to governance.PathStep) error
Expire(ctx context.Context, id uuid.UUID) error
Accept(ctx context.Context, id uuid.UUID) error
Discard(ctx context.Context, id uuid.UUID, because string) error
}
// Notifier tells people what a run found.
@@ -78,25 +84,6 @@ func stepBudget(ctx context.Context, settings Settings) (agentloop.Budget, error
return budget, nil
}
// pinnedDeadline is pinned in the run's first steps, like the model: a run that
// waits keeps the deadline it started with, whatever the setting becomes
// meanwhile. A step's value is encoded, so it travels as the duration's own
// text.
func pinnedDeadline(rc *Context, settings Settings) (time.Duration, error) {
text, err := rc.Step("approval-deadline", func(ctx context.Context, _ string) (string, error) {
d, err := settings.ApprovalDeadline(ctx)
return d.String(), err
})
if err != nil {
return 0, err
}
deadline, err := time.ParseDuration(text)
if err != nil {
return 0, fmt.Errorf("approval deadline %q: %w", text, err)
}
return deadline, nil
}
// Investigate is the workflow that has an agent look into the event that
// started the run and report its findings. The agent model and the tools are
// pinned in the run's first steps, so a resumed run asks the same model with
@@ -111,10 +98,6 @@ func Investigate(settings Settings, box Toolbox, judge Judge, ask Approver, noti
if err != nil {
return nil, err
}
deadline, err := pinnedDeadline(rc, settings)
if err != nil {
return nil, err
}
threshold, err := rc.Step("recurrence-threshold", func(ctx context.Context, _ string) (int, error) {
return settings.RecurrenceThreshold(ctx)
})
@@ -160,7 +143,7 @@ func Investigate(settings Settings, box Toolbox, judge Judge, ask Approver, noti
if err := recordCause(rc, mem, input, res, AgentRequest{Provider: provider, Model: model.Model, ModelInfo: info}); err != nil {
return nil, err
}
outcomes, err := settleProposals(rc, judge, ask, deadline)
outcomes, err := settleProposals(rc, judge, ask, settings)
if err != nil {
return nil, err
}
@@ -286,9 +269,9 @@ type approvalOutcome struct {
// settleProposals puts every action judgement held to a person, one at a time,
// and runs the ones they allow. Each ask, wait and run is its own step, so a
// resumed run picks up where it waited. A proposal nobody answers in time is
// settled as expired and nothing runs.
func settleProposals(rc *Context, judge Judge, ask Approver, deadline time.Duration) ([]approvalOutcome, error) {
// resumed run picks up where it waited. A proposal nobody answers before the
// last escalation step runs out is settled as expired and nothing runs.
func settleProposals(rc *Context, judge Judge, ask Approver, settings Settings) ([]approvalOutcome, error) {
held, err := rc.Step("proposals", func(ctx context.Context, _ string) ([]judgement.Decision, error) {
decisions, err := judge.ForRun(ctx, rc.RunID())
if err != nil {
@@ -301,7 +284,7 @@ func settleProposals(rc *Context, judge Judge, ask Approver, deadline time.Durat
}
out := make([]approvalOutcome, 0, len(held))
for _, d := range held {
outcome, err := settleOne(rc, judge, ask, d, deadline)
outcome, err := settleOne(rc, judge, ask, settings, d)
if err != nil {
return nil, err
}
@@ -310,37 +293,116 @@ func settleProposals(rc *Context, judge Judge, ask Approver, deadline time.Durat
return out, nil
}
func settleOne(rc *Context, judge Judge, ask Approver, d judgement.Decision, deadline time.Duration) (approvalOutcome, error) {
out := approvalOutcome{digest: d.Action.Digest, Action: d.Action.Connector + " " + d.Action.Name, Target: d.Action.Target.Kind + ":" + d.Action.Target.ID}
request, err := rc.Step("ask:"+d.Key, func(ctx context.Context, _ string) (approvals.Request, error) {
return ask.Ask(ctx, rc.RunID(), waitKey(d.Key), d, time.Now().Add(deadline))
// maxReapprovals bounds how many times an action that keeps changing under its
// approval is put to people again.
const maxReapprovals = 2
// settleOne gets one action approved and runs it. Before it runs it is judged
// again; an approval whose action has since changed is discarded and the
// changed action is put to people as a new request.
func settleOne(rc *Context, judge Judge, ask Approver, settings Settings, d judgement.Decision) (approvalOutcome, error) {
out := approvalOutcome{digest: d.Action.Digest}
for round := 0; ; round++ {
out.Action, out.Target = d.Action.Connector+" "+d.Action.Name, d.Action.Target.Kind+":"+d.Action.Target.ID
request, answer, err := askThroughPath(rc, ask, settings, d)
if err != nil {
return out, err
}
out.RequestID, out.State = request.ID, request.State
if request.State != approvals.StateApproved {
if request.State == approvals.StateDenied {
out.By = answer.By
}
return out, nil
}
out.By = answer.By
fresh, err := rc.Step("judge-again:"+d.Key, func(ctx context.Context, key string) (judgement.Decision, error) {
return judge.Assess(ctx, judgement.Request{
RunID: rc.RunID(), Key: key, Connector: d.Action.Connector, Action: d.Action.Name, Input: d.Action.Input,
Workflow: rc.Workflow(),
})
})
if err != nil {
return out, err
}
next, why := afterApproval(d, fresh)
switch next {
case reaskPerson, refuseAction:
if _, err := rc.Step("discard:"+d.Key, func(ctx context.Context, _ string) (bool, error) {
return true, ask.Discard(ctx, request.ID, why)
}); err != nil {
return out, err
}
out.State = approvals.StateDiscarded
if next == refuseAction || round == maxReapprovals {
out.Error = why
return out, nil
}
d = fresh
continue
case runFresh:
// The approval isn't needed and isn't counted.
out.Result, out.Error, out.Unknown = runAction(rc, judge, fresh, nil)
default:
if _, err := rc.Step("count:"+d.Key, func(ctx context.Context, _ string) (bool, error) {
return true, ask.Accept(ctx, request.ID)
}); err != nil {
return out, err
}
out.Result, out.Error, out.Unknown = runAction(rc, judge, d, &answer)
}
out.State = approvals.StateApproved
return out, nil
}
}
// askThroughPath puts d to the first step of its escalation path and waits.
// The path is pinned when it is first asked, so a request that waits keeps the
// deadlines it started with whatever the settings become meanwhile. It returns
// the request as it stood when it settled, and the answer if there was one.
func askThroughPath(rc *Context, ask Approver, settings Settings, d judgement.Decision) (approvals.Request, approvals.Answer, error) {
path, err := rc.Step("path:"+d.Key, func(ctx context.Context, _ string) ([]governance.PathStep, error) {
return settings.EscalationPath(ctx, d.Level == judgement.Escalate)
})
if err != nil {
return out, err
return approvals.Request{}, approvals.Answer{}, err
}
out.RequestID, out.State = request.ID, request.State
if request.State != approvals.StatePending {
return out, nil
}
answer, expired, err := waitForAnswer(rc, ask, d.Key, request, deadline)
request, err := rc.Step("ask:"+d.Key, func(ctx context.Context, _ string) (approvals.Request, error) {
return ask.Ask(ctx, rc.RunID(), waitKey(d.Key), d, path)
})
if err != nil {
return out, err
return request, approvals.Answer{}, err
}
if expired {
out.State = approvals.StateExpired
return out, nil
if request.State != approvals.StatePending && request.State != approvals.StateApproved {
return request, approvals.Answer{}, nil
}
if !answer.Approved {
out.State, out.By = approvals.StateDenied, answer.By
return out, nil
// A request already answered when it was asked again still has its answer
// waiting under the run's key, so the wait returns it at once.
answer, expired, err := waitThroughPath(rc, ask, d.Key, request, path)
switch {
case err != nil:
return request, answer, err
case expired:
request.State = approvals.StateExpired
case answer.Approved:
request.State = approvals.StateApproved
default:
request.State = approvals.StateDenied
}
out.By = answer.By
// The action's own failure is part of what the answer led to, and its step
// has recorded it, so the investigation still reports.
run, runErr := runApproved(rc, judge, d, answer)
out.State, out.Result, out.Error = approvals.StateApproved, run, errText(runErr)
out.Unknown = errors.Is(runErr, ErrOutcomeUnknown)
return out, nil
return request, answer, nil
}
// runAction runs the action judged as d, on approval when there is one, and
// returns what came of it. The action's own failure is part of what the answer
// led to, and its step has recorded it, so the investigation still reports.
func runAction(rc *Context, judge Judge, d judgement.Decision, approval *approvals.Answer) (result, failure string, unknown bool) {
var runErr error
if approval == nil {
result, runErr = runUnattended(rc, judge, d)
} else {
result, runErr = runApproved(rc, judge, d, *approval)
}
return result, errText(runErr), errors.Is(runErr, ErrOutcomeUnknown)
}
// unknownOutcome records a connector's unconfirmed action as a step whose
@@ -360,54 +422,92 @@ func errText(err error) string {
return err.Error()
}
// waitForAnswer waits for the person's answer. When the deadline passes with
// none, the request is settled as expired and the caller runs nothing.
func waitForAnswer(rc *Context, ask Approver, key string, request approvals.Request, deadline time.Duration) (approvals.Answer, bool, error) {
payload, err := rc.Wait(waitKey(key), deadline)
if errors.Is(err, ErrWaitTimedOut) {
if _, eerr := rc.Step("expire:"+key, func(ctx context.Context, _ string) (bool, error) {
return true, ask.Expire(ctx, request.ID)
}); eerr != nil {
return approvals.Answer{}, true, eerr
// waitThroughPath waits for an answer through each step of path in turn. A step
// that runs out with none passes the request to the next person; the last one
// settles it as expired and the caller runs nothing. Every wait is a durable
// run wait, so a restart resumes where each deadline stands.
func waitThroughPath(rc *Context, ask Approver, key string, request approvals.Request, path []governance.PathStep) (approvals.Answer, bool, error) {
for i, step := range path {
if i > 0 {
if _, err := rc.Step("escalate:"+key+":"+strconv.Itoa(i), func(ctx context.Context, _ string) (bool, error) {
return true, ask.Escalate(ctx, request.ID, i, step)
}); err != nil {
return approvals.Answer{}, false, err
}
}
return approvals.Answer{}, true, nil
payload, err := rc.Wait(waitKey(key), step.Deadline())
if errors.Is(err, ErrWaitTimedOut) {
continue
}
if err != nil {
return approvals.Answer{}, false, err
}
var answer approvals.Answer
if err := json.Unmarshal(payload, &answer); err != nil {
return approvals.Answer{}, false, fmt.Errorf("decode approval answer: %w", err)
}
return answer, false, nil
}
if err != nil {
return approvals.Answer{}, false, err
if _, err := rc.Step("expire:"+key, func(ctx context.Context, _ string) (bool, error) {
return true, ask.Expire(ctx, request.ID)
}); err != nil {
return approvals.Answer{}, true, err
}
var answer approvals.Answer
if err := json.Unmarshal(payload, &answer); err != nil {
return approvals.Answer{}, false, fmt.Errorf("decode approval answer: %w", err)
}
return answer, false, nil
return approvals.Answer{}, true, nil
}
// runApproved judges the action again and runs it, once. An approval covers
// the action the person was shown, so a judgement that came back worse than
// it was cannot be run on that approval.
func runApproved(rc *Context, judge Judge, d judgement.Decision, answer approvals.Answer) (string, error) {
fresh, err := rc.Step("judge-again:"+d.Key, func(ctx context.Context, key string) (judgement.Decision, error) {
return judge.Assess(ctx, judgement.Request{
RunID: rc.RunID(), Key: key, Connector: d.Action.Connector, Action: d.Action.Name, Input: d.Action.Input,
Workflow: rc.Workflow(),
})
})
if err != nil {
return "", err
// What an approval allows once the action is judged again.
type afterwards int
const (
// runApproval runs the action that was approved.
runApproval afterwards = iota
// runFresh runs the action on its new judgement, which lets it run without
// anyone's approval.
runFresh
// reaskPerson discards the approval and puts the changed action to people.
reaskPerson
// refuseAction discards the approval: the action is now refused.
refuseAction
)
// afterApproval says what an approval of approved means now that the action was
// judged again as fresh. An approval covers the action as it was shown, so one
// whose verdict got worse, or whose target or effect changed, no longer
// stands. The target and effect are what the connector resolves the action to
// now, which is what judgement has to tell that something else changed them.
func afterApproval(approved, fresh judgement.Decision) (afterwards, string) {
switch {
case fresh.Level == judgement.Never:
return refuseAction, "refused after approval: " + fresh.Reason
case fresh.Level.Runs():
return runFresh, ""
case fresh.Action.Target != approved.Action.Target:
return reaskPerson, "the target changed since it was approved"
case fresh.Action.Effect != approved.Action.Effect:
return reaskPerson, "the effect changed since it was approved"
case fresh.Level > approved.Level:
return reaskPerson, "judgement got worse since it was approved: now " + fresh.Level.String()
}
return runApproval, ""
}
// runUnattended runs an action its fresh judgement lets run without approval,
// once.
func runUnattended(rc *Context, judge Judge, d judgement.Decision) (string, error) {
return rc.Step("run:"+d.Key, func(ctx context.Context, key string) (string, error) {
switch {
case fresh.Level == judgement.Never:
return "", fmt.Errorf("refused after approval: %s", fresh.Reason)
case fresh.Level.Runs():
if err := begun(ctx, rc, key); err != nil {
return "", err
}
out, err := judge.Execute(ctx, fresh, d.Action.Input, key)
return out, unknownOutcome(err)
case fresh.Level > d.Level:
return "", fmt.Errorf("judgement got worse after approval: now %s", fresh.Level)
if err := begun(ctx, rc, key); err != nil {
return "", err
}
out, err := judge.Execute(ctx, d, d.Action.Input, key)
return out, unknownOutcome(err)
})
}
// runApproved runs the action a person approved, once. judgement runs only the
// action that was judged, whatever input comes with it.
func runApproved(rc *Context, judge Judge, d judgement.Decision, answer approvals.Answer) (string, error) {
return rc.Step("run:"+d.Key, func(ctx context.Context, key string) (string, error) {
if err := begun(ctx, rc, key); err != nil {
return "", err
}
+1 -1
View File
@@ -55,7 +55,7 @@ func TestActionStartedBeforeItsResultIsNeverRunAgain(t *testing.T) {
return []store.Step{{RunID: runID, Seq: markerBase, Name: startedPrefix + key, Outcome: store.OutcomeDone, Output: jsontext.Value("true")}}
}
agentKey := runID.String() + ":investigate:3"
approvedKey := runID.String() + ":1"
approvedKey := runID.String() + ":0"
tests := []struct {
name string