diff --git a/modules/approvals/approvals.go b/modules/approvals/approvals.go index e2b05a453..b09c1f05a 100644 --- a/modules/approvals/approvals.go +++ b/modules/approvals/approvals.go @@ -10,6 +10,7 @@ package approvals import ( "context" + "database/sql" "encoding/json/jsontext" json "encoding/json/v2" "errors" @@ -31,12 +32,14 @@ import ( "github.com/samcharles93/archie-core/modules/judgement" ) -// The states a request can be in. A settled request never changes again. +// The states a request can be in. A settled request never changes again, +// except that an approval the action no longer matched is discarded. const ( - StatePending = "pending" - StateApproved = "approved" - StateDenied = "denied" - StateExpired = "expired" + StatePending = "pending" + StateApproved = "approved" + StateDenied = "denied" + StateExpired = "expired" + StateDiscarded = "discarded" ) var ( @@ -50,6 +53,12 @@ var ( ErrUnverified = errors.New("that identity isn't held by any person, so its answer is refused") // ErrNotApprover means the person may not answer approvals for actions. ErrNotApprover = errors.New("that person isn't the default owner, on the escalation path or an authority, so their answer is refused") + // ErrNotAsked means the request hasn't yet reached that person on its + // escalation path. + ErrNotAsked = errors.New("that person hasn't been asked yet, so their answer is refused") + // ErrNeedsAuthority means the action needs a person with authority, and + // the answer came from one without. + ErrNeedsAuthority = errors.New("that action needs a person with authority, so their answer is refused") // ErrNoAuthority means the answer to a raise came from a person without // authority over policy and ceilings. ErrNoAuthority = errors.New("that person holds no authority over policy and ceilings, so their answer to a raise is refused") @@ -79,10 +88,19 @@ type Request struct { Level string Summary string State string - Deadline time.Time - AnsweredAt time.Time - AnsweredBy string - AnsweredVia string + // Deadline is when the request runs out for good: the end of the last + // escalation step. + Deadline time.Time + // Step is the escalation step the request is at, and StepDeadline when it + // moves on. + Step int + StepDeadline time.Time + // ChangeRequest is whether the request went through a change request + // rather than the approvers' channels. + ChangeRequest bool + AnsweredAt time.Time + AnsweredBy string + AnsweredVia string } // Answer is what the run wakes with. @@ -110,6 +128,10 @@ type View struct { Summary string `json:"summary"` State string `json:"state"` Deadline time.Time `json:"deadline"` + // Step is the escalation step the request is at, and StepDeadline when it + // moves to the next. + Step int `json:"step"` + StepDeadline time.Time `json:"step_deadline,omitzero"` // DecisionID is the judged action the request is about. DecisionID uuid.UUID `json:"decision_id"` RequestedAt time.Time `json:"requested_at"` @@ -123,7 +145,7 @@ func (r Request) View() View { return View{ ID: r.ID, Kind: r.Kind, RunID: r.RunID, Action: r.Connector + " " + r.Action, Target: r.TargetKind + ":" + r.TargetID, Level: r.Level, - Summary: r.Summary, State: r.State, Deadline: r.Deadline, + Summary: r.Summary, State: r.State, Deadline: r.Deadline, Step: r.Step, StepDeadline: r.StepDeadline, DecisionID: r.DecisionID, RequestedAt: r.RequestedAt, AnsweredBy: r.AnsweredBy, AnsweredVia: r.AnsweredVia, AnsweredAt: r.AnsweredAt, } @@ -172,6 +194,7 @@ type Module struct { classes Classes policy Policy people People + changes ChangeRequests log *slog.Logger now func() time.Time } @@ -183,20 +206,44 @@ type Module struct { func NewModule(db *sqlx.DB, log *slog.Logger, orgID string, tell Delivery, wake Signaller, classes Classes, policy Policy, people People) *Module { return &Module{ orgID: orgID, store: store.New(db), tell: tell, wake: wake, classes: classes, policy: policy, people: people, - log: log, now: time.Now, + changes: NoChangeRequests{}, log: log, now: time.Now, } } -// Ask records a request for d and tells people about it. Asking again for the -// same run and key returns the request that stands, so a replayed step asks -// once. -func (m *Module) Ask(ctx context.Context, runID uuid.UUID, key string, d judgement.Decision, deadline time.Time) (Request, error) { +// Ask records a request for d and puts it to the first step of path. It runs +// out for good when the last step does. Asking again for the same run and key +// returns the request that stands, so a replayed step asks once. An action the +// change-request rules pick goes through a change request, and every other one +// through the approvers' channels. +func (m *Module) Ask(ctx context.Context, runID uuid.UUID, key string, d judgement.Decision, path []governance.PathStep) (Request, error) { + if len(path) == 0 { + return Request{}, ErrNoPath + } + changeRequest, err := m.changes.Needed(ctx, d) + if err != nil { + return Request{}, fmt.Errorf("route approval: %w", err) + } + now := m.now().UTC() + var total time.Duration + for _, st := range path { + total += st.Deadline() + } + asked := []string{} + if path[0].Person != "" { + asked = append(asked, path[0].Person) + } + askedJSON, err := json.Marshal(asked) + if err != nil { + return Request{}, fmt.Errorf("encode asked: %w", err) + } row, err := m.store.Put(ctx, store.Request{ ID: uuid.New(), OrgID: m.orgID, RunID: runID, Key: key, DecisionID: d.ID, Kind: KindAction, Digest: d.Action.Digest, Level: d.Level.String(), Connector: d.Action.Connector, Action: d.Action.Name, TargetKind: d.Action.Target.Kind, TargetID: d.Action.Target.ID, - Summary: Summary(d), State: StatePending, Deadline: deadline.UTC(), + Summary: Summary(d), State: StatePending, Deadline: now.Add(total), + StepDeadline: sql.NullTime{Time: now.Add(path[0].Deadline()), Valid: true}, Asked: string(askedJSON), + ChangeRequest: changeRequest, }) if err != nil { return Request{}, err @@ -205,10 +252,16 @@ func (m *Module) Ask(ctx context.Context, runID uuid.UUID, key string, d judgeme if row.State != StatePending { return r, nil } + if row.ChangeRequest { + if err := m.changes.Open(ctx, r); err != nil { + return Request{}, fmt.Errorf("open change request: %w", err) + } + return r, nil + } if err := m.tell.Ask(ctx, deliveries.ApprovalRequest{ RunID: r.RunID, RequestID: r.ID, Key: r.Key + ":ask", Title: "Approval needed: " + d.Action.Connector + " " + d.Action.Name, - Body: r.Summary, ExpiresAt: r.Deadline, + Body: r.Summary, ExpiresAt: r.StepDeadline, }); err != nil { return Request{}, fmt.Errorf("send approval request: %w", err) } @@ -244,20 +297,20 @@ func (m *Module) answerAs(ctx context.Context, id uuid.UUID, by, via string, app if err := m.wakeWith(ctx, row, answer); err != nil { return Request{}, err } - // The run has what it waited for; the answer's bearing on the class comes - // after, so a failure here is reported without holding the run. - if err := m.learn(ctx, row, answer); err != nil { - return Request{}, err + // The run has what it waited for. A denial tightens the class now. An + // approval counts only once the action is about to run on it, so one that + // is discarded first counts for nothing. + if !answer.Approved { + if err := m.classes.Denied(ctx, row.DecisionID); err != nil { + return Request{}, err + } } return requestOf(row, judgement.Decision{}), nil } -// learn counts an answer to an action into its class. An approval that +// learn counts an approval of an action into its class. An approval that // completes a run of them earns a raise, which is put to a person. func (m *Module) learn(ctx context.Context, row store.Request, a Answer) error { - if !a.Approved { - return m.classes.Denied(ctx, row.DecisionID) - } need, err := m.policy.RaiseAfter(ctx) if err != nil { return fmt.Errorf("read raise count: %w", err) @@ -342,7 +395,7 @@ func (m *Module) Settle(ctx context.Context, id uuid.UUID, identity, via string, if err != nil { return false, err } - who, err := m.verify(ctx, identity, row.Kind) + who, err := m.verify(ctx, identity, row) if refusal(err) { m.log.WarnContext(ctx, "approval answer refused", "request", id, "identity", identity, "reason", err.Error()) return false, m.store.Refuse(ctx, store.Refusal{ @@ -362,31 +415,57 @@ func (m *Module) Settle(ctx context.Context, id uuid.UUID, identity, via string, return false, err } -// verify says whether identity may answer a request of kind. An action is -// answered by anyone who answers for actions; a raise changes policy, so only -// a person with authority answers it. -func (m *Module) verify(ctx context.Context, identity, kind string) (governance.Verified, error) { +// verify says whether identity may answer request row. An action is answered +// by someone who answers for actions and whom the request has reached on its +// escalation path, or who holds authority, and an action at the escalate level +// needs authority. A raise changes policy, so only a person with authority +// answers it. +func (m *Module) verify(ctx context.Context, identity string, row store.Request) (governance.Verified, error) { who, ok, err := m.people.Verify(ctx, identity) - switch { - case err != nil: + if err != nil { return governance.Verified{}, fmt.Errorf("verify %s: %w", identity, err) - case !ok: - return governance.Verified{}, ErrUnverified - case kind == KindRaise && !who.Authority: - return governance.Verified{}, ErrNoAuthority - case kind != KindRaise && !who.Approver: - return governance.Verified{}, ErrNotApprover } - return who, nil + if !ok { + return governance.Verified{}, ErrUnverified + } + return who, mayAnswer(who, row) +} + +func mayAnswer(who governance.Verified, row store.Request) error { + switch { + case row.Kind == KindRaise && !who.Authority: + return ErrNoAuthority + case row.Kind == KindRaise: + return nil + case !who.Approver: + return ErrNotApprover + case row.Level == judgement.Escalate.String() && !who.Authority: + return ErrNeedsAuthority + case !who.Authority && !slices.Contains(askedOf(row), who.ID): + return ErrNotAsked + } + return nil +} + +// askedOf is the ids of the people the request has reached. +func askedOf(row store.Request) []string { + var ids []string + if err := json.Unmarshal([]byte(row.Asked), &ids); err != nil { + return nil + } + return ids } // refusal reports whether err is a refused answer rather than a failure. func refusal(err error) bool { - return errors.Is(err, ErrUnverified) || errors.Is(err, ErrNotApprover) || errors.Is(err, ErrNoAuthority) + return errors.Is(err, ErrUnverified) || errors.Is(err, ErrNotApprover) || errors.Is(err, ErrNoAuthority) || + errors.Is(err, ErrNotAsked) || errors.Is(err, ErrNeedsAuthority) } -// Expire settles a request whose deadline passed, so the trail shows why it -// never ran. +// Expire settles a request whose last escalation step ran out, so the trail +// shows why it never ran. Nothing is acted on, an open change request is +// withdrawn, and the report goes to every channel. Expiry says nothing about +// the action's class, so it is not tightened. func (m *Module) Expire(ctx context.Context, id uuid.UUID) error { row, err := m.store.Answer(ctx, m.orgID, id, StateExpired, "deadline", "clock", m.now().UTC()) if err != nil { @@ -399,18 +478,26 @@ func (m *Module) Expire(ctx context.Context, id uuid.UUID) error { return nil } m.log.InfoContext(ctx, "approval request expired", "request", row.ID, "run", row.RunID) + withdrawn := "" + if row.ChangeRequest { + if err := m.changes.Withdraw(ctx, requestOf(row, judgement.Decision{})); err != nil { + return fmt.Errorf("withdraw change request: %w", err) + } + withdrawn = " Its change request was withdrawn." + } if err := m.tell.Told(ctx, deliveries.ApprovalRequest{ RunID: row.RunID, RequestID: row.ID, Key: row.Key + ":ask", - Title: "Approval expired", Body: "The request ran out of time, so nothing was run.", + Title: "Approval expired", Body: "The request ran out of time, so nothing was run." + withdrawn, }, "Expired without an answer."); err != nil { return err } - // The escalation goes to every channel. Its key is stable, so an Expire + // The report goes to every channel. Its key is stable, so an Expire // retried after a failed delivery sends it once. return m.tell.Notify(ctx, deliveries.Notification{ RunID: row.RunID, Key: row.Key + ":expired", Title: "Approval timed out: " + row.Connector + " " + row.Action, - Body: "Nobody answered before the deadline, so " + row.Connector + " " + row.Action + " did not run. " + row.Summary, + Body: "Nobody answered before the last escalation step ran out, so " + row.Connector + " " + row.Action + + " did not run." + withdrawn + " " + row.Summary, }) } @@ -619,6 +706,10 @@ func requestOf(row store.Request, d judgement.Decision) Request { ID: row.ID, Kind: row.Kind, RunID: row.RunID, Key: row.Key, Decision: d, DecisionID: row.DecisionID, RequestedAt: row.CreatedAt, Connector: row.Connector, Action: row.Action, TargetKind: row.TargetKind, TargetID: row.TargetID, Level: row.Level, Summary: row.Summary, State: row.State, Deadline: row.Deadline, + Step: row.Step, ChangeRequest: row.ChangeRequest, + } + if row.StepDeadline.Valid { + r.StepDeadline = row.StepDeadline.Time } if row.AnsweredAt.Valid { r.AnsweredAt = row.AnsweredAt.Time diff --git a/modules/approvals/approvals_test.go b/modules/approvals/approvals_test.go index b983513ed..ab06dfdab 100644 --- a/modules/approvals/approvals_test.go +++ b/modules/approvals/approvals_test.go @@ -3,11 +3,13 @@ package approvals import ( "context" "errors" + "strings" "testing" "time" "github.com/samcharles93/archie-core/modules/approvals/internal/store" "github.com/samcharles93/archie-core/modules/governance" + "github.com/samcharles93/archie-core/modules/judgement" ) type fakePeople map[string]governance.Verified @@ -19,31 +21,42 @@ func (f fakePeople) Verify(_ context.Context, identity string) (governance.Verif // TestVerifyAcceptsAnAnswerOnlyFromAPersonWhoMayGiveIt guards the rule that an // approval counts only from a verified identity, on every channel: an identity -// no person holds, or a person without the standing for that kind of request, -// is refused rather than settling it. +// no person holds, a person without the standing for that kind of request, a +// person the request hasn't reached on its escalation path, or one without the +// authority the action's level needs, is refused rather than settling it. func TestVerifyAcceptsAnAnswerOnlyFromAPersonWhoMayGiveIt(t *testing.T) { people := fakePeople{ - "telegram:1001": {Name: "Owner", Approver: true}, - "telegram:1002": {Name: "Authority", Approver: true, Authority: true}, - "telegram:1003": {Name: "Bystander"}, + "telegram:1001": {ID: "owner", Name: "Owner", Approver: true}, + "telegram:1002": {ID: "lead", Name: "Authority", Approver: true, Authority: true}, + "telegram:1003": {ID: "bystander", Name: "Bystander"}, + "telegram:1004": {ID: "second", Name: "Second", Approver: true}, } + action := func(level string, asked ...string) store.Request { + return store.Request{Kind: KindAction, Level: level, Asked: `["` + strings.Join(asked, `","`) + `"]`} + } + propose, escalate := judgement.ProposeAndWait.String(), judgement.Escalate.String() tests := []struct { name string identity string - kind string + row store.Request wantErr error }{ - {name: "an approver answers an action", identity: "telegram:1001", kind: KindAction}, - {name: "an unmapped identity is refused", identity: "telegram:9999", kind: KindAction, wantErr: ErrUnverified}, - {name: "an unmapped identity can't answer a raise", identity: "telegram:9999", kind: KindRaise, wantErr: ErrUnverified}, - {name: "a person who isn't an approver is refused an action", identity: "telegram:1003", kind: KindAction, wantErr: ErrNotApprover}, - {name: "an approver without authority is refused a raise", identity: "telegram:1001", kind: KindRaise, wantErr: ErrNoAuthority}, - {name: "authority answers a raise", identity: "telegram:1002", kind: KindRaise}, + {name: "an approver the request has reached answers", identity: "telegram:1001", row: action(propose, "owner")}, + {name: "an unmapped identity is refused", identity: "telegram:9999", row: action(propose, "owner"), wantErr: ErrUnverified}, + {name: "an unmapped identity can't answer a raise", identity: "telegram:9999", row: store.Request{Kind: KindRaise}, wantErr: ErrUnverified}, + {name: "a person who isn't an approver is refused an action", identity: "telegram:1003", row: action(propose, "owner"), wantErr: ErrNotApprover}, + {name: "an approver without authority is refused a raise", identity: "telegram:1001", row: store.Request{Kind: KindRaise}, wantErr: ErrNoAuthority}, + {name: "authority answers a raise", identity: "telegram:1002", row: store.Request{Kind: KindRaise}}, + {name: "a person on the path is refused until the request reaches them", identity: "telegram:1004", row: action(propose, "owner"), wantErr: ErrNotAsked}, + {name: "a person on the path answers once the request reaches them", identity: "telegram:1004", row: action(propose, "owner", "second")}, + {name: "authority answers before it is asked", identity: "telegram:1002", row: action(propose, "owner")}, + {name: "an action at escalate needs authority even from someone asked", identity: "telegram:1001", row: action(escalate, "owner"), wantErr: ErrNeedsAuthority}, + {name: "authority answers an action at escalate", identity: "telegram:1002", row: action(escalate, "owner")}, } m := &Module{people: people} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - _, err := m.verify(context.Background(), tt.identity, tt.kind) + _, err := m.verify(context.Background(), tt.identity, tt.row) if !errors.Is(err, tt.wantErr) { t.Fatalf("err = %v, want %v", err, tt.wantErr) } diff --git a/modules/approvals/escalation.go b/modules/approvals/escalation.go new file mode 100644 index 000000000..535a1aa27 --- /dev/null +++ b/modules/approvals/escalation.go @@ -0,0 +1,110 @@ +package approvals + +import ( + "context" + "errors" + "fmt" + "strconv" + + "github.com/google/uuid" + + "github.com/samcharles93/archie-core/modules/deliveries" + "github.com/samcharles93/archie-core/modules/governance" + "github.com/samcharles93/archie-core/modules/judgement" +) + +// ChangeRequests is the seam between approval routing and a change-request +// connector. Routing asks one question of an action, whether it needs a change +// request; every action that doesn't goes to the approvers' channels. The +// rules that answer it are deterministic and run after the guards. +type ChangeRequests interface { + // Needed reports whether d goes through a change request. + Needed(ctx context.Context, d judgement.Decision) (bool, error) + // Open raises the change request for r. It may be called again for the same + // request and must not open a second one. + Open(ctx context.Context, r Request) error + // Withdraw withdraws r's change request if it is still open. + Withdraw(ctx context.Context, r Request) error +} + +// NoChangeRequests is the routing while no change-request connector exists: +// no action needs one, so there is nothing to open or withdraw. +type NoChangeRequests struct{} + +// Needed reports false. +func (NoChangeRequests) Needed(context.Context, judgement.Decision) (bool, error) { return false, nil } + +// Open does nothing. +func (NoChangeRequests) Open(context.Context, Request) error { return nil } + +// Withdraw does nothing. +func (NoChangeRequests) Withdraw(context.Context, Request) error { return nil } + +// UseChangeRequests makes the module route approvals through c. +func (m *Module) UseChangeRequests(c ChangeRequests) { m.changes = c } + +// Escalate passes a pending request to step n of its path, telling every +// channel who it has reached and until when. The step's deadline runs from +// now. A request that has since been answered is left alone. +func (m *Module) Escalate(ctx context.Context, id uuid.UUID, n int, to governance.PathStep) error { + row, err := m.store.ByID(ctx, m.orgID, id) + if err != nil { + return err + } + if row.State != StatePending { + return nil + } + until := m.now().UTC().Add(to.Deadline()) + who := "the next person on the escalation path" + if to.Name != "" { + who = to.Name + } + // The notice goes first and is idempotent by key, so an escalation retried + // after the move failed still tells people once. + if err := m.tell.Notify(ctx, deliveries.Notification{ + RunID: row.RunID, Key: row.Key + ":escalated:" + strconv.Itoa(n), + Title: "Approval escalated: " + row.Connector + " " + row.Action, + Body: "Nobody answered in time, so it now goes to " + who + " and waits until " + until.Format("2006-01-02 15:04 UTC") + ". " + row.Summary, + }); err != nil { + return err + } + if _, _, err := m.store.Escalate(ctx, m.orgID, id, n, to.Person, until); err != nil { + return err + } + m.log.InfoContext(ctx, "approval request escalated", "request", id, "step", n) + return nil +} + +// Accept counts an approval toward raising its action class, once, when the +// action it covers is about to run. An approval that is discarded first never +// counts. +func (m *Module) Accept(ctx context.Context, id uuid.UUID) error { + row, counted, err := m.store.Count(ctx, m.orgID, id) + if err != nil || !counted { + return err + } + return m.learn(ctx, row, Answer{ + RequestID: row.ID, Approved: true, By: row.AnsweredBy.String, Via: row.AnsweredVia.String, At: row.AnsweredAt.Time, + }) +} + +// Discard withdraws an approval whose action changed before it could run, and +// says why. A discarded approval doesn't count toward a raise, and a request +// for the changed action is a new one. +func (m *Module) Discard(ctx context.Context, id uuid.UUID, because string) error { + row, discarded, err := m.store.Discard(ctx, m.orgID, id, because) + if err != nil || !discarded { + return err + } + m.log.InfoContext(ctx, "approval discarded", "request", id, "because", because) + if err := m.tell.Told(ctx, deliveries.ApprovalRequest{ + RunID: row.RunID, RequestID: row.ID, Key: row.Key + ":ask", + Title: "Approval discarded", Body: because, + }, "Discarded: "+because); err != nil { + return fmt.Errorf("tell discard: %w", err) + } + return nil +} + +// ErrNoPath means a request was asked without anyone to put it to. +var ErrNoPath = errors.New("an approval request needs an escalation path") diff --git a/modules/approvals/internal/store/store.go b/modules/approvals/internal/store/store.go index d928bc145..14bd5fd24 100644 --- a/modules/approvals/internal/store/store.go +++ b/modules/approvals/internal/store/store.go @@ -38,6 +38,19 @@ type Request struct { AnsweredAt sql.NullTime `db:"answered_at"` AnsweredBy sql.NullString `db:"answered_by"` AnsweredVia sql.NullString `db:"answered_via"` + // Step is the escalation step the request is at, StepDeadline when that + // step runs out, and Asked a JSON array of the ids of the people it has + // reached so far. + Step int `db:"step"` + StepDeadline sql.NullTime `db:"step_deadline"` + Asked string `db:"asked"` + // ChangeRequest is whether the request went through a change request + // rather than the approvers' channels. + ChangeRequest bool `db:"change_request"` + // Counted is whether the approval was counted toward raising its class. + Counted bool `db:"counted"` + // DiscardedBecause is why an approval no longer stood. + DiscardedBecause sql.NullString `db:"discarded_because"` } // Store is the approvals module's Postgres store. @@ -49,7 +62,8 @@ type Store struct { func New(db *sqlx.DB) *Store { return &Store{db: db} } const requestColumns = `id, org_id, run_id, key, decision_id, kind, digest, connector, action, target_kind, - target_id, level, summary, state, deadline, created_at, answered_at, answered_by, answered_via` + target_id, level, summary, state, deadline, created_at, answered_at, answered_by, answered_via, + step, step_deadline, asked::text AS asked, change_request, counted, discarded_because` // Put records r unless a request already exists for its run and key, and // returns the request that stands either way. @@ -57,12 +71,12 @@ func (s *Store) Put(ctx context.Context, r Request) (Request, error) { var out Request err := s.db.GetContext(ctx, &out, ` INSERT INTO approvals.requests (id, org_id, run_id, key, decision_id, kind, digest, connector, action, - target_kind, target_id, level, summary, state, deadline) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15) + target_kind, target_id, level, summary, state, deadline, step_deadline, asked, change_request) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, CAST($17 AS jsonb), $18) ON CONFLICT (org_id, run_id, key) DO NOTHING RETURNING `+requestColumns, r.ID, r.OrgID, r.RunID, r.Key, r.DecisionID, r.Kind, r.Digest, r.Connector, r.Action, r.TargetKind, - r.TargetID, r.Level, r.Summary, r.State, r.Deadline) + r.TargetID, r.Level, r.Summary, r.State, r.Deadline, r.StepDeadline, r.askedOrEmpty(), r.ChangeRequest) if errors.Is(err, sql.ErrNoRows) { return s.Get(ctx, r.OrgID, r.RunID, r.Key) } @@ -72,6 +86,13 @@ func (s *Store) Put(ctx context.Context, r Request) (Request, error) { return out, nil } +func (r Request) askedOrEmpty() string { + if r.Asked == "" { + return "[]" + } + return r.Asked +} + // Get returns the request for one run's key. func (s *Store) Get(ctx context.Context, orgID string, runID uuid.UUID, key string) (Request, error) { var out Request @@ -104,6 +125,60 @@ func (s *Store) Answer(ctx context.Context, orgID string, id uuid.UUID, state, b return out, nil } +// Escalate moves a pending request to step, which has reached person and runs +// out at until. It returns false when the request is no longer pending, or has +// already moved that far, so a replayed escalation changes nothing. +func (s *Store) Escalate(ctx context.Context, orgID string, id uuid.UUID, step int, person string, until time.Time) (Request, bool, error) { + var out Request + err := s.db.GetContext(ctx, &out, ` + UPDATE approvals.requests + SET step = $3, step_deadline = $4, + asked = CASE WHEN $5::text = '' OR asked @> to_jsonb($5::text) THEN asked ELSE asked || to_jsonb($5::text) END + WHERE org_id = $1 AND id = $2 AND state = 'pending' AND step < $3 + RETURNING `+requestColumns, orgID, id, step, until, person) + if errors.Is(err, sql.ErrNoRows) { + return Request{}, false, nil + } + if err != nil { + return Request{}, false, fmt.Errorf("escalate request: %w", err) + } + return out, true, nil +} + +// Discard withdraws an approval that no longer stands, and only while it has +// not been counted. It returns false when there was nothing to discard. +func (s *Store) Discard(ctx context.Context, orgID string, id uuid.UUID, because string) (Request, bool, error) { + var out Request + err := s.db.GetContext(ctx, &out, ` + UPDATE approvals.requests SET state = 'discarded', discarded_because = $3 + WHERE org_id = $1 AND id = $2 AND state = 'approved' AND NOT counted + RETURNING `+requestColumns, orgID, id, because) + if errors.Is(err, sql.ErrNoRows) { + return Request{}, false, nil + } + if err != nil { + return Request{}, false, fmt.Errorf("discard approval: %w", err) + } + return out, true, nil +} + +// Count marks an approval as counted, once. It returns false when the approval +// was already counted or no longer stands. +func (s *Store) Count(ctx context.Context, orgID string, id uuid.UUID) (Request, bool, error) { + var out Request + err := s.db.GetContext(ctx, &out, ` + UPDATE approvals.requests SET counted = true + WHERE org_id = $1 AND id = $2 AND state = 'approved' AND NOT counted AND kind = 'action' + RETURNING `+requestColumns, orgID, id) + if errors.Is(err, sql.ErrNoRows) { + return Request{}, false, nil + } + if err != nil { + return Request{}, false, fmt.Errorf("count approval: %w", err) + } + return out, true, nil +} + // ByID returns one request. func (s *Store) ByID(ctx context.Context, orgID string, id uuid.UUID) (Request, error) { var out Request diff --git a/modules/runs/escalation_test.go b/modules/runs/escalation_test.go new file mode 100644 index 000000000..f0e85ea23 --- /dev/null +++ b/modules/runs/escalation_test.go @@ -0,0 +1,140 @@ +package runs + +import ( + "context" + "encoding/json/jsontext" + "slices" + "testing" + + "github.com/google/uuid" + + "github.com/samcharles93/archie-core/modules/approvals" + "github.com/samcharles93/archie-core/modules/connectors" + "github.com/samcharles93/archie-core/modules/governance" + "github.com/samcharles93/archie-core/modules/judgement" + "github.com/samcharles93/archie-core/modules/runs/internal/store" +) + +// TestAfterApprovalDiscardsAnApprovalThatNoLongerMatchesTheAction guards the +// rule that an approval covers the action as it was shown: when something else +// changed the target or effect, or judgement got worse, the approval is +// discarded and the action is put to people again, never run on the old one. +func TestAfterApprovalDiscardsAnApprovalThatNoLongerMatchesTheAction(t *testing.T) { + decided := func(level judgement.Level, mutate func(*judgement.Decision)) judgement.Decision { + d := judgement.Decision{Level: level, Action: judgement.Action{ + Target: connectors.Target{Kind: "service", ID: "api", Environment: "prod"}, + Effect: connectors.Effect{Summary: "restart"}, + }} + if mutate != nil { + mutate(&d) + } + return d + } + approved := decided(judgement.ProposeAndWait, nil) + tests := []struct { + name string + fresh judgement.Decision + want afterwards + }{ + {name: "an unchanged action runs on its approval", fresh: decided(judgement.ProposeAndWait, nil), want: runApproval}, + {name: "a worse verdict is put to people again", fresh: decided(judgement.Escalate, nil), want: reaskPerson}, + {name: "a changed target is put to people again", fresh: decided(judgement.ProposeAndWait, func(d *judgement.Decision) { + d.Action.Target.ID = "db" + }), want: reaskPerson}, + {name: "a changed environment is put to people again", fresh: decided(judgement.ProposeAndWait, func(d *judgement.Decision) { + d.Action.Target.Environment = "staging" + }), want: reaskPerson}, + {name: "a changed effect is put to people again", fresh: decided(judgement.ProposeAndWait, func(d *judgement.Decision) { + d.Action.Effect.Summary = "delete" + }), want: reaskPerson}, + {name: "a refusal discards the approval", fresh: decided(judgement.Never, nil), want: refuseAction}, + {name: "a verdict that lets it run needs no approval", fresh: decided(judgement.ActAndTell, nil), want: runFresh}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, why := afterApproval(approved, tt.fresh) + if got != tt.want { + t.Fatalf("afterApproval = %v (%s), want %v", got, why, tt.want) + } + if (got == reaskPerson || got == refuseAction) && why == "" { + t.Fatal("a discarded approval gives no reason") + } + }) + } +} + +// signalAfter is a worker store whose answer arrives once the run has waited +// that many times, as when a person answers during a later escalation step. +type signalAfter struct { + fakeWork + waits int + arrives int +} + +func (s *signalAfter) Signalled(context.Context, uuid.UUID, string) (jsontext.Value, bool, error) { + s.waits++ + if s.arrives > 0 && s.waits >= s.arrives { + return jsontext.Value(`{"approved":true,"by":"Second"}`), true, nil + } + return nil, false, nil +} + +// escalations records what the run asked of approvals. +type escalations struct { + steps []int + expired int +} + +func (e *escalations) Ask(context.Context, uuid.UUID, string, judgement.Decision, []governance.PathStep) (approvals.Request, error) { + return approvals.Request{}, nil +} + +func (e *escalations) Escalate(_ context.Context, _ uuid.UUID, step int, _ governance.PathStep) error { + e.steps = append(e.steps, step) + return nil +} + +func (e *escalations) Expire(context.Context, uuid.UUID) error { e.expired++; return nil } + +func (e *escalations) Accept(context.Context, uuid.UUID) error { return nil } + +func (e *escalations) Discard(context.Context, uuid.UUID, string) error { return nil } + +// TestWaitThroughPathMovesOnAtEachDeadlineAndExpiresAtTheEnd guards the timed +// escalation path: an unanswered request passes to each next step in order, +// the last step's expiry settles it as expired with nothing acted on, and an +// answer at any step stops the escalation there. +func TestWaitThroughPathMovesOnAtEachDeadlineAndExpiresAtTheEnd(t *testing.T) { + // A zero deadline has passed by the time it is waited on. + step := func(person string) governance.PathStep { return governance.PathStep{Person: person} } + tests := []struct { + name string + path []governance.PathStep + arrives int + wantSteps []int + wantExpired int + wantAnswered bool + }{ + {name: "one step with no answer expires without escalating", path: []governance.PathStep{step("a")}, wantExpired: 1}, + {name: "every step runs out and then the request expires", path: []governance.PathStep{step("a"), step("b"), step("c")}, wantSteps: []int{1, 2}, wantExpired: 1}, + {name: "an answer at the second step stops the escalation", path: []governance.PathStep{step("a"), step("b"), step("c")}, arrives: 2, wantSteps: []int{1}, wantAnswered: true}, + {name: "an answer at the first step escalates nowhere", path: []governance.PathStep{step("a"), step("b")}, arrives: 1, wantAnswered: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + work := &signalAfter{arrives: tt.arrives} + rc := newContext(context.Background(), work, store.Run{ID: uuid.New()}, uuid.New(), nil) + ask := &escalations{} + answer, expired, err := waitThroughPath(rc, ask, "k", approvals.Request{ID: uuid.New()}, tt.path) + if err != nil { + t.Fatalf("waitThroughPath: %v", err) + } + if !slices.Equal(ask.steps, tt.wantSteps) || ask.expired != tt.wantExpired { + t.Fatalf("escalated %v and expired %d times, want %v and %d", ask.steps, ask.expired, tt.wantSteps, tt.wantExpired) + } + if expired != (tt.wantExpired > 0) || answer.Approved != tt.wantAnswered { + t.Fatalf("expired = %v, approved = %v", expired, answer.Approved) + } + }) + } +} diff --git a/modules/runs/investigate.go b/modules/runs/investigate.go index d6e2fd0a0..42a9a3a0a 100644 --- a/modules/runs/investigate.go +++ b/modules/runs/investigate.go @@ -8,6 +8,7 @@ import ( "fmt" "regexp" "slices" + "strconv" "strings" "time" @@ -25,10 +26,10 @@ import ( ) // Settings is where a workflow finds what the organisation decided: the agent -// model it runs, and how long an approval request waits for an answer. +// model it runs, and the escalation path an approval request takes. type Settings interface { AgentModel(ctx context.Context) (governance.Model, error) - ApprovalDeadline(ctx context.Context) (time.Duration, error) + EscalationPath(ctx context.Context, needAuthority bool) ([]governance.PathStep, error) RecurrenceThreshold(ctx context.Context) (int, error) AttemptsPerRun(ctx context.Context) (int, error) } @@ -49,11 +50,16 @@ type Judge interface { ForRun(ctx context.Context, runID uuid.UUID) ([]judgement.Decision, error) } -// Approver asks people to allow an action judgement held, and settles a request -// whose deadline passed. +// Approver asks people to allow an action judgement held, moves a request +// that nobody answered to the next step of its path, and settles one whose last +// step ran out. An approval is counted when the action is about to run on it, +// and discarded when the action no longer matches it. type Approver interface { - Ask(ctx context.Context, runID uuid.UUID, key string, d judgement.Decision, deadline time.Time) (approvals.Request, error) + Ask(ctx context.Context, runID uuid.UUID, key string, d judgement.Decision, path []governance.PathStep) (approvals.Request, error) + Escalate(ctx context.Context, id uuid.UUID, step int, to governance.PathStep) error Expire(ctx context.Context, id uuid.UUID) error + Accept(ctx context.Context, id uuid.UUID) error + Discard(ctx context.Context, id uuid.UUID, because string) error } // Notifier tells people what a run found. @@ -78,25 +84,6 @@ func stepBudget(ctx context.Context, settings Settings) (agentloop.Budget, error return budget, nil } -// pinnedDeadline is pinned in the run's first steps, like the model: a run that -// waits keeps the deadline it started with, whatever the setting becomes -// meanwhile. A step's value is encoded, so it travels as the duration's own -// text. -func pinnedDeadline(rc *Context, settings Settings) (time.Duration, error) { - text, err := rc.Step("approval-deadline", func(ctx context.Context, _ string) (string, error) { - d, err := settings.ApprovalDeadline(ctx) - return d.String(), err - }) - if err != nil { - return 0, err - } - deadline, err := time.ParseDuration(text) - if err != nil { - return 0, fmt.Errorf("approval deadline %q: %w", text, err) - } - return deadline, nil -} - // Investigate is the workflow that has an agent look into the event that // started the run and report its findings. The agent model and the tools are // pinned in the run's first steps, so a resumed run asks the same model with @@ -111,10 +98,6 @@ func Investigate(settings Settings, box Toolbox, judge Judge, ask Approver, noti if err != nil { return nil, err } - deadline, err := pinnedDeadline(rc, settings) - if err != nil { - return nil, err - } threshold, err := rc.Step("recurrence-threshold", func(ctx context.Context, _ string) (int, error) { return settings.RecurrenceThreshold(ctx) }) @@ -160,7 +143,7 @@ func Investigate(settings Settings, box Toolbox, judge Judge, ask Approver, noti if err := recordCause(rc, mem, input, res, AgentRequest{Provider: provider, Model: model.Model, ModelInfo: info}); err != nil { return nil, err } - outcomes, err := settleProposals(rc, judge, ask, deadline) + outcomes, err := settleProposals(rc, judge, ask, settings) if err != nil { return nil, err } @@ -286,9 +269,9 @@ type approvalOutcome struct { // settleProposals puts every action judgement held to a person, one at a time, // and runs the ones they allow. Each ask, wait and run is its own step, so a -// resumed run picks up where it waited. A proposal nobody answers in time is -// settled as expired and nothing runs. -func settleProposals(rc *Context, judge Judge, ask Approver, deadline time.Duration) ([]approvalOutcome, error) { +// resumed run picks up where it waited. A proposal nobody answers before the +// last escalation step runs out is settled as expired and nothing runs. +func settleProposals(rc *Context, judge Judge, ask Approver, settings Settings) ([]approvalOutcome, error) { held, err := rc.Step("proposals", func(ctx context.Context, _ string) ([]judgement.Decision, error) { decisions, err := judge.ForRun(ctx, rc.RunID()) if err != nil { @@ -301,7 +284,7 @@ func settleProposals(rc *Context, judge Judge, ask Approver, deadline time.Durat } out := make([]approvalOutcome, 0, len(held)) for _, d := range held { - outcome, err := settleOne(rc, judge, ask, d, deadline) + outcome, err := settleOne(rc, judge, ask, settings, d) if err != nil { return nil, err } @@ -310,37 +293,116 @@ func settleProposals(rc *Context, judge Judge, ask Approver, deadline time.Durat return out, nil } -func settleOne(rc *Context, judge Judge, ask Approver, d judgement.Decision, deadline time.Duration) (approvalOutcome, error) { - out := approvalOutcome{digest: d.Action.Digest, Action: d.Action.Connector + " " + d.Action.Name, Target: d.Action.Target.Kind + ":" + d.Action.Target.ID} - request, err := rc.Step("ask:"+d.Key, func(ctx context.Context, _ string) (approvals.Request, error) { - return ask.Ask(ctx, rc.RunID(), waitKey(d.Key), d, time.Now().Add(deadline)) +// maxReapprovals bounds how many times an action that keeps changing under its +// approval is put to people again. +const maxReapprovals = 2 + +// settleOne gets one action approved and runs it. Before it runs it is judged +// again; an approval whose action has since changed is discarded and the +// changed action is put to people as a new request. +func settleOne(rc *Context, judge Judge, ask Approver, settings Settings, d judgement.Decision) (approvalOutcome, error) { + out := approvalOutcome{digest: d.Action.Digest} + for round := 0; ; round++ { + out.Action, out.Target = d.Action.Connector+" "+d.Action.Name, d.Action.Target.Kind+":"+d.Action.Target.ID + request, answer, err := askThroughPath(rc, ask, settings, d) + if err != nil { + return out, err + } + out.RequestID, out.State = request.ID, request.State + if request.State != approvals.StateApproved { + if request.State == approvals.StateDenied { + out.By = answer.By + } + return out, nil + } + out.By = answer.By + fresh, err := rc.Step("judge-again:"+d.Key, func(ctx context.Context, key string) (judgement.Decision, error) { + return judge.Assess(ctx, judgement.Request{ + RunID: rc.RunID(), Key: key, Connector: d.Action.Connector, Action: d.Action.Name, Input: d.Action.Input, + Workflow: rc.Workflow(), + }) + }) + if err != nil { + return out, err + } + next, why := afterApproval(d, fresh) + switch next { + case reaskPerson, refuseAction: + if _, err := rc.Step("discard:"+d.Key, func(ctx context.Context, _ string) (bool, error) { + return true, ask.Discard(ctx, request.ID, why) + }); err != nil { + return out, err + } + out.State = approvals.StateDiscarded + if next == refuseAction || round == maxReapprovals { + out.Error = why + return out, nil + } + d = fresh + continue + case runFresh: + // The approval isn't needed and isn't counted. + out.Result, out.Error, out.Unknown = runAction(rc, judge, fresh, nil) + default: + if _, err := rc.Step("count:"+d.Key, func(ctx context.Context, _ string) (bool, error) { + return true, ask.Accept(ctx, request.ID) + }); err != nil { + return out, err + } + out.Result, out.Error, out.Unknown = runAction(rc, judge, d, &answer) + } + out.State = approvals.StateApproved + return out, nil + } +} + +// askThroughPath puts d to the first step of its escalation path and waits. +// The path is pinned when it is first asked, so a request that waits keeps the +// deadlines it started with whatever the settings become meanwhile. It returns +// the request as it stood when it settled, and the answer if there was one. +func askThroughPath(rc *Context, ask Approver, settings Settings, d judgement.Decision) (approvals.Request, approvals.Answer, error) { + path, err := rc.Step("path:"+d.Key, func(ctx context.Context, _ string) ([]governance.PathStep, error) { + return settings.EscalationPath(ctx, d.Level == judgement.Escalate) }) if err != nil { - return out, err + return approvals.Request{}, approvals.Answer{}, err } - out.RequestID, out.State = request.ID, request.State - if request.State != approvals.StatePending { - return out, nil - } - answer, expired, err := waitForAnswer(rc, ask, d.Key, request, deadline) + request, err := rc.Step("ask:"+d.Key, func(ctx context.Context, _ string) (approvals.Request, error) { + return ask.Ask(ctx, rc.RunID(), waitKey(d.Key), d, path) + }) if err != nil { - return out, err + return request, approvals.Answer{}, err } - if expired { - out.State = approvals.StateExpired - return out, nil + if request.State != approvals.StatePending && request.State != approvals.StateApproved { + return request, approvals.Answer{}, nil } - if !answer.Approved { - out.State, out.By = approvals.StateDenied, answer.By - return out, nil + // A request already answered when it was asked again still has its answer + // waiting under the run's key, so the wait returns it at once. + answer, expired, err := waitThroughPath(rc, ask, d.Key, request, path) + switch { + case err != nil: + return request, answer, err + case expired: + request.State = approvals.StateExpired + case answer.Approved: + request.State = approvals.StateApproved + default: + request.State = approvals.StateDenied } - out.By = answer.By - // The action's own failure is part of what the answer led to, and its step - // has recorded it, so the investigation still reports. - run, runErr := runApproved(rc, judge, d, answer) - out.State, out.Result, out.Error = approvals.StateApproved, run, errText(runErr) - out.Unknown = errors.Is(runErr, ErrOutcomeUnknown) - return out, nil + return request, answer, nil +} + +// runAction runs the action judged as d, on approval when there is one, and +// returns what came of it. The action's own failure is part of what the answer +// led to, and its step has recorded it, so the investigation still reports. +func runAction(rc *Context, judge Judge, d judgement.Decision, approval *approvals.Answer) (result, failure string, unknown bool) { + var runErr error + if approval == nil { + result, runErr = runUnattended(rc, judge, d) + } else { + result, runErr = runApproved(rc, judge, d, *approval) + } + return result, errText(runErr), errors.Is(runErr, ErrOutcomeUnknown) } // unknownOutcome records a connector's unconfirmed action as a step whose @@ -360,54 +422,92 @@ func errText(err error) string { return err.Error() } -// waitForAnswer waits for the person's answer. When the deadline passes with -// none, the request is settled as expired and the caller runs nothing. -func waitForAnswer(rc *Context, ask Approver, key string, request approvals.Request, deadline time.Duration) (approvals.Answer, bool, error) { - payload, err := rc.Wait(waitKey(key), deadline) - if errors.Is(err, ErrWaitTimedOut) { - if _, eerr := rc.Step("expire:"+key, func(ctx context.Context, _ string) (bool, error) { - return true, ask.Expire(ctx, request.ID) - }); eerr != nil { - return approvals.Answer{}, true, eerr +// waitThroughPath waits for an answer through each step of path in turn. A step +// that runs out with none passes the request to the next person; the last one +// settles it as expired and the caller runs nothing. Every wait is a durable +// run wait, so a restart resumes where each deadline stands. +func waitThroughPath(rc *Context, ask Approver, key string, request approvals.Request, path []governance.PathStep) (approvals.Answer, bool, error) { + for i, step := range path { + if i > 0 { + if _, err := rc.Step("escalate:"+key+":"+strconv.Itoa(i), func(ctx context.Context, _ string) (bool, error) { + return true, ask.Escalate(ctx, request.ID, i, step) + }); err != nil { + return approvals.Answer{}, false, err + } } - return approvals.Answer{}, true, nil + payload, err := rc.Wait(waitKey(key), step.Deadline()) + if errors.Is(err, ErrWaitTimedOut) { + continue + } + if err != nil { + return approvals.Answer{}, false, err + } + var answer approvals.Answer + if err := json.Unmarshal(payload, &answer); err != nil { + return approvals.Answer{}, false, fmt.Errorf("decode approval answer: %w", err) + } + return answer, false, nil } - if err != nil { - return approvals.Answer{}, false, err + if _, err := rc.Step("expire:"+key, func(ctx context.Context, _ string) (bool, error) { + return true, ask.Expire(ctx, request.ID) + }); err != nil { + return approvals.Answer{}, true, err } - var answer approvals.Answer - if err := json.Unmarshal(payload, &answer); err != nil { - return approvals.Answer{}, false, fmt.Errorf("decode approval answer: %w", err) - } - return answer, false, nil + return approvals.Answer{}, true, nil } -// runApproved judges the action again and runs it, once. An approval covers -// the action the person was shown, so a judgement that came back worse than -// it was cannot be run on that approval. -func runApproved(rc *Context, judge Judge, d judgement.Decision, answer approvals.Answer) (string, error) { - fresh, err := rc.Step("judge-again:"+d.Key, func(ctx context.Context, key string) (judgement.Decision, error) { - return judge.Assess(ctx, judgement.Request{ - RunID: rc.RunID(), Key: key, Connector: d.Action.Connector, Action: d.Action.Name, Input: d.Action.Input, - Workflow: rc.Workflow(), - }) - }) - if err != nil { - return "", err +// What an approval allows once the action is judged again. +type afterwards int + +const ( + // runApproval runs the action that was approved. + runApproval afterwards = iota + // runFresh runs the action on its new judgement, which lets it run without + // anyone's approval. + runFresh + // reaskPerson discards the approval and puts the changed action to people. + reaskPerson + // refuseAction discards the approval: the action is now refused. + refuseAction +) + +// afterApproval says what an approval of approved means now that the action was +// judged again as fresh. An approval covers the action as it was shown, so one +// whose verdict got worse, or whose target or effect changed, no longer +// stands. The target and effect are what the connector resolves the action to +// now, which is what judgement has to tell that something else changed them. +func afterApproval(approved, fresh judgement.Decision) (afterwards, string) { + switch { + case fresh.Level == judgement.Never: + return refuseAction, "refused after approval: " + fresh.Reason + case fresh.Level.Runs(): + return runFresh, "" + case fresh.Action.Target != approved.Action.Target: + return reaskPerson, "the target changed since it was approved" + case fresh.Action.Effect != approved.Action.Effect: + return reaskPerson, "the effect changed since it was approved" + case fresh.Level > approved.Level: + return reaskPerson, "judgement got worse since it was approved: now " + fresh.Level.String() } + return runApproval, "" +} + +// runUnattended runs an action its fresh judgement lets run without approval, +// once. +func runUnattended(rc *Context, judge Judge, d judgement.Decision) (string, error) { return rc.Step("run:"+d.Key, func(ctx context.Context, key string) (string, error) { - switch { - case fresh.Level == judgement.Never: - return "", fmt.Errorf("refused after approval: %s", fresh.Reason) - case fresh.Level.Runs(): - if err := begun(ctx, rc, key); err != nil { - return "", err - } - out, err := judge.Execute(ctx, fresh, d.Action.Input, key) - return out, unknownOutcome(err) - case fresh.Level > d.Level: - return "", fmt.Errorf("judgement got worse after approval: now %s", fresh.Level) + if err := begun(ctx, rc, key); err != nil { + return "", err } + out, err := judge.Execute(ctx, d, d.Action.Input, key) + return out, unknownOutcome(err) + }) +} + +// runApproved runs the action a person approved, once. judgement runs only the +// action that was judged, whatever input comes with it. +func runApproved(rc *Context, judge Judge, d judgement.Decision, answer approvals.Answer) (string, error) { + return rc.Step("run:"+d.Key, func(ctx context.Context, key string) (string, error) { if err := begun(ctx, rc, key); err != nil { return "", err } diff --git a/modules/runs/started_test.go b/modules/runs/started_test.go index b56b0a01f..397a08783 100644 --- a/modules/runs/started_test.go +++ b/modules/runs/started_test.go @@ -55,7 +55,7 @@ func TestActionStartedBeforeItsResultIsNeverRunAgain(t *testing.T) { return []store.Step{{RunID: runID, Seq: markerBase, Name: startedPrefix + key, Outcome: store.OutcomeDone, Output: jsontext.Value("true")}} } agentKey := runID.String() + ":investigate:3" - approvedKey := runID.String() + ":1" + approvedKey := runID.String() + ":0" tests := []struct { name string