feat(approvals,runs): an unanswered approval escalates step by step and then expires, a discarded approval never counts, and a change-request seam routes and withdraws (#1985)
This commit is contained in:
7 files changed
+689
-160
No files matched your search
+136
-45
@@ -10,6 +10,7 @@ package approvals
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json/jsontext"
|
||||
json "encoding/json/v2"
|
||||
"errors"
|
||||
@@ -31,12 +32,14 @@ import (
|
||||
"github.com/samcharles93/archie-core/modules/judgement"
|
||||
)
|
||||
|
||||
// The states a request can be in. A settled request never changes again.
|
||||
// The states a request can be in. A settled request never changes again,
|
||||
// except that an approval the action no longer matched is discarded.
|
||||
const (
|
||||
StatePending = "pending"
|
||||
StateApproved = "approved"
|
||||
StateDenied = "denied"
|
||||
StateExpired = "expired"
|
||||
StatePending = "pending"
|
||||
StateApproved = "approved"
|
||||
StateDenied = "denied"
|
||||
StateExpired = "expired"
|
||||
StateDiscarded = "discarded"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -50,6 +53,12 @@ var (
|
||||
ErrUnverified = errors.New("that identity isn't held by any person, so its answer is refused")
|
||||
// ErrNotApprover means the person may not answer approvals for actions.
|
||||
ErrNotApprover = errors.New("that person isn't the default owner, on the escalation path or an authority, so their answer is refused")
|
||||
// ErrNotAsked means the request hasn't yet reached that person on its
|
||||
// escalation path.
|
||||
ErrNotAsked = errors.New("that person hasn't been asked yet, so their answer is refused")
|
||||
// ErrNeedsAuthority means the action needs a person with authority, and
|
||||
// the answer came from one without.
|
||||
ErrNeedsAuthority = errors.New("that action needs a person with authority, so their answer is refused")
|
||||
// ErrNoAuthority means the answer to a raise came from a person without
|
||||
// authority over policy and ceilings.
|
||||
ErrNoAuthority = errors.New("that person holds no authority over policy and ceilings, so their answer to a raise is refused")
|
||||
@@ -79,10 +88,19 @@ type Request struct {
|
||||
Level string
|
||||
Summary string
|
||||
State string
|
||||
Deadline time.Time
|
||||
AnsweredAt time.Time
|
||||
AnsweredBy string
|
||||
AnsweredVia string
|
||||
// Deadline is when the request runs out for good: the end of the last
|
||||
// escalation step.
|
||||
Deadline time.Time
|
||||
// Step is the escalation step the request is at, and StepDeadline when it
|
||||
// moves on.
|
||||
Step int
|
||||
StepDeadline time.Time
|
||||
// ChangeRequest is whether the request went through a change request
|
||||
// rather than the approvers' channels.
|
||||
ChangeRequest bool
|
||||
AnsweredAt time.Time
|
||||
AnsweredBy string
|
||||
AnsweredVia string
|
||||
}
|
||||
|
||||
// Answer is what the run wakes with.
|
||||
@@ -110,6 +128,10 @@ type View struct {
|
||||
Summary string `json:"summary"`
|
||||
State string `json:"state"`
|
||||
Deadline time.Time `json:"deadline"`
|
||||
// Step is the escalation step the request is at, and StepDeadline when it
|
||||
// moves to the next.
|
||||
Step int `json:"step"`
|
||||
StepDeadline time.Time `json:"step_deadline,omitzero"`
|
||||
// DecisionID is the judged action the request is about.
|
||||
DecisionID uuid.UUID `json:"decision_id"`
|
||||
RequestedAt time.Time `json:"requested_at"`
|
||||
@@ -123,7 +145,7 @@ func (r Request) View() View {
|
||||
return View{
|
||||
ID: r.ID, Kind: r.Kind, RunID: r.RunID, Action: r.Connector + " " + r.Action,
|
||||
Target: r.TargetKind + ":" + r.TargetID, Level: r.Level,
|
||||
Summary: r.Summary, State: r.State, Deadline: r.Deadline,
|
||||
Summary: r.Summary, State: r.State, Deadline: r.Deadline, Step: r.Step, StepDeadline: r.StepDeadline,
|
||||
DecisionID: r.DecisionID, RequestedAt: r.RequestedAt,
|
||||
AnsweredBy: r.AnsweredBy, AnsweredVia: r.AnsweredVia, AnsweredAt: r.AnsweredAt,
|
||||
}
|
||||
@@ -172,6 +194,7 @@ type Module struct {
|
||||
classes Classes
|
||||
policy Policy
|
||||
people People
|
||||
changes ChangeRequests
|
||||
log *slog.Logger
|
||||
now func() time.Time
|
||||
}
|
||||
@@ -183,20 +206,44 @@ type Module struct {
|
||||
func NewModule(db *sqlx.DB, log *slog.Logger, orgID string, tell Delivery, wake Signaller, classes Classes, policy Policy, people People) *Module {
|
||||
return &Module{
|
||||
orgID: orgID, store: store.New(db), tell: tell, wake: wake, classes: classes, policy: policy, people: people,
|
||||
log: log, now: time.Now,
|
||||
changes: NoChangeRequests{}, log: log, now: time.Now,
|
||||
}
|
||||
}
|
||||
|
||||
// Ask records a request for d and tells people about it. Asking again for the
|
||||
// same run and key returns the request that stands, so a replayed step asks
|
||||
// once.
|
||||
func (m *Module) Ask(ctx context.Context, runID uuid.UUID, key string, d judgement.Decision, deadline time.Time) (Request, error) {
|
||||
// Ask records a request for d and puts it to the first step of path. It runs
|
||||
// out for good when the last step does. Asking again for the same run and key
|
||||
// returns the request that stands, so a replayed step asks once. An action the
|
||||
// change-request rules pick goes through a change request, and every other one
|
||||
// through the approvers' channels.
|
||||
func (m *Module) Ask(ctx context.Context, runID uuid.UUID, key string, d judgement.Decision, path []governance.PathStep) (Request, error) {
|
||||
if len(path) == 0 {
|
||||
return Request{}, ErrNoPath
|
||||
}
|
||||
changeRequest, err := m.changes.Needed(ctx, d)
|
||||
if err != nil {
|
||||
return Request{}, fmt.Errorf("route approval: %w", err)
|
||||
}
|
||||
now := m.now().UTC()
|
||||
var total time.Duration
|
||||
for _, st := range path {
|
||||
total += st.Deadline()
|
||||
}
|
||||
asked := []string{}
|
||||
if path[0].Person != "" {
|
||||
asked = append(asked, path[0].Person)
|
||||
}
|
||||
askedJSON, err := json.Marshal(asked)
|
||||
if err != nil {
|
||||
return Request{}, fmt.Errorf("encode asked: %w", err)
|
||||
}
|
||||
row, err := m.store.Put(ctx, store.Request{
|
||||
ID: uuid.New(), OrgID: m.orgID, RunID: runID, Key: key,
|
||||
DecisionID: d.ID, Kind: KindAction, Digest: d.Action.Digest, Level: d.Level.String(),
|
||||
Connector: d.Action.Connector, Action: d.Action.Name,
|
||||
TargetKind: d.Action.Target.Kind, TargetID: d.Action.Target.ID,
|
||||
Summary: Summary(d), State: StatePending, Deadline: deadline.UTC(),
|
||||
Summary: Summary(d), State: StatePending, Deadline: now.Add(total),
|
||||
StepDeadline: sql.NullTime{Time: now.Add(path[0].Deadline()), Valid: true}, Asked: string(askedJSON),
|
||||
ChangeRequest: changeRequest,
|
||||
})
|
||||
if err != nil {
|
||||
return Request{}, err
|
||||
@@ -205,10 +252,16 @@ func (m *Module) Ask(ctx context.Context, runID uuid.UUID, key string, d judgeme
|
||||
if row.State != StatePending {
|
||||
return r, nil
|
||||
}
|
||||
if row.ChangeRequest {
|
||||
if err := m.changes.Open(ctx, r); err != nil {
|
||||
return Request{}, fmt.Errorf("open change request: %w", err)
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
if err := m.tell.Ask(ctx, deliveries.ApprovalRequest{
|
||||
RunID: r.RunID, RequestID: r.ID, Key: r.Key + ":ask",
|
||||
Title: "Approval needed: " + d.Action.Connector + " " + d.Action.Name,
|
||||
Body: r.Summary, ExpiresAt: r.Deadline,
|
||||
Body: r.Summary, ExpiresAt: r.StepDeadline,
|
||||
}); err != nil {
|
||||
return Request{}, fmt.Errorf("send approval request: %w", err)
|
||||
}
|
||||
@@ -244,20 +297,20 @@ func (m *Module) answerAs(ctx context.Context, id uuid.UUID, by, via string, app
|
||||
if err := m.wakeWith(ctx, row, answer); err != nil {
|
||||
return Request{}, err
|
||||
}
|
||||
// The run has what it waited for; the answer's bearing on the class comes
|
||||
// after, so a failure here is reported without holding the run.
|
||||
if err := m.learn(ctx, row, answer); err != nil {
|
||||
return Request{}, err
|
||||
// The run has what it waited for. A denial tightens the class now. An
|
||||
// approval counts only once the action is about to run on it, so one that
|
||||
// is discarded first counts for nothing.
|
||||
if !answer.Approved {
|
||||
if err := m.classes.Denied(ctx, row.DecisionID); err != nil {
|
||||
return Request{}, err
|
||||
}
|
||||
}
|
||||
return requestOf(row, judgement.Decision{}), nil
|
||||
}
|
||||
|
||||
// learn counts an answer to an action into its class. An approval that
|
||||
// learn counts an approval of an action into its class. An approval that
|
||||
// completes a run of them earns a raise, which is put to a person.
|
||||
func (m *Module) learn(ctx context.Context, row store.Request, a Answer) error {
|
||||
if !a.Approved {
|
||||
return m.classes.Denied(ctx, row.DecisionID)
|
||||
}
|
||||
need, err := m.policy.RaiseAfter(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read raise count: %w", err)
|
||||
@@ -342,7 +395,7 @@ func (m *Module) Settle(ctx context.Context, id uuid.UUID, identity, via string,
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
who, err := m.verify(ctx, identity, row.Kind)
|
||||
who, err := m.verify(ctx, identity, row)
|
||||
if refusal(err) {
|
||||
m.log.WarnContext(ctx, "approval answer refused", "request", id, "identity", identity, "reason", err.Error())
|
||||
return false, m.store.Refuse(ctx, store.Refusal{
|
||||
@@ -362,31 +415,57 @@ func (m *Module) Settle(ctx context.Context, id uuid.UUID, identity, via string,
|
||||
return false, err
|
||||
}
|
||||
|
||||
// verify says whether identity may answer a request of kind. An action is
|
||||
// answered by anyone who answers for actions; a raise changes policy, so only
|
||||
// a person with authority answers it.
|
||||
func (m *Module) verify(ctx context.Context, identity, kind string) (governance.Verified, error) {
|
||||
// verify says whether identity may answer request row. An action is answered
|
||||
// by someone who answers for actions and whom the request has reached on its
|
||||
// escalation path, or who holds authority, and an action at the escalate level
|
||||
// needs authority. A raise changes policy, so only a person with authority
|
||||
// answers it.
|
||||
func (m *Module) verify(ctx context.Context, identity string, row store.Request) (governance.Verified, error) {
|
||||
who, ok, err := m.people.Verify(ctx, identity)
|
||||
switch {
|
||||
case err != nil:
|
||||
if err != nil {
|
||||
return governance.Verified{}, fmt.Errorf("verify %s: %w", identity, err)
|
||||
case !ok:
|
||||
return governance.Verified{}, ErrUnverified
|
||||
case kind == KindRaise && !who.Authority:
|
||||
return governance.Verified{}, ErrNoAuthority
|
||||
case kind != KindRaise && !who.Approver:
|
||||
return governance.Verified{}, ErrNotApprover
|
||||
}
|
||||
return who, nil
|
||||
if !ok {
|
||||
return governance.Verified{}, ErrUnverified
|
||||
}
|
||||
return who, mayAnswer(who, row)
|
||||
}
|
||||
|
||||
func mayAnswer(who governance.Verified, row store.Request) error {
|
||||
switch {
|
||||
case row.Kind == KindRaise && !who.Authority:
|
||||
return ErrNoAuthority
|
||||
case row.Kind == KindRaise:
|
||||
return nil
|
||||
case !who.Approver:
|
||||
return ErrNotApprover
|
||||
case row.Level == judgement.Escalate.String() && !who.Authority:
|
||||
return ErrNeedsAuthority
|
||||
case !who.Authority && !slices.Contains(askedOf(row), who.ID):
|
||||
return ErrNotAsked
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// askedOf is the ids of the people the request has reached.
|
||||
func askedOf(row store.Request) []string {
|
||||
var ids []string
|
||||
if err := json.Unmarshal([]byte(row.Asked), &ids); err != nil {
|
||||
return nil
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
// refusal reports whether err is a refused answer rather than a failure.
|
||||
func refusal(err error) bool {
|
||||
return errors.Is(err, ErrUnverified) || errors.Is(err, ErrNotApprover) || errors.Is(err, ErrNoAuthority)
|
||||
return errors.Is(err, ErrUnverified) || errors.Is(err, ErrNotApprover) || errors.Is(err, ErrNoAuthority) ||
|
||||
errors.Is(err, ErrNotAsked) || errors.Is(err, ErrNeedsAuthority)
|
||||
}
|
||||
|
||||
// Expire settles a request whose deadline passed, so the trail shows why it
|
||||
// never ran.
|
||||
// Expire settles a request whose last escalation step ran out, so the trail
|
||||
// shows why it never ran. Nothing is acted on, an open change request is
|
||||
// withdrawn, and the report goes to every channel. Expiry says nothing about
|
||||
// the action's class, so it is not tightened.
|
||||
func (m *Module) Expire(ctx context.Context, id uuid.UUID) error {
|
||||
row, err := m.store.Answer(ctx, m.orgID, id, StateExpired, "deadline", "clock", m.now().UTC())
|
||||
if err != nil {
|
||||
@@ -399,18 +478,26 @@ func (m *Module) Expire(ctx context.Context, id uuid.UUID) error {
|
||||
return nil
|
||||
}
|
||||
m.log.InfoContext(ctx, "approval request expired", "request", row.ID, "run", row.RunID)
|
||||
withdrawn := ""
|
||||
if row.ChangeRequest {
|
||||
if err := m.changes.Withdraw(ctx, requestOf(row, judgement.Decision{})); err != nil {
|
||||
return fmt.Errorf("withdraw change request: %w", err)
|
||||
}
|
||||
withdrawn = " Its change request was withdrawn."
|
||||
}
|
||||
if err := m.tell.Told(ctx, deliveries.ApprovalRequest{
|
||||
RunID: row.RunID, RequestID: row.ID, Key: row.Key + ":ask",
|
||||
Title: "Approval expired", Body: "The request ran out of time, so nothing was run.",
|
||||
Title: "Approval expired", Body: "The request ran out of time, so nothing was run." + withdrawn,
|
||||
}, "Expired without an answer."); err != nil {
|
||||
return err
|
||||
}
|
||||
// The escalation goes to every channel. Its key is stable, so an Expire
|
||||
// The report goes to every channel. Its key is stable, so an Expire
|
||||
// retried after a failed delivery sends it once.
|
||||
return m.tell.Notify(ctx, deliveries.Notification{
|
||||
RunID: row.RunID, Key: row.Key + ":expired",
|
||||
Title: "Approval timed out: " + row.Connector + " " + row.Action,
|
||||
Body: "Nobody answered before the deadline, so " + row.Connector + " " + row.Action + " did not run. " + row.Summary,
|
||||
Body: "Nobody answered before the last escalation step ran out, so " + row.Connector + " " + row.Action +
|
||||
" did not run." + withdrawn + " " + row.Summary,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -619,6 +706,10 @@ func requestOf(row store.Request, d judgement.Decision) Request {
|
||||
ID: row.ID, Kind: row.Kind, RunID: row.RunID, Key: row.Key, Decision: d, DecisionID: row.DecisionID, RequestedAt: row.CreatedAt,
|
||||
Connector: row.Connector, Action: row.Action, TargetKind: row.TargetKind, TargetID: row.TargetID,
|
||||
Level: row.Level, Summary: row.Summary, State: row.State, Deadline: row.Deadline,
|
||||
Step: row.Step, ChangeRequest: row.ChangeRequest,
|
||||
}
|
||||
if row.StepDeadline.Valid {
|
||||
r.StepDeadline = row.StepDeadline.Time
|
||||
}
|
||||
if row.AnsweredAt.Valid {
|
||||
r.AnsweredAt = row.AnsweredAt.Time
|
||||
|
||||
@@ -3,11 +3,13 @@ package approvals
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/modules/approvals/internal/store"
|
||||
"github.com/samcharles93/archie-core/modules/governance"
|
||||
"github.com/samcharles93/archie-core/modules/judgement"
|
||||
)
|
||||
|
||||
type fakePeople map[string]governance.Verified
|
||||
@@ -19,31 +21,42 @@ func (f fakePeople) Verify(_ context.Context, identity string) (governance.Verif
|
||||
|
||||
// TestVerifyAcceptsAnAnswerOnlyFromAPersonWhoMayGiveIt guards the rule that an
|
||||
// approval counts only from a verified identity, on every channel: an identity
|
||||
// no person holds, or a person without the standing for that kind of request,
|
||||
// is refused rather than settling it.
|
||||
// no person holds, a person without the standing for that kind of request, a
|
||||
// person the request hasn't reached on its escalation path, or one without the
|
||||
// authority the action's level needs, is refused rather than settling it.
|
||||
func TestVerifyAcceptsAnAnswerOnlyFromAPersonWhoMayGiveIt(t *testing.T) {
|
||||
people := fakePeople{
|
||||
"telegram:1001": {Name: "Owner", Approver: true},
|
||||
"telegram:1002": {Name: "Authority", Approver: true, Authority: true},
|
||||
"telegram:1003": {Name: "Bystander"},
|
||||
"telegram:1001": {ID: "owner", Name: "Owner", Approver: true},
|
||||
"telegram:1002": {ID: "lead", Name: "Authority", Approver: true, Authority: true},
|
||||
"telegram:1003": {ID: "bystander", Name: "Bystander"},
|
||||
"telegram:1004": {ID: "second", Name: "Second", Approver: true},
|
||||
}
|
||||
action := func(level string, asked ...string) store.Request {
|
||||
return store.Request{Kind: KindAction, Level: level, Asked: `["` + strings.Join(asked, `","`) + `"]`}
|
||||
}
|
||||
propose, escalate := judgement.ProposeAndWait.String(), judgement.Escalate.String()
|
||||
tests := []struct {
|
||||
name string
|
||||
identity string
|
||||
kind string
|
||||
row store.Request
|
||||
wantErr error
|
||||
}{
|
||||
{name: "an approver answers an action", identity: "telegram:1001", kind: KindAction},
|
||||
{name: "an unmapped identity is refused", identity: "telegram:9999", kind: KindAction, wantErr: ErrUnverified},
|
||||
{name: "an unmapped identity can't answer a raise", identity: "telegram:9999", kind: KindRaise, wantErr: ErrUnverified},
|
||||
{name: "a person who isn't an approver is refused an action", identity: "telegram:1003", kind: KindAction, wantErr: ErrNotApprover},
|
||||
{name: "an approver without authority is refused a raise", identity: "telegram:1001", kind: KindRaise, wantErr: ErrNoAuthority},
|
||||
{name: "authority answers a raise", identity: "telegram:1002", kind: KindRaise},
|
||||
{name: "an approver the request has reached answers", identity: "telegram:1001", row: action(propose, "owner")},
|
||||
{name: "an unmapped identity is refused", identity: "telegram:9999", row: action(propose, "owner"), wantErr: ErrUnverified},
|
||||
{name: "an unmapped identity can't answer a raise", identity: "telegram:9999", row: store.Request{Kind: KindRaise}, wantErr: ErrUnverified},
|
||||
{name: "a person who isn't an approver is refused an action", identity: "telegram:1003", row: action(propose, "owner"), wantErr: ErrNotApprover},
|
||||
{name: "an approver without authority is refused a raise", identity: "telegram:1001", row: store.Request{Kind: KindRaise}, wantErr: ErrNoAuthority},
|
||||
{name: "authority answers a raise", identity: "telegram:1002", row: store.Request{Kind: KindRaise}},
|
||||
{name: "a person on the path is refused until the request reaches them", identity: "telegram:1004", row: action(propose, "owner"), wantErr: ErrNotAsked},
|
||||
{name: "a person on the path answers once the request reaches them", identity: "telegram:1004", row: action(propose, "owner", "second")},
|
||||
{name: "authority answers before it is asked", identity: "telegram:1002", row: action(propose, "owner")},
|
||||
{name: "an action at escalate needs authority even from someone asked", identity: "telegram:1001", row: action(escalate, "owner"), wantErr: ErrNeedsAuthority},
|
||||
{name: "authority answers an action at escalate", identity: "telegram:1002", row: action(escalate, "owner")},
|
||||
}
|
||||
m := &Module{people: people}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
_, err := m.verify(context.Background(), tt.identity, tt.kind)
|
||||
_, err := m.verify(context.Background(), tt.identity, tt.row)
|
||||
if !errors.Is(err, tt.wantErr) {
|
||||
t.Fatalf("err = %v, want %v", err, tt.wantErr)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
package approvals
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/samcharles93/archie-core/modules/deliveries"
|
||||
"github.com/samcharles93/archie-core/modules/governance"
|
||||
"github.com/samcharles93/archie-core/modules/judgement"
|
||||
)
|
||||
|
||||
// ChangeRequests is the seam between approval routing and a change-request
|
||||
// connector. Routing asks one question of an action, whether it needs a change
|
||||
// request; every action that doesn't goes to the approvers' channels. The
|
||||
// rules that answer it are deterministic and run after the guards.
|
||||
type ChangeRequests interface {
|
||||
// Needed reports whether d goes through a change request.
|
||||
Needed(ctx context.Context, d judgement.Decision) (bool, error)
|
||||
// Open raises the change request for r. It may be called again for the same
|
||||
// request and must not open a second one.
|
||||
Open(ctx context.Context, r Request) error
|
||||
// Withdraw withdraws r's change request if it is still open.
|
||||
Withdraw(ctx context.Context, r Request) error
|
||||
}
|
||||
|
||||
// NoChangeRequests is the routing while no change-request connector exists:
|
||||
// no action needs one, so there is nothing to open or withdraw.
|
||||
type NoChangeRequests struct{}
|
||||
|
||||
// Needed reports false.
|
||||
func (NoChangeRequests) Needed(context.Context, judgement.Decision) (bool, error) { return false, nil }
|
||||
|
||||
// Open does nothing.
|
||||
func (NoChangeRequests) Open(context.Context, Request) error { return nil }
|
||||
|
||||
// Withdraw does nothing.
|
||||
func (NoChangeRequests) Withdraw(context.Context, Request) error { return nil }
|
||||
|
||||
// UseChangeRequests makes the module route approvals through c.
|
||||
func (m *Module) UseChangeRequests(c ChangeRequests) { m.changes = c }
|
||||
|
||||
// Escalate passes a pending request to step n of its path, telling every
|
||||
// channel who it has reached and until when. The step's deadline runs from
|
||||
// now. A request that has since been answered is left alone.
|
||||
func (m *Module) Escalate(ctx context.Context, id uuid.UUID, n int, to governance.PathStep) error {
|
||||
row, err := m.store.ByID(ctx, m.orgID, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if row.State != StatePending {
|
||||
return nil
|
||||
}
|
||||
until := m.now().UTC().Add(to.Deadline())
|
||||
who := "the next person on the escalation path"
|
||||
if to.Name != "" {
|
||||
who = to.Name
|
||||
}
|
||||
// The notice goes first and is idempotent by key, so an escalation retried
|
||||
// after the move failed still tells people once.
|
||||
if err := m.tell.Notify(ctx, deliveries.Notification{
|
||||
RunID: row.RunID, Key: row.Key + ":escalated:" + strconv.Itoa(n),
|
||||
Title: "Approval escalated: " + row.Connector + " " + row.Action,
|
||||
Body: "Nobody answered in time, so it now goes to " + who + " and waits until " + until.Format("2006-01-02 15:04 UTC") + ". " + row.Summary,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, _, err := m.store.Escalate(ctx, m.orgID, id, n, to.Person, until); err != nil {
|
||||
return err
|
||||
}
|
||||
m.log.InfoContext(ctx, "approval request escalated", "request", id, "step", n)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Accept counts an approval toward raising its action class, once, when the
|
||||
// action it covers is about to run. An approval that is discarded first never
|
||||
// counts.
|
||||
func (m *Module) Accept(ctx context.Context, id uuid.UUID) error {
|
||||
row, counted, err := m.store.Count(ctx, m.orgID, id)
|
||||
if err != nil || !counted {
|
||||
return err
|
||||
}
|
||||
return m.learn(ctx, row, Answer{
|
||||
RequestID: row.ID, Approved: true, By: row.AnsweredBy.String, Via: row.AnsweredVia.String, At: row.AnsweredAt.Time,
|
||||
})
|
||||
}
|
||||
|
||||
// Discard withdraws an approval whose action changed before it could run, and
|
||||
// says why. A discarded approval doesn't count toward a raise, and a request
|
||||
// for the changed action is a new one.
|
||||
func (m *Module) Discard(ctx context.Context, id uuid.UUID, because string) error {
|
||||
row, discarded, err := m.store.Discard(ctx, m.orgID, id, because)
|
||||
if err != nil || !discarded {
|
||||
return err
|
||||
}
|
||||
m.log.InfoContext(ctx, "approval discarded", "request", id, "because", because)
|
||||
if err := m.tell.Told(ctx, deliveries.ApprovalRequest{
|
||||
RunID: row.RunID, RequestID: row.ID, Key: row.Key + ":ask",
|
||||
Title: "Approval discarded", Body: because,
|
||||
}, "Discarded: "+because); err != nil {
|
||||
return fmt.Errorf("tell discard: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ErrNoPath means a request was asked without anyone to put it to.
|
||||
var ErrNoPath = errors.New("an approval request needs an escalation path")
|
||||
@@ -38,6 +38,19 @@ type Request struct {
|
||||
AnsweredAt sql.NullTime `db:"answered_at"`
|
||||
AnsweredBy sql.NullString `db:"answered_by"`
|
||||
AnsweredVia sql.NullString `db:"answered_via"`
|
||||
// Step is the escalation step the request is at, StepDeadline when that
|
||||
// step runs out, and Asked a JSON array of the ids of the people it has
|
||||
// reached so far.
|
||||
Step int `db:"step"`
|
||||
StepDeadline sql.NullTime `db:"step_deadline"`
|
||||
Asked string `db:"asked"`
|
||||
// ChangeRequest is whether the request went through a change request
|
||||
// rather than the approvers' channels.
|
||||
ChangeRequest bool `db:"change_request"`
|
||||
// Counted is whether the approval was counted toward raising its class.
|
||||
Counted bool `db:"counted"`
|
||||
// DiscardedBecause is why an approval no longer stood.
|
||||
DiscardedBecause sql.NullString `db:"discarded_because"`
|
||||
}
|
||||
|
||||
// Store is the approvals module's Postgres store.
|
||||
@@ -49,7 +62,8 @@ type Store struct {
|
||||
func New(db *sqlx.DB) *Store { return &Store{db: db} }
|
||||
|
||||
const requestColumns = `id, org_id, run_id, key, decision_id, kind, digest, connector, action, target_kind,
|
||||
target_id, level, summary, state, deadline, created_at, answered_at, answered_by, answered_via`
|
||||
target_id, level, summary, state, deadline, created_at, answered_at, answered_by, answered_via,
|
||||
step, step_deadline, asked::text AS asked, change_request, counted, discarded_because`
|
||||
|
||||
// Put records r unless a request already exists for its run and key, and
|
||||
// returns the request that stands either way.
|
||||
@@ -57,12 +71,12 @@ func (s *Store) Put(ctx context.Context, r Request) (Request, error) {
|
||||
var out Request
|
||||
err := s.db.GetContext(ctx, &out, `
|
||||
INSERT INTO approvals.requests (id, org_id, run_id, key, decision_id, kind, digest, connector, action,
|
||||
target_kind, target_id, level, summary, state, deadline)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)
|
||||
target_kind, target_id, level, summary, state, deadline, step_deadline, asked, change_request)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, CAST($17 AS jsonb), $18)
|
||||
ON CONFLICT (org_id, run_id, key) DO NOTHING
|
||||
RETURNING `+requestColumns,
|
||||
r.ID, r.OrgID, r.RunID, r.Key, r.DecisionID, r.Kind, r.Digest, r.Connector, r.Action, r.TargetKind,
|
||||
r.TargetID, r.Level, r.Summary, r.State, r.Deadline)
|
||||
r.TargetID, r.Level, r.Summary, r.State, r.Deadline, r.StepDeadline, r.askedOrEmpty(), r.ChangeRequest)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return s.Get(ctx, r.OrgID, r.RunID, r.Key)
|
||||
}
|
||||
@@ -72,6 +86,13 @@ func (s *Store) Put(ctx context.Context, r Request) (Request, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (r Request) askedOrEmpty() string {
|
||||
if r.Asked == "" {
|
||||
return "[]"
|
||||
}
|
||||
return r.Asked
|
||||
}
|
||||
|
||||
// Get returns the request for one run's key.
|
||||
func (s *Store) Get(ctx context.Context, orgID string, runID uuid.UUID, key string) (Request, error) {
|
||||
var out Request
|
||||
@@ -104,6 +125,60 @@ func (s *Store) Answer(ctx context.Context, orgID string, id uuid.UUID, state, b
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Escalate moves a pending request to step, which has reached person and runs
|
||||
// out at until. It returns false when the request is no longer pending, or has
|
||||
// already moved that far, so a replayed escalation changes nothing.
|
||||
func (s *Store) Escalate(ctx context.Context, orgID string, id uuid.UUID, step int, person string, until time.Time) (Request, bool, error) {
|
||||
var out Request
|
||||
err := s.db.GetContext(ctx, &out, `
|
||||
UPDATE approvals.requests
|
||||
SET step = $3, step_deadline = $4,
|
||||
asked = CASE WHEN $5::text = '' OR asked @> to_jsonb($5::text) THEN asked ELSE asked || to_jsonb($5::text) END
|
||||
WHERE org_id = $1 AND id = $2 AND state = 'pending' AND step < $3
|
||||
RETURNING `+requestColumns, orgID, id, step, until, person)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return Request{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return Request{}, false, fmt.Errorf("escalate request: %w", err)
|
||||
}
|
||||
return out, true, nil
|
||||
}
|
||||
|
||||
// Discard withdraws an approval that no longer stands, and only while it has
|
||||
// not been counted. It returns false when there was nothing to discard.
|
||||
func (s *Store) Discard(ctx context.Context, orgID string, id uuid.UUID, because string) (Request, bool, error) {
|
||||
var out Request
|
||||
err := s.db.GetContext(ctx, &out, `
|
||||
UPDATE approvals.requests SET state = 'discarded', discarded_because = $3
|
||||
WHERE org_id = $1 AND id = $2 AND state = 'approved' AND NOT counted
|
||||
RETURNING `+requestColumns, orgID, id, because)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return Request{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return Request{}, false, fmt.Errorf("discard approval: %w", err)
|
||||
}
|
||||
return out, true, nil
|
||||
}
|
||||
|
||||
// Count marks an approval as counted, once. It returns false when the approval
|
||||
// was already counted or no longer stands.
|
||||
func (s *Store) Count(ctx context.Context, orgID string, id uuid.UUID) (Request, bool, error) {
|
||||
var out Request
|
||||
err := s.db.GetContext(ctx, &out, `
|
||||
UPDATE approvals.requests SET counted = true
|
||||
WHERE org_id = $1 AND id = $2 AND state = 'approved' AND NOT counted AND kind = 'action'
|
||||
RETURNING `+requestColumns, orgID, id)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return Request{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return Request{}, false, fmt.Errorf("count approval: %w", err)
|
||||
}
|
||||
return out, true, nil
|
||||
}
|
||||
|
||||
// ByID returns one request.
|
||||
func (s *Store) ByID(ctx context.Context, orgID string, id uuid.UUID) (Request, error) {
|
||||
var out Request
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
package runs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json/jsontext"
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/samcharles93/archie-core/modules/approvals"
|
||||
"github.com/samcharles93/archie-core/modules/connectors"
|
||||
"github.com/samcharles93/archie-core/modules/governance"
|
||||
"github.com/samcharles93/archie-core/modules/judgement"
|
||||
"github.com/samcharles93/archie-core/modules/runs/internal/store"
|
||||
)
|
||||
|
||||
// TestAfterApprovalDiscardsAnApprovalThatNoLongerMatchesTheAction guards the
|
||||
// rule that an approval covers the action as it was shown: when something else
|
||||
// changed the target or effect, or judgement got worse, the approval is
|
||||
// discarded and the action is put to people again, never run on the old one.
|
||||
func TestAfterApprovalDiscardsAnApprovalThatNoLongerMatchesTheAction(t *testing.T) {
|
||||
decided := func(level judgement.Level, mutate func(*judgement.Decision)) judgement.Decision {
|
||||
d := judgement.Decision{Level: level, Action: judgement.Action{
|
||||
Target: connectors.Target{Kind: "service", ID: "api", Environment: "prod"},
|
||||
Effect: connectors.Effect{Summary: "restart"},
|
||||
}}
|
||||
if mutate != nil {
|
||||
mutate(&d)
|
||||
}
|
||||
return d
|
||||
}
|
||||
approved := decided(judgement.ProposeAndWait, nil)
|
||||
tests := []struct {
|
||||
name string
|
||||
fresh judgement.Decision
|
||||
want afterwards
|
||||
}{
|
||||
{name: "an unchanged action runs on its approval", fresh: decided(judgement.ProposeAndWait, nil), want: runApproval},
|
||||
{name: "a worse verdict is put to people again", fresh: decided(judgement.Escalate, nil), want: reaskPerson},
|
||||
{name: "a changed target is put to people again", fresh: decided(judgement.ProposeAndWait, func(d *judgement.Decision) {
|
||||
d.Action.Target.ID = "db"
|
||||
}), want: reaskPerson},
|
||||
{name: "a changed environment is put to people again", fresh: decided(judgement.ProposeAndWait, func(d *judgement.Decision) {
|
||||
d.Action.Target.Environment = "staging"
|
||||
}), want: reaskPerson},
|
||||
{name: "a changed effect is put to people again", fresh: decided(judgement.ProposeAndWait, func(d *judgement.Decision) {
|
||||
d.Action.Effect.Summary = "delete"
|
||||
}), want: reaskPerson},
|
||||
{name: "a refusal discards the approval", fresh: decided(judgement.Never, nil), want: refuseAction},
|
||||
{name: "a verdict that lets it run needs no approval", fresh: decided(judgement.ActAndTell, nil), want: runFresh},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, why := afterApproval(approved, tt.fresh)
|
||||
if got != tt.want {
|
||||
t.Fatalf("afterApproval = %v (%s), want %v", got, why, tt.want)
|
||||
}
|
||||
if (got == reaskPerson || got == refuseAction) && why == "" {
|
||||
t.Fatal("a discarded approval gives no reason")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// signalAfter is a worker store whose answer arrives once the run has waited
|
||||
// that many times, as when a person answers during a later escalation step.
|
||||
type signalAfter struct {
|
||||
fakeWork
|
||||
waits int
|
||||
arrives int
|
||||
}
|
||||
|
||||
func (s *signalAfter) Signalled(context.Context, uuid.UUID, string) (jsontext.Value, bool, error) {
|
||||
s.waits++
|
||||
if s.arrives > 0 && s.waits >= s.arrives {
|
||||
return jsontext.Value(`{"approved":true,"by":"Second"}`), true, nil
|
||||
}
|
||||
return nil, false, nil
|
||||
}
|
||||
|
||||
// escalations records what the run asked of approvals.
|
||||
type escalations struct {
|
||||
steps []int
|
||||
expired int
|
||||
}
|
||||
|
||||
func (e *escalations) Ask(context.Context, uuid.UUID, string, judgement.Decision, []governance.PathStep) (approvals.Request, error) {
|
||||
return approvals.Request{}, nil
|
||||
}
|
||||
|
||||
func (e *escalations) Escalate(_ context.Context, _ uuid.UUID, step int, _ governance.PathStep) error {
|
||||
e.steps = append(e.steps, step)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *escalations) Expire(context.Context, uuid.UUID) error { e.expired++; return nil }
|
||||
|
||||
func (e *escalations) Accept(context.Context, uuid.UUID) error { return nil }
|
||||
|
||||
func (e *escalations) Discard(context.Context, uuid.UUID, string) error { return nil }
|
||||
|
||||
// TestWaitThroughPathMovesOnAtEachDeadlineAndExpiresAtTheEnd guards the timed
|
||||
// escalation path: an unanswered request passes to each next step in order,
|
||||
// the last step's expiry settles it as expired with nothing acted on, and an
|
||||
// answer at any step stops the escalation there.
|
||||
func TestWaitThroughPathMovesOnAtEachDeadlineAndExpiresAtTheEnd(t *testing.T) {
|
||||
// A zero deadline has passed by the time it is waited on.
|
||||
step := func(person string) governance.PathStep { return governance.PathStep{Person: person} }
|
||||
tests := []struct {
|
||||
name string
|
||||
path []governance.PathStep
|
||||
arrives int
|
||||
wantSteps []int
|
||||
wantExpired int
|
||||
wantAnswered bool
|
||||
}{
|
||||
{name: "one step with no answer expires without escalating", path: []governance.PathStep{step("a")}, wantExpired: 1},
|
||||
{name: "every step runs out and then the request expires", path: []governance.PathStep{step("a"), step("b"), step("c")}, wantSteps: []int{1, 2}, wantExpired: 1},
|
||||
{name: "an answer at the second step stops the escalation", path: []governance.PathStep{step("a"), step("b"), step("c")}, arrives: 2, wantSteps: []int{1}, wantAnswered: true},
|
||||
{name: "an answer at the first step escalates nowhere", path: []governance.PathStep{step("a"), step("b")}, arrives: 1, wantAnswered: true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
work := &signalAfter{arrives: tt.arrives}
|
||||
rc := newContext(context.Background(), work, store.Run{ID: uuid.New()}, uuid.New(), nil)
|
||||
ask := &escalations{}
|
||||
answer, expired, err := waitThroughPath(rc, ask, "k", approvals.Request{ID: uuid.New()}, tt.path)
|
||||
if err != nil {
|
||||
t.Fatalf("waitThroughPath: %v", err)
|
||||
}
|
||||
if !slices.Equal(ask.steps, tt.wantSteps) || ask.expired != tt.wantExpired {
|
||||
t.Fatalf("escalated %v and expired %d times, want %v and %d", ask.steps, ask.expired, tt.wantSteps, tt.wantExpired)
|
||||
}
|
||||
if expired != (tt.wantExpired > 0) || answer.Approved != tt.wantAnswered {
|
||||
t.Fatalf("expired = %v, approved = %v", expired, answer.Approved)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+197
-97
@@ -8,6 +8,7 @@ import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -25,10 +26,10 @@ import (
|
||||
)
|
||||
|
||||
// Settings is where a workflow finds what the organisation decided: the agent
|
||||
// model it runs, and how long an approval request waits for an answer.
|
||||
// model it runs, and the escalation path an approval request takes.
|
||||
type Settings interface {
|
||||
AgentModel(ctx context.Context) (governance.Model, error)
|
||||
ApprovalDeadline(ctx context.Context) (time.Duration, error)
|
||||
EscalationPath(ctx context.Context, needAuthority bool) ([]governance.PathStep, error)
|
||||
RecurrenceThreshold(ctx context.Context) (int, error)
|
||||
AttemptsPerRun(ctx context.Context) (int, error)
|
||||
}
|
||||
@@ -49,11 +50,16 @@ type Judge interface {
|
||||
ForRun(ctx context.Context, runID uuid.UUID) ([]judgement.Decision, error)
|
||||
}
|
||||
|
||||
// Approver asks people to allow an action judgement held, and settles a request
|
||||
// whose deadline passed.
|
||||
// Approver asks people to allow an action judgement held, moves a request
|
||||
// that nobody answered to the next step of its path, and settles one whose last
|
||||
// step ran out. An approval is counted when the action is about to run on it,
|
||||
// and discarded when the action no longer matches it.
|
||||
type Approver interface {
|
||||
Ask(ctx context.Context, runID uuid.UUID, key string, d judgement.Decision, deadline time.Time) (approvals.Request, error)
|
||||
Ask(ctx context.Context, runID uuid.UUID, key string, d judgement.Decision, path []governance.PathStep) (approvals.Request, error)
|
||||
Escalate(ctx context.Context, id uuid.UUID, step int, to governance.PathStep) error
|
||||
Expire(ctx context.Context, id uuid.UUID) error
|
||||
Accept(ctx context.Context, id uuid.UUID) error
|
||||
Discard(ctx context.Context, id uuid.UUID, because string) error
|
||||
}
|
||||
|
||||
// Notifier tells people what a run found.
|
||||
@@ -78,25 +84,6 @@ func stepBudget(ctx context.Context, settings Settings) (agentloop.Budget, error
|
||||
return budget, nil
|
||||
}
|
||||
|
||||
// pinnedDeadline is pinned in the run's first steps, like the model: a run that
|
||||
// waits keeps the deadline it started with, whatever the setting becomes
|
||||
// meanwhile. A step's value is encoded, so it travels as the duration's own
|
||||
// text.
|
||||
func pinnedDeadline(rc *Context, settings Settings) (time.Duration, error) {
|
||||
text, err := rc.Step("approval-deadline", func(ctx context.Context, _ string) (string, error) {
|
||||
d, err := settings.ApprovalDeadline(ctx)
|
||||
return d.String(), err
|
||||
})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
deadline, err := time.ParseDuration(text)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("approval deadline %q: %w", text, err)
|
||||
}
|
||||
return deadline, nil
|
||||
}
|
||||
|
||||
// Investigate is the workflow that has an agent look into the event that
|
||||
// started the run and report its findings. The agent model and the tools are
|
||||
// pinned in the run's first steps, so a resumed run asks the same model with
|
||||
@@ -111,10 +98,6 @@ func Investigate(settings Settings, box Toolbox, judge Judge, ask Approver, noti
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
deadline, err := pinnedDeadline(rc, settings)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
threshold, err := rc.Step("recurrence-threshold", func(ctx context.Context, _ string) (int, error) {
|
||||
return settings.RecurrenceThreshold(ctx)
|
||||
})
|
||||
@@ -160,7 +143,7 @@ func Investigate(settings Settings, box Toolbox, judge Judge, ask Approver, noti
|
||||
if err := recordCause(rc, mem, input, res, AgentRequest{Provider: provider, Model: model.Model, ModelInfo: info}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
outcomes, err := settleProposals(rc, judge, ask, deadline)
|
||||
outcomes, err := settleProposals(rc, judge, ask, settings)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -286,9 +269,9 @@ type approvalOutcome struct {
|
||||
|
||||
// settleProposals puts every action judgement held to a person, one at a time,
|
||||
// and runs the ones they allow. Each ask, wait and run is its own step, so a
|
||||
// resumed run picks up where it waited. A proposal nobody answers in time is
|
||||
// settled as expired and nothing runs.
|
||||
func settleProposals(rc *Context, judge Judge, ask Approver, deadline time.Duration) ([]approvalOutcome, error) {
|
||||
// resumed run picks up where it waited. A proposal nobody answers before the
|
||||
// last escalation step runs out is settled as expired and nothing runs.
|
||||
func settleProposals(rc *Context, judge Judge, ask Approver, settings Settings) ([]approvalOutcome, error) {
|
||||
held, err := rc.Step("proposals", func(ctx context.Context, _ string) ([]judgement.Decision, error) {
|
||||
decisions, err := judge.ForRun(ctx, rc.RunID())
|
||||
if err != nil {
|
||||
@@ -301,7 +284,7 @@ func settleProposals(rc *Context, judge Judge, ask Approver, deadline time.Durat
|
||||
}
|
||||
out := make([]approvalOutcome, 0, len(held))
|
||||
for _, d := range held {
|
||||
outcome, err := settleOne(rc, judge, ask, d, deadline)
|
||||
outcome, err := settleOne(rc, judge, ask, settings, d)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -310,37 +293,116 @@ func settleProposals(rc *Context, judge Judge, ask Approver, deadline time.Durat
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func settleOne(rc *Context, judge Judge, ask Approver, d judgement.Decision, deadline time.Duration) (approvalOutcome, error) {
|
||||
out := approvalOutcome{digest: d.Action.Digest, Action: d.Action.Connector + " " + d.Action.Name, Target: d.Action.Target.Kind + ":" + d.Action.Target.ID}
|
||||
request, err := rc.Step("ask:"+d.Key, func(ctx context.Context, _ string) (approvals.Request, error) {
|
||||
return ask.Ask(ctx, rc.RunID(), waitKey(d.Key), d, time.Now().Add(deadline))
|
||||
// maxReapprovals bounds how many times an action that keeps changing under its
|
||||
// approval is put to people again.
|
||||
const maxReapprovals = 2
|
||||
|
||||
// settleOne gets one action approved and runs it. Before it runs it is judged
|
||||
// again; an approval whose action has since changed is discarded and the
|
||||
// changed action is put to people as a new request.
|
||||
func settleOne(rc *Context, judge Judge, ask Approver, settings Settings, d judgement.Decision) (approvalOutcome, error) {
|
||||
out := approvalOutcome{digest: d.Action.Digest}
|
||||
for round := 0; ; round++ {
|
||||
out.Action, out.Target = d.Action.Connector+" "+d.Action.Name, d.Action.Target.Kind+":"+d.Action.Target.ID
|
||||
request, answer, err := askThroughPath(rc, ask, settings, d)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
out.RequestID, out.State = request.ID, request.State
|
||||
if request.State != approvals.StateApproved {
|
||||
if request.State == approvals.StateDenied {
|
||||
out.By = answer.By
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
out.By = answer.By
|
||||
fresh, err := rc.Step("judge-again:"+d.Key, func(ctx context.Context, key string) (judgement.Decision, error) {
|
||||
return judge.Assess(ctx, judgement.Request{
|
||||
RunID: rc.RunID(), Key: key, Connector: d.Action.Connector, Action: d.Action.Name, Input: d.Action.Input,
|
||||
Workflow: rc.Workflow(),
|
||||
})
|
||||
})
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
next, why := afterApproval(d, fresh)
|
||||
switch next {
|
||||
case reaskPerson, refuseAction:
|
||||
if _, err := rc.Step("discard:"+d.Key, func(ctx context.Context, _ string) (bool, error) {
|
||||
return true, ask.Discard(ctx, request.ID, why)
|
||||
}); err != nil {
|
||||
return out, err
|
||||
}
|
||||
out.State = approvals.StateDiscarded
|
||||
if next == refuseAction || round == maxReapprovals {
|
||||
out.Error = why
|
||||
return out, nil
|
||||
}
|
||||
d = fresh
|
||||
continue
|
||||
case runFresh:
|
||||
// The approval isn't needed and isn't counted.
|
||||
out.Result, out.Error, out.Unknown = runAction(rc, judge, fresh, nil)
|
||||
default:
|
||||
if _, err := rc.Step("count:"+d.Key, func(ctx context.Context, _ string) (bool, error) {
|
||||
return true, ask.Accept(ctx, request.ID)
|
||||
}); err != nil {
|
||||
return out, err
|
||||
}
|
||||
out.Result, out.Error, out.Unknown = runAction(rc, judge, d, &answer)
|
||||
}
|
||||
out.State = approvals.StateApproved
|
||||
return out, nil
|
||||
}
|
||||
}
|
||||
|
||||
// askThroughPath puts d to the first step of its escalation path and waits.
|
||||
// The path is pinned when it is first asked, so a request that waits keeps the
|
||||
// deadlines it started with whatever the settings become meanwhile. It returns
|
||||
// the request as it stood when it settled, and the answer if there was one.
|
||||
func askThroughPath(rc *Context, ask Approver, settings Settings, d judgement.Decision) (approvals.Request, approvals.Answer, error) {
|
||||
path, err := rc.Step("path:"+d.Key, func(ctx context.Context, _ string) ([]governance.PathStep, error) {
|
||||
return settings.EscalationPath(ctx, d.Level == judgement.Escalate)
|
||||
})
|
||||
if err != nil {
|
||||
return out, err
|
||||
return approvals.Request{}, approvals.Answer{}, err
|
||||
}
|
||||
out.RequestID, out.State = request.ID, request.State
|
||||
if request.State != approvals.StatePending {
|
||||
return out, nil
|
||||
}
|
||||
answer, expired, err := waitForAnswer(rc, ask, d.Key, request, deadline)
|
||||
request, err := rc.Step("ask:"+d.Key, func(ctx context.Context, _ string) (approvals.Request, error) {
|
||||
return ask.Ask(ctx, rc.RunID(), waitKey(d.Key), d, path)
|
||||
})
|
||||
if err != nil {
|
||||
return out, err
|
||||
return request, approvals.Answer{}, err
|
||||
}
|
||||
if expired {
|
||||
out.State = approvals.StateExpired
|
||||
return out, nil
|
||||
if request.State != approvals.StatePending && request.State != approvals.StateApproved {
|
||||
return request, approvals.Answer{}, nil
|
||||
}
|
||||
if !answer.Approved {
|
||||
out.State, out.By = approvals.StateDenied, answer.By
|
||||
return out, nil
|
||||
// A request already answered when it was asked again still has its answer
|
||||
// waiting under the run's key, so the wait returns it at once.
|
||||
answer, expired, err := waitThroughPath(rc, ask, d.Key, request, path)
|
||||
switch {
|
||||
case err != nil:
|
||||
return request, answer, err
|
||||
case expired:
|
||||
request.State = approvals.StateExpired
|
||||
case answer.Approved:
|
||||
request.State = approvals.StateApproved
|
||||
default:
|
||||
request.State = approvals.StateDenied
|
||||
}
|
||||
out.By = answer.By
|
||||
// The action's own failure is part of what the answer led to, and its step
|
||||
// has recorded it, so the investigation still reports.
|
||||
run, runErr := runApproved(rc, judge, d, answer)
|
||||
out.State, out.Result, out.Error = approvals.StateApproved, run, errText(runErr)
|
||||
out.Unknown = errors.Is(runErr, ErrOutcomeUnknown)
|
||||
return out, nil
|
||||
return request, answer, nil
|
||||
}
|
||||
|
||||
// runAction runs the action judged as d, on approval when there is one, and
|
||||
// returns what came of it. The action's own failure is part of what the answer
|
||||
// led to, and its step has recorded it, so the investigation still reports.
|
||||
func runAction(rc *Context, judge Judge, d judgement.Decision, approval *approvals.Answer) (result, failure string, unknown bool) {
|
||||
var runErr error
|
||||
if approval == nil {
|
||||
result, runErr = runUnattended(rc, judge, d)
|
||||
} else {
|
||||
result, runErr = runApproved(rc, judge, d, *approval)
|
||||
}
|
||||
return result, errText(runErr), errors.Is(runErr, ErrOutcomeUnknown)
|
||||
}
|
||||
|
||||
// unknownOutcome records a connector's unconfirmed action as a step whose
|
||||
@@ -360,54 +422,92 @@ func errText(err error) string {
|
||||
return err.Error()
|
||||
}
|
||||
|
||||
// waitForAnswer waits for the person's answer. When the deadline passes with
|
||||
// none, the request is settled as expired and the caller runs nothing.
|
||||
func waitForAnswer(rc *Context, ask Approver, key string, request approvals.Request, deadline time.Duration) (approvals.Answer, bool, error) {
|
||||
payload, err := rc.Wait(waitKey(key), deadline)
|
||||
if errors.Is(err, ErrWaitTimedOut) {
|
||||
if _, eerr := rc.Step("expire:"+key, func(ctx context.Context, _ string) (bool, error) {
|
||||
return true, ask.Expire(ctx, request.ID)
|
||||
}); eerr != nil {
|
||||
return approvals.Answer{}, true, eerr
|
||||
// waitThroughPath waits for an answer through each step of path in turn. A step
|
||||
// that runs out with none passes the request to the next person; the last one
|
||||
// settles it as expired and the caller runs nothing. Every wait is a durable
|
||||
// run wait, so a restart resumes where each deadline stands.
|
||||
func waitThroughPath(rc *Context, ask Approver, key string, request approvals.Request, path []governance.PathStep) (approvals.Answer, bool, error) {
|
||||
for i, step := range path {
|
||||
if i > 0 {
|
||||
if _, err := rc.Step("escalate:"+key+":"+strconv.Itoa(i), func(ctx context.Context, _ string) (bool, error) {
|
||||
return true, ask.Escalate(ctx, request.ID, i, step)
|
||||
}); err != nil {
|
||||
return approvals.Answer{}, false, err
|
||||
}
|
||||
}
|
||||
return approvals.Answer{}, true, nil
|
||||
payload, err := rc.Wait(waitKey(key), step.Deadline())
|
||||
if errors.Is(err, ErrWaitTimedOut) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return approvals.Answer{}, false, err
|
||||
}
|
||||
var answer approvals.Answer
|
||||
if err := json.Unmarshal(payload, &answer); err != nil {
|
||||
return approvals.Answer{}, false, fmt.Errorf("decode approval answer: %w", err)
|
||||
}
|
||||
return answer, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return approvals.Answer{}, false, err
|
||||
if _, err := rc.Step("expire:"+key, func(ctx context.Context, _ string) (bool, error) {
|
||||
return true, ask.Expire(ctx, request.ID)
|
||||
}); err != nil {
|
||||
return approvals.Answer{}, true, err
|
||||
}
|
||||
var answer approvals.Answer
|
||||
if err := json.Unmarshal(payload, &answer); err != nil {
|
||||
return approvals.Answer{}, false, fmt.Errorf("decode approval answer: %w", err)
|
||||
}
|
||||
return answer, false, nil
|
||||
return approvals.Answer{}, true, nil
|
||||
}
|
||||
|
||||
// runApproved judges the action again and runs it, once. An approval covers
|
||||
// the action the person was shown, so a judgement that came back worse than
|
||||
// it was cannot be run on that approval.
|
||||
func runApproved(rc *Context, judge Judge, d judgement.Decision, answer approvals.Answer) (string, error) {
|
||||
fresh, err := rc.Step("judge-again:"+d.Key, func(ctx context.Context, key string) (judgement.Decision, error) {
|
||||
return judge.Assess(ctx, judgement.Request{
|
||||
RunID: rc.RunID(), Key: key, Connector: d.Action.Connector, Action: d.Action.Name, Input: d.Action.Input,
|
||||
Workflow: rc.Workflow(),
|
||||
})
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
// What an approval allows once the action is judged again.
|
||||
type afterwards int
|
||||
|
||||
const (
|
||||
// runApproval runs the action that was approved.
|
||||
runApproval afterwards = iota
|
||||
// runFresh runs the action on its new judgement, which lets it run without
|
||||
// anyone's approval.
|
||||
runFresh
|
||||
// reaskPerson discards the approval and puts the changed action to people.
|
||||
reaskPerson
|
||||
// refuseAction discards the approval: the action is now refused.
|
||||
refuseAction
|
||||
)
|
||||
|
||||
// afterApproval says what an approval of approved means now that the action was
|
||||
// judged again as fresh. An approval covers the action as it was shown, so one
|
||||
// whose verdict got worse, or whose target or effect changed, no longer
|
||||
// stands. The target and effect are what the connector resolves the action to
|
||||
// now, which is what judgement has to tell that something else changed them.
|
||||
func afterApproval(approved, fresh judgement.Decision) (afterwards, string) {
|
||||
switch {
|
||||
case fresh.Level == judgement.Never:
|
||||
return refuseAction, "refused after approval: " + fresh.Reason
|
||||
case fresh.Level.Runs():
|
||||
return runFresh, ""
|
||||
case fresh.Action.Target != approved.Action.Target:
|
||||
return reaskPerson, "the target changed since it was approved"
|
||||
case fresh.Action.Effect != approved.Action.Effect:
|
||||
return reaskPerson, "the effect changed since it was approved"
|
||||
case fresh.Level > approved.Level:
|
||||
return reaskPerson, "judgement got worse since it was approved: now " + fresh.Level.String()
|
||||
}
|
||||
return runApproval, ""
|
||||
}
|
||||
|
||||
// runUnattended runs an action its fresh judgement lets run without approval,
|
||||
// once.
|
||||
func runUnattended(rc *Context, judge Judge, d judgement.Decision) (string, error) {
|
||||
return rc.Step("run:"+d.Key, func(ctx context.Context, key string) (string, error) {
|
||||
switch {
|
||||
case fresh.Level == judgement.Never:
|
||||
return "", fmt.Errorf("refused after approval: %s", fresh.Reason)
|
||||
case fresh.Level.Runs():
|
||||
if err := begun(ctx, rc, key); err != nil {
|
||||
return "", err
|
||||
}
|
||||
out, err := judge.Execute(ctx, fresh, d.Action.Input, key)
|
||||
return out, unknownOutcome(err)
|
||||
case fresh.Level > d.Level:
|
||||
return "", fmt.Errorf("judgement got worse after approval: now %s", fresh.Level)
|
||||
if err := begun(ctx, rc, key); err != nil {
|
||||
return "", err
|
||||
}
|
||||
out, err := judge.Execute(ctx, d, d.Action.Input, key)
|
||||
return out, unknownOutcome(err)
|
||||
})
|
||||
}
|
||||
|
||||
// runApproved runs the action a person approved, once. judgement runs only the
|
||||
// action that was judged, whatever input comes with it.
|
||||
func runApproved(rc *Context, judge Judge, d judgement.Decision, answer approvals.Answer) (string, error) {
|
||||
return rc.Step("run:"+d.Key, func(ctx context.Context, key string) (string, error) {
|
||||
if err := begun(ctx, rc, key); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
@@ -55,7 +55,7 @@ func TestActionStartedBeforeItsResultIsNeverRunAgain(t *testing.T) {
|
||||
return []store.Step{{RunID: runID, Seq: markerBase, Name: startedPrefix + key, Outcome: store.OutcomeDone, Output: jsontext.Value("true")}}
|
||||
}
|
||||
agentKey := runID.String() + ":investigate:3"
|
||||
approvedKey := runID.String() + ":1"
|
||||
approvedKey := runID.String() + ":0"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
|
||||
Reference in new issue
Block a user