// Package memory is what Archie remembers about a target: each incident // investigated on it, the cause found, and what was done about it. A cause that // keeps coming back on a target, whatever became of each attempt to handle it, // raises a proposal for a root-cause fix. package memory import ( "context" "encoding/json/jsontext" json "encoding/json/v2" "errors" "fmt" "regexp" "slices" "strings" "time" "github.com/google/uuid" "github.com/jmoiron/sqlx" "github.com/samcharles93/archie-core/modules/memory/internal/store" ) // maxHistory bounds the incidents read for one target. The recurrence setting // is capped below it. const maxHistory = 200 // MaxRecurrence is the largest recurrence threshold the history can confirm. const MaxRecurrence = 100 // Target is what an incident happened to: a kind and an id, such as host web-1. type Target struct { Kind string `json:"kind"` ID string `json:"id"` } // Action is something done during an incident and how it ended. type Action struct { Action string `json:"action"` Target string `json:"target"` // State is what became of it: ran, refused, or the approval's state. State string `json:"state"` Detail string `json:"detail,omitzero"` } // Incident is one investigation of a target. type Incident struct { At time.Time `json:"at"` Cause string `json:"cause"` Summary string `json:"summary"` Actions []Action `json:"actions"` } // Line is the incident as one entry of a list an agent reads. func (i Incident) Line() string { line := fmt.Sprintf("- %s, cause %q: %s", i.At.Format("2006-01-02"), i.Cause, i.Summary) if len(i.Actions) == 0 { return line + " Actions: none." } parts := make([]string, 0, len(i.Actions)) for _, a := range i.Actions { part := a.Action + " on " + a.Target + ": " + a.State if a.Detail != "" { part += " (" + a.Detail + ")" } parts = append(parts, part) } return line + " Actions: " + strings.Join(parts, "; ") + "." } // Review is what recording a run's actions found about its cause. type Review struct { // Repeats is how many incidents on the target have this cause, this one // included. Repeats int // Proposal is the root-cause fix asked for, empty until the cause has // recurred enough. Proposal string } // ledger is where incidents and proposals are kept. type ledger interface { Upsert(ctx context.Context, orgID string, in store.Incident) error SetActions(ctx context.Context, orgID string, runID uuid.UUID, actions []byte) error Incident(ctx context.Context, orgID string, runID uuid.UUID) (store.Incident, bool, error) ByTarget(ctx context.Context, orgID, kind, id string, limit int) ([]store.Incident, error) Propose(ctx context.Context, orgID string, p store.Proposal) (store.Proposal, error) } // Module remembers one organisation's incidents. type Module struct { orgID string ledger ledger } // NewModule returns the memory module for orgID. func NewModule(db *sqlx.DB, orgID string) *Module { return &Module{orgID: orgID, ledger: store.New(db)} } var unsafeCause = regexp.MustCompile(`[^a-z0-9]+`) // Cause is a cause as it is compared: lower case words joined by hyphens, so // "Disk pressure" and "disk-pressure" are one cause. func Cause(s string) string { return strings.Trim(unsafeCause.ReplaceAllString(strings.ToLower(s), "-"), "-") } // Record remembers what a run's investigation found: the event, the target it // concerned and the cause. Recording again replaces the earlier attempt's. func (m *Module) Record(ctx context.Context, runID uuid.UUID, event jsontext.Value, t Target, cause, summary string) error { t = Target{Kind: strings.TrimSpace(t.Kind), ID: strings.TrimSpace(t.ID)} cause = Cause(cause) summary = strings.TrimSpace(summary) if t.Kind == "" || t.ID == "" || cause == "" || summary == "" { return errors.New("a target kind, a target id, a cause and a summary are all required") } if !event.IsValid() { event = jsontext.Value(`null`) } return m.ledger.Upsert(ctx, m.orgID, store.Incident{ ID: uuid.New(), RunID: runID, TargetKind: t.Kind, TargetID: t.ID, Cause: cause, Summary: summary, Event: event, }) } // Past returns the incidents recorded on t before the run's own, newest first. func (m *Module) Past(ctx context.Context, runID uuid.UUID, t Target) ([]Incident, error) { rows, err := m.ledger.ByTarget(ctx, m.orgID, strings.TrimSpace(t.Kind), strings.TrimSpace(t.ID), maxHistory) if err != nil { return nil, err } out := make([]Incident, 0, len(rows)) for _, r := range rows { if r.RunID == runID { continue } in, err := incident(r) if err != nil { return nil, err } out = append(out, in) } return out, nil } // Finish remembers what was done in a run's incident and checks whether its // cause has come back enough to ask for a root-cause fix: that is when // threshold incidents on the target share the cause, whatever their outcomes. // A run that recorded no incident has nothing to finish. func (m *Module) Finish(ctx context.Context, runID uuid.UUID, actions []Action, threshold int) (Review, error) { current, ok, err := m.ledger.Incident(ctx, m.orgID, runID) if err != nil || !ok { return Review{}, err } raw, err := json.Marshal(actions) if err != nil { return Review{}, fmt.Errorf("encode actions: %w", err) } if err := m.ledger.SetActions(ctx, m.orgID, runID, raw); err != nil { return Review{}, err } rows, err := m.ledger.ByTarget(ctx, m.orgID, current.TargetKind, current.TargetID, maxHistory) if err != nil { return Review{}, err } var same []store.Incident for _, r := range rows { if r.Cause == current.Cause { same = append(same, r) } } review := Review{Repeats: len(same)} if review.Repeats < threshold { return review, nil } proposal, err := m.ledger.Propose(ctx, m.orgID, store.Proposal{ ID: uuid.New(), TargetKind: current.TargetKind, TargetID: current.TargetID, Cause: current.Cause, Incidents: review.Repeats, Body: proposalBody(current, same), }) if err != nil { return Review{}, err } review.Proposal = proposal.Body return review, nil } func incident(r store.Incident) (Incident, error) { out := Incident{At: r.At, Cause: r.Cause, Summary: r.Summary} if len(r.Actions) > 0 { if err := json.Unmarshal(r.Actions, &out.Actions); err != nil { return Incident{}, fmt.Errorf("decode actions of incident %s: %w", r.ID, err) } } return out, nil } // proposalBody asks for the cause to be fixed rather than handled again. same // is newest first. func proposalBody(current store.Incident, same []store.Incident) string { var tried []string for _, r := range same { in, err := incident(r) if err != nil { continue } for _, a := range in.Actions { if line := a.Action + " on " + a.Target; !slices.Contains(tried, line) { tried = append(tried, line) } } } body := fmt.Sprintf("%q has been the cause of %d incidents on %s %s since %s. Handling each one has not stopped it, so find what keeps producing it and fix that.", current.Cause, len(same), current.TargetKind, current.TargetID, same[len(same)-1].At.Format("2006-01-02")) if len(tried) > 0 { body += " Handled so far with: " + strings.Join(tried, "; ") + "." } return body }