219 lines
7.0 KiB
Go
219 lines
7.0 KiB
Go
// Package memory is what Archie remembers about a target: each incident
|
|
// investigated on it, the cause found, and what was done about it. A cause that
|
|
// keeps coming back on a target, whatever became of each attempt to handle it,
|
|
// raises a proposal for a root-cause fix.
|
|
package memory
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json/jsontext"
|
|
json "encoding/json/v2"
|
|
"errors"
|
|
"fmt"
|
|
"regexp"
|
|
"slices"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/jmoiron/sqlx"
|
|
|
|
"github.com/samcharles93/archie-core/modules/memory/internal/store"
|
|
)
|
|
|
|
// maxHistory bounds the incidents read for one target. The recurrence setting
|
|
// is capped below it.
|
|
const maxHistory = 200
|
|
|
|
// MaxRecurrence is the largest recurrence threshold the history can confirm.
|
|
const MaxRecurrence = 100
|
|
|
|
// Target is what an incident happened to: a kind and an id, such as host web-1.
|
|
type Target struct {
|
|
Kind string `json:"kind"`
|
|
ID string `json:"id"`
|
|
}
|
|
|
|
// Action is something done during an incident and how it ended.
|
|
type Action struct {
|
|
Action string `json:"action"`
|
|
Target string `json:"target"`
|
|
// State is what became of it: ran, refused, or the approval's state.
|
|
State string `json:"state"`
|
|
Detail string `json:"detail,omitzero"`
|
|
}
|
|
|
|
// Incident is one investigation of a target.
|
|
type Incident struct {
|
|
At time.Time `json:"at"`
|
|
Cause string `json:"cause"`
|
|
Summary string `json:"summary"`
|
|
Actions []Action `json:"actions"`
|
|
}
|
|
|
|
// Line is the incident as one entry of a list an agent reads.
|
|
func (i Incident) Line() string {
|
|
line := fmt.Sprintf("- %s, cause %q: %s", i.At.Format("2006-01-02"), i.Cause, i.Summary)
|
|
if len(i.Actions) == 0 {
|
|
return line + " Actions: none."
|
|
}
|
|
parts := make([]string, 0, len(i.Actions))
|
|
for _, a := range i.Actions {
|
|
part := a.Action + " on " + a.Target + ": " + a.State
|
|
if a.Detail != "" {
|
|
part += " (" + a.Detail + ")"
|
|
}
|
|
parts = append(parts, part)
|
|
}
|
|
return line + " Actions: " + strings.Join(parts, "; ") + "."
|
|
}
|
|
|
|
// Review is what recording a run's actions found about its cause.
|
|
type Review struct {
|
|
// Repeats is how many incidents on the target have this cause, this one
|
|
// included.
|
|
Repeats int
|
|
// Proposal is the root-cause fix asked for, empty until the cause has
|
|
// recurred enough.
|
|
Proposal string
|
|
}
|
|
|
|
// ledger is where incidents and proposals are kept.
|
|
type ledger interface {
|
|
Upsert(ctx context.Context, orgID string, in store.Incident) error
|
|
SetActions(ctx context.Context, orgID string, runID uuid.UUID, actions []byte) error
|
|
Incident(ctx context.Context, orgID string, runID uuid.UUID) (store.Incident, bool, error)
|
|
ByTarget(ctx context.Context, orgID, kind, id string, limit int) ([]store.Incident, error)
|
|
Propose(ctx context.Context, orgID string, p store.Proposal) (store.Proposal, error)
|
|
}
|
|
|
|
// Module remembers one organisation's incidents.
|
|
type Module struct {
|
|
orgID string
|
|
ledger ledger
|
|
}
|
|
|
|
// NewModule returns the memory module for orgID.
|
|
func NewModule(db *sqlx.DB, orgID string) *Module {
|
|
return &Module{orgID: orgID, ledger: store.New(db)}
|
|
}
|
|
|
|
var unsafeCause = regexp.MustCompile(`[^a-z0-9]+`)
|
|
|
|
// Cause is a cause as it is compared: lower case words joined by hyphens, so
|
|
// "Disk pressure" and "disk-pressure" are one cause.
|
|
func Cause(s string) string {
|
|
return strings.Trim(unsafeCause.ReplaceAllString(strings.ToLower(s), "-"), "-")
|
|
}
|
|
|
|
// Record remembers what a run's investigation found: the event, the target it
|
|
// concerned and the cause. Recording again replaces the earlier attempt's.
|
|
func (m *Module) Record(ctx context.Context, runID uuid.UUID, event jsontext.Value, t Target, cause, summary string) error {
|
|
t = Target{Kind: strings.TrimSpace(t.Kind), ID: strings.TrimSpace(t.ID)}
|
|
cause = Cause(cause)
|
|
summary = strings.TrimSpace(summary)
|
|
if t.Kind == "" || t.ID == "" || cause == "" || summary == "" {
|
|
return errors.New("a target kind, a target id, a cause and a summary are all required")
|
|
}
|
|
if !event.IsValid() {
|
|
event = jsontext.Value(`null`)
|
|
}
|
|
return m.ledger.Upsert(ctx, m.orgID, store.Incident{
|
|
ID: uuid.New(), RunID: runID, TargetKind: t.Kind, TargetID: t.ID, Cause: cause, Summary: summary, Event: event,
|
|
})
|
|
}
|
|
|
|
// Past returns the incidents recorded on t before the run's own, newest first.
|
|
func (m *Module) Past(ctx context.Context, runID uuid.UUID, t Target) ([]Incident, error) {
|
|
rows, err := m.ledger.ByTarget(ctx, m.orgID, strings.TrimSpace(t.Kind), strings.TrimSpace(t.ID), maxHistory)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out := make([]Incident, 0, len(rows))
|
|
for _, r := range rows {
|
|
if r.RunID == runID {
|
|
continue
|
|
}
|
|
in, err := incident(r)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out = append(out, in)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// Finish remembers what was done in a run's incident and checks whether its
|
|
// cause has come back enough to ask for a root-cause fix: that is when
|
|
// threshold incidents on the target share the cause, whatever their outcomes.
|
|
// A run that recorded no incident has nothing to finish.
|
|
func (m *Module) Finish(ctx context.Context, runID uuid.UUID, actions []Action, threshold int) (Review, error) {
|
|
current, ok, err := m.ledger.Incident(ctx, m.orgID, runID)
|
|
if err != nil || !ok {
|
|
return Review{}, err
|
|
}
|
|
raw, err := json.Marshal(actions)
|
|
if err != nil {
|
|
return Review{}, fmt.Errorf("encode actions: %w", err)
|
|
}
|
|
if err := m.ledger.SetActions(ctx, m.orgID, runID, raw); err != nil {
|
|
return Review{}, err
|
|
}
|
|
rows, err := m.ledger.ByTarget(ctx, m.orgID, current.TargetKind, current.TargetID, maxHistory)
|
|
if err != nil {
|
|
return Review{}, err
|
|
}
|
|
var same []store.Incident
|
|
for _, r := range rows {
|
|
if r.Cause == current.Cause {
|
|
same = append(same, r)
|
|
}
|
|
}
|
|
review := Review{Repeats: len(same)}
|
|
if review.Repeats < threshold {
|
|
return review, nil
|
|
}
|
|
proposal, err := m.ledger.Propose(ctx, m.orgID, store.Proposal{
|
|
ID: uuid.New(), TargetKind: current.TargetKind, TargetID: current.TargetID, Cause: current.Cause,
|
|
Incidents: review.Repeats, Body: proposalBody(current, same),
|
|
})
|
|
if err != nil {
|
|
return Review{}, err
|
|
}
|
|
review.Proposal = proposal.Body
|
|
return review, nil
|
|
}
|
|
|
|
func incident(r store.Incident) (Incident, error) {
|
|
out := Incident{At: r.At, Cause: r.Cause, Summary: r.Summary}
|
|
if len(r.Actions) > 0 {
|
|
if err := json.Unmarshal(r.Actions, &out.Actions); err != nil {
|
|
return Incident{}, fmt.Errorf("decode actions of incident %s: %w", r.ID, err)
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// proposalBody asks for the cause to be fixed rather than handled again. same
|
|
// is newest first.
|
|
func proposalBody(current store.Incident, same []store.Incident) string {
|
|
var tried []string
|
|
for _, r := range same {
|
|
in, err := incident(r)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
for _, a := range in.Actions {
|
|
if line := a.Action + " on " + a.Target; !slices.Contains(tried, line) {
|
|
tried = append(tried, line)
|
|
}
|
|
}
|
|
}
|
|
body := fmt.Sprintf("%q has been the cause of %d incidents on %s %s since %s. Handling each one has not stopped it, so find what keeps producing it and fix that.",
|
|
current.Cause, len(same), current.TargetKind, current.TargetID, same[len(same)-1].At.Format("2006-01-02"))
|
|
if len(tried) > 0 {
|
|
body += " Handled so far with: " + strings.Join(tried, "; ") + "."
|
|
}
|
|
return body
|
|
}
|