Files
archie/modules/memory/memory.go
T

219 lines
7.0 KiB
Go

// Package memory is what Archie remembers about a target: each incident
// investigated on it, the cause found, and what was done about it. A cause that
// keeps coming back on a target, whatever became of each attempt to handle it,
// raises a proposal for a root-cause fix.
package memory
import (
"context"
"encoding/json/jsontext"
json "encoding/json/v2"
"errors"
"fmt"
"regexp"
"slices"
"strings"
"time"
"github.com/google/uuid"
"github.com/jmoiron/sqlx"
"github.com/samcharles93/archie-core/modules/memory/internal/store"
)
// maxHistory bounds the incidents read for one target. The recurrence setting
// is capped below it.
const maxHistory = 200
// MaxRecurrence is the largest recurrence threshold the history can confirm.
const MaxRecurrence = 100
// Target is what an incident happened to: a kind and an id, such as host web-1.
type Target struct {
Kind string `json:"kind"`
ID string `json:"id"`
}
// Action is something done during an incident and how it ended.
type Action struct {
Action string `json:"action"`
Target string `json:"target"`
// State is what became of it: ran, refused, or the approval's state.
State string `json:"state"`
Detail string `json:"detail,omitzero"`
}
// Incident is one investigation of a target.
type Incident struct {
At time.Time `json:"at"`
Cause string `json:"cause"`
Summary string `json:"summary"`
Actions []Action `json:"actions"`
}
// Line is the incident as one entry of a list an agent reads.
func (i Incident) Line() string {
line := fmt.Sprintf("- %s, cause %q: %s", i.At.Format("2006-01-02"), i.Cause, i.Summary)
if len(i.Actions) == 0 {
return line + " Actions: none."
}
parts := make([]string, 0, len(i.Actions))
for _, a := range i.Actions {
part := a.Action + " on " + a.Target + ": " + a.State
if a.Detail != "" {
part += " (" + a.Detail + ")"
}
parts = append(parts, part)
}
return line + " Actions: " + strings.Join(parts, "; ") + "."
}
// Review is what recording a run's actions found about its cause.
type Review struct {
// Repeats is how many incidents on the target have this cause, this one
// included.
Repeats int
// Proposal is the root-cause fix asked for, empty until the cause has
// recurred enough.
Proposal string
}
// ledger is where incidents and proposals are kept.
type ledger interface {
Upsert(ctx context.Context, orgID string, in store.Incident) error
SetActions(ctx context.Context, orgID string, runID uuid.UUID, actions []byte) error
Incident(ctx context.Context, orgID string, runID uuid.UUID) (store.Incident, bool, error)
ByTarget(ctx context.Context, orgID, kind, id string, limit int) ([]store.Incident, error)
Propose(ctx context.Context, orgID string, p store.Proposal) (store.Proposal, error)
}
// Module remembers one organisation's incidents.
type Module struct {
orgID string
ledger ledger
}
// NewModule returns the memory module for orgID.
func NewModule(db *sqlx.DB, orgID string) *Module {
return &Module{orgID: orgID, ledger: store.New(db)}
}
var unsafeCause = regexp.MustCompile(`[^a-z0-9]+`)
// Cause is a cause as it is compared: lower case words joined by hyphens, so
// "Disk pressure" and "disk-pressure" are one cause.
func Cause(s string) string {
return strings.Trim(unsafeCause.ReplaceAllString(strings.ToLower(s), "-"), "-")
}
// Record remembers what a run's investigation found: the event, the target it
// concerned and the cause. Recording again replaces the earlier attempt's.
func (m *Module) Record(ctx context.Context, runID uuid.UUID, event jsontext.Value, t Target, cause, summary string) error {
t = Target{Kind: strings.TrimSpace(t.Kind), ID: strings.TrimSpace(t.ID)}
cause = Cause(cause)
summary = strings.TrimSpace(summary)
if t.Kind == "" || t.ID == "" || cause == "" || summary == "" {
return errors.New("a target kind, a target id, a cause and a summary are all required")
}
if !event.IsValid() {
event = jsontext.Value(`null`)
}
return m.ledger.Upsert(ctx, m.orgID, store.Incident{
ID: uuid.New(), RunID: runID, TargetKind: t.Kind, TargetID: t.ID, Cause: cause, Summary: summary, Event: event,
})
}
// Past returns the incidents recorded on t before the run's own, newest first.
func (m *Module) Past(ctx context.Context, runID uuid.UUID, t Target) ([]Incident, error) {
rows, err := m.ledger.ByTarget(ctx, m.orgID, strings.TrimSpace(t.Kind), strings.TrimSpace(t.ID), maxHistory)
if err != nil {
return nil, err
}
out := make([]Incident, 0, len(rows))
for _, r := range rows {
if r.RunID == runID {
continue
}
in, err := incident(r)
if err != nil {
return nil, err
}
out = append(out, in)
}
return out, nil
}
// Finish remembers what was done in a run's incident and checks whether its
// cause has come back enough to ask for a root-cause fix: that is when
// threshold incidents on the target share the cause, whatever their outcomes.
// A run that recorded no incident has nothing to finish.
func (m *Module) Finish(ctx context.Context, runID uuid.UUID, actions []Action, threshold int) (Review, error) {
current, ok, err := m.ledger.Incident(ctx, m.orgID, runID)
if err != nil || !ok {
return Review{}, err
}
raw, err := json.Marshal(actions)
if err != nil {
return Review{}, fmt.Errorf("encode actions: %w", err)
}
if err := m.ledger.SetActions(ctx, m.orgID, runID, raw); err != nil {
return Review{}, err
}
rows, err := m.ledger.ByTarget(ctx, m.orgID, current.TargetKind, current.TargetID, maxHistory)
if err != nil {
return Review{}, err
}
var same []store.Incident
for _, r := range rows {
if r.Cause == current.Cause {
same = append(same, r)
}
}
review := Review{Repeats: len(same)}
if review.Repeats < threshold {
return review, nil
}
proposal, err := m.ledger.Propose(ctx, m.orgID, store.Proposal{
ID: uuid.New(), TargetKind: current.TargetKind, TargetID: current.TargetID, Cause: current.Cause,
Incidents: review.Repeats, Body: proposalBody(current, same),
})
if err != nil {
return Review{}, err
}
review.Proposal = proposal.Body
return review, nil
}
func incident(r store.Incident) (Incident, error) {
out := Incident{At: r.At, Cause: r.Cause, Summary: r.Summary}
if len(r.Actions) > 0 {
if err := json.Unmarshal(r.Actions, &out.Actions); err != nil {
return Incident{}, fmt.Errorf("decode actions of incident %s: %w", r.ID, err)
}
}
return out, nil
}
// proposalBody asks for the cause to be fixed rather than handled again. same
// is newest first.
func proposalBody(current store.Incident, same []store.Incident) string {
var tried []string
for _, r := range same {
in, err := incident(r)
if err != nil {
continue
}
for _, a := range in.Actions {
if line := a.Action + " on " + a.Target; !slices.Contains(tried, line) {
tried = append(tried, line)
}
}
}
body := fmt.Sprintf("%q has been the cause of %d incidents on %s %s since %s. Handling each one has not stopped it, so find what keeps producing it and fix that.",
current.Cause, len(same), current.TargetKind, current.TargetID, same[len(same)-1].At.Format("2006-01-02"))
if len(tried) > 0 {
body += " Handled so far with: " + strings.Join(tried, "; ") + "."
}
return body
}