Also fix repository-free message.send validation (#2046) and add a single-binary development smoke path (#1977). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
216 lines
7.0 KiB
Go
216 lines
7.0 KiB
Go
// Package memory records investigated incidents by target, including their
|
|
// causes and action outcomes. When a cause reaches the recurrence threshold
|
|
// for a target, the module proposes a root-cause fix regardless of action outcomes.
|
|
package memory
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json/jsontext"
|
|
json "encoding/json/v2"
|
|
"errors"
|
|
"fmt"
|
|
"regexp"
|
|
"slices"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/jmoiron/sqlx"
|
|
|
|
"github.com/samcharles93/archie-core/modules/memory/internal/store"
|
|
)
|
|
|
|
// maxHistory bounds the incidents read for one target. The recurrence setting
|
|
// is capped below it.
|
|
const maxHistory = 200
|
|
|
|
// MaxRecurrence is the largest recurrence threshold the history can confirm.
|
|
const MaxRecurrence = 100
|
|
|
|
// Target identifies the affected resource by kind and ID, such as host web-1.
|
|
type Target struct {
|
|
Kind string `json:"kind"`
|
|
ID string `json:"id"`
|
|
}
|
|
|
|
// Action records an action considered during an incident and its outcome.
|
|
type Action struct {
|
|
Action string `json:"action"`
|
|
Target string `json:"target"`
|
|
// State records whether the action ran, was refused or has an approval status.
|
|
State string `json:"state"`
|
|
Detail string `json:"detail,omitzero"`
|
|
}
|
|
|
|
// Incident is one investigation of a target.
|
|
type Incident struct {
|
|
At time.Time `json:"at"`
|
|
Cause string `json:"cause"`
|
|
Summary string `json:"summary"`
|
|
Actions []Action `json:"actions"`
|
|
}
|
|
|
|
// Line formats the incident as one entry in the history shown to the agent.
|
|
func (i Incident) Line() string {
|
|
line := fmt.Sprintf("- %s, cause %q: %s", i.At.Format("2006-01-02"), i.Cause, i.Summary)
|
|
if len(i.Actions) == 0 {
|
|
return line + " Actions: none."
|
|
}
|
|
parts := make([]string, 0, len(i.Actions))
|
|
for _, a := range i.Actions {
|
|
part := a.Action + " on " + a.Target + ": " + a.State
|
|
if a.Detail != "" {
|
|
part += " (" + a.Detail + ")"
|
|
}
|
|
parts = append(parts, part)
|
|
}
|
|
return line + " Actions: " + strings.Join(parts, "; ") + "."
|
|
}
|
|
|
|
// Review reports how often the cause occurred and any proposal for a fix.
|
|
type Review struct {
|
|
// Repeats counts matching incidents in the target's recent history,
|
|
// including the current incident if it is in that history.
|
|
Repeats int
|
|
// Proposal is the text of the request for a root-cause fix. It is empty
|
|
// below the recurrence threshold.
|
|
Proposal string
|
|
}
|
|
|
|
type ledger interface {
|
|
Upsert(ctx context.Context, orgID string, in store.Incident) error
|
|
SetActions(ctx context.Context, orgID string, runID uuid.UUID, actions []byte) error
|
|
Incident(ctx context.Context, orgID string, runID uuid.UUID) (store.Incident, bool, error)
|
|
ByTarget(ctx context.Context, orgID, kind, id string, limit int) ([]store.Incident, error)
|
|
Propose(ctx context.Context, orgID string, p store.Proposal) (store.Proposal, error)
|
|
}
|
|
|
|
// Module manages incident history and proposals for one organisation.
|
|
type Module struct {
|
|
orgID string
|
|
ledger ledger
|
|
}
|
|
|
|
// NewModule returns the memory module for orgID.
|
|
func NewModule(db *sqlx.DB, orgID string) *Module {
|
|
return &Module{orgID: orgID, ledger: store.New(db)}
|
|
}
|
|
|
|
var unsafeCause = regexp.MustCompile(`[^a-z0-9]+`)
|
|
|
|
// Cause normalises labels for comparison: "Disk pressure" and "disk-pressure"
|
|
// both produce "disk-pressure".
|
|
func Cause(s string) string {
|
|
return strings.Trim(unsafeCause.ReplaceAllString(strings.ToLower(s), "-"), "-")
|
|
}
|
|
|
|
// Record saves the incident for a run. Recording it again updates the target,
|
|
// cause and summary without replacing the original event.
|
|
func (m *Module) Record(ctx context.Context, runID uuid.UUID, event jsontext.Value, t Target, cause, summary string) error {
|
|
t = Target{Kind: strings.TrimSpace(t.Kind), ID: strings.TrimSpace(t.ID)}
|
|
cause = Cause(cause)
|
|
summary = strings.TrimSpace(summary)
|
|
if t.Kind == "" || t.ID == "" || cause == "" || summary == "" {
|
|
return errors.New("a target kind, a target id, a cause and a summary are all required")
|
|
}
|
|
if !event.IsValid() {
|
|
event = jsontext.Value(`null`)
|
|
}
|
|
return m.ledger.Upsert(ctx, m.orgID, store.Incident{
|
|
ID: uuid.New(), RunID: runID, TargetKind: t.Kind, TargetID: t.ID, Cause: cause, Summary: summary, Event: event,
|
|
})
|
|
}
|
|
|
|
// Past returns recent incidents for t, newest first, excluding the current run.
|
|
func (m *Module) Past(ctx context.Context, runID uuid.UUID, t Target) ([]Incident, error) {
|
|
rows, err := m.ledger.ByTarget(ctx, m.orgID, strings.TrimSpace(t.Kind), strings.TrimSpace(t.ID), maxHistory)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out := make([]Incident, 0, len(rows))
|
|
for _, r := range rows {
|
|
if r.RunID == runID {
|
|
continue
|
|
}
|
|
in, err := incident(r)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out = append(out, in)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// Finish records action outcomes. It requests a root-cause fix if at least
|
|
// threshold incidents in the target's recent history share the current cause,
|
|
// regardless of their action outcomes. A run with no recorded incident is ignored.
|
|
func (m *Module) Finish(ctx context.Context, runID uuid.UUID, actions []Action, threshold int) (Review, error) {
|
|
current, ok, err := m.ledger.Incident(ctx, m.orgID, runID)
|
|
if err != nil || !ok {
|
|
return Review{}, err
|
|
}
|
|
raw, err := json.Marshal(actions)
|
|
if err != nil {
|
|
return Review{}, fmt.Errorf("encode actions: %w", err)
|
|
}
|
|
if err := m.ledger.SetActions(ctx, m.orgID, runID, raw); err != nil {
|
|
return Review{}, err
|
|
}
|
|
rows, err := m.ledger.ByTarget(ctx, m.orgID, current.TargetKind, current.TargetID, maxHistory)
|
|
if err != nil {
|
|
return Review{}, err
|
|
}
|
|
var same []store.Incident
|
|
for _, r := range rows {
|
|
if r.Cause == current.Cause {
|
|
same = append(same, r)
|
|
}
|
|
}
|
|
review := Review{Repeats: len(same)}
|
|
if review.Repeats < threshold {
|
|
return review, nil
|
|
}
|
|
proposal, err := m.ledger.Propose(ctx, m.orgID, store.Proposal{
|
|
ID: uuid.New(), TargetKind: current.TargetKind, TargetID: current.TargetID, Cause: current.Cause,
|
|
Incidents: review.Repeats, Body: proposalBody(current, same),
|
|
})
|
|
if err != nil {
|
|
return Review{}, err
|
|
}
|
|
review.Proposal = proposal.Body
|
|
return review, nil
|
|
}
|
|
|
|
func incident(r store.Incident) (Incident, error) {
|
|
out := Incident{At: r.At, Cause: r.Cause, Summary: r.Summary}
|
|
if len(r.Actions) > 0 {
|
|
if err := json.Unmarshal(r.Actions, &out.Actions); err != nil {
|
|
return Incident{}, fmt.Errorf("decode actions of incident %s: %w", r.ID, err)
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// proposalBody expects matching incidents in newest-first order so it can
|
|
// report the date of the oldest match.
|
|
func proposalBody(current store.Incident, same []store.Incident) string {
|
|
var tried []string
|
|
for _, r := range same {
|
|
in, err := incident(r)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
for _, a := range in.Actions {
|
|
if line := a.Action + " on " + a.Target; !slices.Contains(tried, line) {
|
|
tried = append(tried, line)
|
|
}
|
|
}
|
|
}
|
|
body := fmt.Sprintf("%q caused %d recorded incidents on %s %s since %s. Identify and fix the underlying problem.",
|
|
current.Cause, len(same), current.TargetKind, current.TargetID, same[len(same)-1].At.Format("2006-01-02"))
|
|
if len(tried) > 0 {
|
|
body += " Recorded actions: " + strings.Join(tried, "; ") + "."
|
|
}
|
|
return body
|
|
}
|