Files
archie/internal/app/archied/alerts.go
T

373 lines
13 KiB
Go

package archied
import (
"context"
"fmt"
"maps"
"net/http"
"slices"
"strings"
"sync"
"time"
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/domain/alert"
"github.com/samcharles93/archie-core/internal/domain/org"
"github.com/samcharles93/archie-core/internal/infrastructure/alertwebhook"
"github.com/samcharles93/archie-core/internal/infrastructure/controlplanerpc"
"github.com/samcharles93/archie-core/internal/infrastructure/modelcatalog"
"github.com/samcharles93/archie-core/internal/releaseupdate"
)
const (
// deliveryInterval is how often due webhook deliveries are claimed.
deliveryInterval = 15 * time.Second
// releaseCheckInterval is how often GitHub is asked for a newer release.
releaseCheckInterval = 6 * time.Hour
// alertCallTimeout bounds one check's calls to the State Store.
alertCallTimeout = 20 * time.Second
// credentialReportEvery is how long an unchanged credential state goes
// unreported, so the egress path does not call the State Store per request.
credentialReportEvery = 10 * time.Minute
)
// alertChecks raises the alerts that have no transition to hang on: the
// daemon's configuration and the release channel. Each check lists what it
// owns, so alerts from before a restart are still cleared once fixed.
type alertChecks struct {
store alert.Store
log func(msg string, args ...any)
cfg func() config.Config
// catalog is the models the providers currently offer.
catalog func() modelcatalog.Snapshot
// release is the newest release the update channel offers.
release func(context.Context) (releaseupdate.Snapshot, error)
}
func (c alertChecks) reporter() alert.Reporter {
return alert.Reporter{Raiser: c.store, By: "archied"}
}
// sync makes the open alerts of kind in orgID match active: it raises each
// input and resolves the alerts, snoozed or open, whose condition cleared.
func (c alertChecks) sync(ctx context.Context, orgID org.OrgID, kind string, active []alert.Input) {
reporter := c.reporter()
subjects := make([]string, 0, len(active))
for _, in := range active {
reporter.Raise(ctx, in)
subjects = append(subjects, in.Subject)
}
existing, err := c.store.ListAlerts(ctx, orgID, alert.Filter{States: []alert.State{alert.StateOpen, alert.StateDismissed}, Limit: 500})
if err != nil {
c.log("alerts not checked", "kind", kind, "err", err)
return
}
for _, a := range existing {
if a.Kind == kind && a.Condition && !slices.Contains(subjects, a.Subject) {
reporter.Resolve(ctx, a.Key)
}
}
}
// checkConfig raises the alerts the effective configuration implies: a
// missing or dangling model alias, and providers and granted services with no
// usable credential.
func (c alertChecks) checkConfig(ctx context.Context) {
cfg := c.cfg()
c.sync(ctx, org.DefaultOrgID, alert.KindModelAlias, modelAliasAlerts(cfg.Models, c.catalog()))
c.sync(ctx, org.DefaultOrgID, alert.KindProviderNoKey, providerAlerts(cfg.Providers))
c.sync(ctx, org.DefaultOrgID, alert.KindUptimeShort, uptimeAlerts(cfg))
unbound := unboundAlerts(cfg)
for _, orgID := range credentialOrgs(cfg) {
c.sync(ctx, orgID, alert.KindToolNoBinding, unbound[orgID])
}
}
func modelAliasAlerts(aliases map[string]string, catalog modelcatalog.Snapshot) []alert.Input {
var out []alert.Input
input := func(alias, title, body string) alert.Input {
return alert.Input{
Org: org.DefaultOrgID, Kind: alert.KindModelAlias, Subject: "alias:" + alias,
Severity: alert.SeverityAction, Condition: true, Title: title, Body: body, Link: "/settings/models",
}
}
if strings.TrimSpace(aliases[config.DefaultModelAlias]) == "" {
out = append(out, input(config.DefaultModelAlias, "No default model alias is set",
"Set it to a model."))
}
chat := config.ChatAliases(aliases)
for _, alias := range slices.Sorted(maps.Keys(aliases)) {
// The catalogue lists tool-calling chat models only, so it cannot say
// whether an embedding or transcription model still exists.
if !slices.Contains(chat, alias) {
continue
}
providerID, modelID, ok := strings.Cut(aliases[alias], "/")
if !ok {
continue
}
offered := catalogModels(catalog, providerID)
// A provider the catalogue does not list is a credential problem, not
// a retired model.
if len(offered) > 0 && !slices.Contains(offered, modelID) {
out = append(out, input(alias, fmt.Sprintf("Model alias %q points at a retired model", alias),
fmt.Sprintf("%s no longer offers %s. Choose another model.", providerID, modelID)))
}
}
return out
}
func catalogModels(catalog modelcatalog.Snapshot, providerID string) []string {
for _, provider := range catalog.Providers {
if provider.ID == providerID {
ids := make([]string, len(provider.Models))
for i, model := range provider.Models {
ids[i] = model.ID
}
return ids
}
}
return nil
}
// uptimeAlerts flags a container uptime shorter than the task time limit: the
// container is killed before the task can reach its own limit.
func uptimeAlerts(cfg config.Config) []alert.Input {
uptime, limit := cfg.Containers.MaxUptime.Std(), cfg.Budgets.TaskWallClock.Std()
if uptime <= 0 || (limit > 0 && limit <= uptime) {
return nil
}
return []alert.Input{{
Org: org.DefaultOrgID, Kind: alert.KindUptimeShort, Subject: "containers.max_uptime",
Severity: alert.SeverityWarning, Condition: true,
Title: "Container max uptime is shorter than the task time limit",
Body: "Tasks are killed at the uptime before they reach their limit. Raise the max uptime.",
Link: "/settings/container-runtime",
}}
}
func providerAlerts(providers map[string]config.Provider) []alert.Input {
var out []alert.Input
for _, name := range slices.Sorted(maps.Keys(providers)) {
if !providers[name].Disabled {
continue
}
out = append(out, alert.Input{
Org: org.DefaultOrgID, Kind: alert.KindProviderNoKey, Subject: "provider:" + name,
Severity: alert.SeverityAction, Condition: true,
Title: fmt.Sprintf("Provider %q has no usable credential", name),
Body: "Key unresolved; its models cannot run. Check the binding.",
Link: "/settings/credential-bindings",
})
}
return out
}
// credentialOrgs are the orgs whose identities are granted credentials, the
// instance org always among them.
func credentialOrgs(cfg config.Config) []org.OrgID {
orgs := []org.OrgID{org.DefaultOrgID}
for _, name := range append([]string{""}, identityNames(cfg)...) {
orgName, _ := cfg.CredentialAccess(name)
if id := org.OrgID(credentialOrg(orgName)); !slices.Contains(orgs, id) {
orgs = append(orgs, id)
}
}
return orgs
}
func identityNames(cfg config.Config) []string {
names := make([]string, 0, len(cfg.Identities))
for _, id := range cfg.Identities {
names = append(names, id.Name)
}
return names
}
func credentialOrg(name string) string {
if name == "" {
return string(org.DefaultOrgID)
}
return name
}
// unboundAlerts are the granted services with no credential binding in the
// granting identity's org, by org. A model provider needs no binding of its
// own here: its key is the provider's.
func unboundAlerts(cfg config.Config) map[org.OrgID][]alert.Input {
out := map[org.OrgID][]alert.Input{}
seen := map[alert.Key]bool{}
for _, name := range append([]string{""}, identityNames(cfg)...) {
orgName, granted := cfg.CredentialAccess(name)
orgID := org.OrgID(credentialOrg(orgName))
bound := cfg.Containers.BoundCredentials(orgName, granted, granted)
for _, service := range granted {
key := alert.Key{Org: orgID, Kind: alert.KindToolNoBinding, Subject: "service:" + service}
if _, ok := cfg.Providers[service]; ok || bound[service].Service != "" || seen[key] {
continue
}
seen[key] = true
out[orgID] = append(out[orgID], alert.Input{
Key: key, Severity: alert.SeverityAction, Condition: true,
Title: fmt.Sprintf("%q is granted but has no credential binding", service),
Body: "Runs cannot authenticate. Bind a credential.",
Link: "/settings/credential-bindings",
})
}
}
return out
}
// checkRelease raises an info alert while the update channel offers a newer
// release. A failed check changes nothing: the next one decides.
func (c alertChecks) checkRelease(ctx context.Context) {
snapshot, err := c.release(ctx)
if err != nil {
c.log("release check failed", "err", err)
return
}
var active []alert.Input
if available := snapshot.Available(); len(available) > 0 {
parts := make([]string, len(available))
for i, component := range available {
parts[i] = fmt.Sprintf("%s %s", component.Label, component.Available)
}
active = append(active, alert.Input{
Org: org.DefaultOrgID, Kind: alert.KindReleaseAvailable, Subject: "release",
Severity: alert.SeverityInfo, Condition: true,
Title: "Newer release available", Body: strings.Join(parts, ", "),
Link: "/settings/status",
})
}
c.sync(ctx, org.DefaultOrgID, alert.KindReleaseAvailable, active)
}
// every runs fn now and then each interval until ctx ends.
func every(ctx context.Context, interval time.Duration, fn func(context.Context)) {
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
callCtx, cancel := context.WithTimeout(ctx, alertCallTimeout)
fn(callCtx)
cancel()
select {
case <-ctx.Done():
return
case <-ticker.C:
}
}
}
// startAlerts starts the daemon's alert checks and the webhook delivery loop.
// The State Store being away costs the checks a tick, nothing else.
func (b *boot) startAlerts(ctx context.Context) {
client, ok := b.stateStore.(alert.Store)
if !ok {
return
}
reader := controlplanerpc.NewRPCClient(b.controlPlaneRPC)
catalog := controlplanerpc.UpdateService(reader, "").Catalog
checks := alertChecks{
store: client, log: b.log.Warn, cfg: b.cfgHolder.Get, catalog: b.catalogState,
release: func(ctx context.Context) (releaseupdate.Snapshot, error) { return catalog.Check(ctx) },
}
b.alertChecks = &checks
go every(ctx, releaseCheckInterval, checks.checkRelease)
go every(ctx, deliveryInterval, b.deliverAlerts(client))
checks.checkConfig(ctx)
}
// recheckAlerts re-evaluates the configuration alerts after the configuration
// changed. It never blocks the change that caused it.
func (b *boot) recheckAlerts(ctx context.Context) {
checks := b.alertChecks
if checks == nil {
return
}
go func() {
ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), alertCallTimeout)
defer cancel()
checks.checkConfig(ctx)
}()
}
// deliverAlerts sends due webhook deliveries. A delivery the webhook is not
// configured for, or whose alert is below the severity threshold, is closed
// as skipped; one that fails is retried by the store on its schedule.
func (b *boot) deliverAlerts(store alert.Deliverer) func(context.Context) {
client := &http.Client{Timeout: 15 * time.Second}
return func(ctx context.Context) {
due, err := store.ClaimAlertDeliveries(ctx, 20)
if err != nil {
b.log.WarnContext(ctx, "alert deliveries not claimed", "err", err)
return
}
notify := b.cfgHolder.Get().Notify
for _, d := range due {
outcome := alert.Outcome{DeliveryID: d.ID}
threshold := alert.Severity(notify.MinSeverity)
if !threshold.Valid() {
threshold = alert.SeverityWarning
}
if notify.Webhook == "" || d.Alert.Severity.Rank() < threshold.Rank() {
outcome.Skipped = true
} else {
secret := ""
if notify.Secret != (config.SecretRef{}) {
secret, err = b.secrets.Resolve(notify.Secret)
}
if err != nil {
outcome.Error = "notify secret could not be resolved"
} else {
outcome = alertwebhook.Send(ctx, client, notify.Webhook, secret, d)
}
}
if err := store.RecordAlertAttempt(ctx, outcome); err != nil {
b.log.WarnContext(ctx, "alert delivery not recorded", "delivery", d.ID, "err", err)
}
}
}
}
// credentialWatch raises an alert when a bound credential cannot be resolved
// and clears it once it can. The egress path calls observe per request, so an
// unchanged state is reported at most every credentialReportEvery.
type credentialWatch struct {
reporter alert.Reporter
mu sync.Mutex
last map[alert.Key]credentialState
}
type credentialState struct {
rejected bool
at time.Time
}
func (w *credentialWatch) observe(ctx context.Context, orgName, service string, rejected bool) {
if w == nil {
return
}
key := alert.Key{Org: org.OrgID(credentialOrg(orgName)), Kind: alert.KindCredentialInvalid, Subject: "service:" + service}
w.mu.Lock()
if w.last == nil {
w.last = map[alert.Key]credentialState{}
}
previous, known := w.last[key]
if known && previous.rejected == rejected && time.Since(previous.at) < credentialReportEvery {
w.mu.Unlock()
return
}
w.last[key] = credentialState{rejected: rejected, at: time.Now()}
w.mu.Unlock()
go func() {
ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), alertCallTimeout)
defer cancel()
w.reporter.Sync(ctx, rejected, alert.Input{
Key: key, Severity: alert.SeverityAction, Condition: true,
Title: fmt.Sprintf("The credential bound to %q is not working", service),
Body: "Secret unresolved; runs cannot authenticate. Check the binding.",
Link: "/settings/credential-bindings",
})
}()
}