356 lines
12 KiB
Go
356 lines
12 KiB
Go
package archied
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"maps"
|
|
"net/http"
|
|
"slices"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/samcharles93/archie-core/internal/config"
|
|
"github.com/samcharles93/archie-core/internal/domain/alert"
|
|
"github.com/samcharles93/archie-core/internal/domain/org"
|
|
"github.com/samcharles93/archie-core/internal/infrastructure/alertwebhook"
|
|
"github.com/samcharles93/archie-core/internal/infrastructure/controlplanerpc"
|
|
"github.com/samcharles93/archie-core/internal/infrastructure/modelcatalog"
|
|
"github.com/samcharles93/archie-core/internal/releaseupdate"
|
|
)
|
|
|
|
const (
|
|
// deliveryInterval is how often due webhook deliveries are claimed.
|
|
deliveryInterval = 15 * time.Second
|
|
// releaseCheckInterval is how often GitHub is asked for a newer release.
|
|
releaseCheckInterval = 6 * time.Hour
|
|
// alertCallTimeout bounds one check's calls to the State Store.
|
|
alertCallTimeout = 20 * time.Second
|
|
// credentialReportEvery is how long an unchanged credential state goes
|
|
// unreported, so the egress path does not call the State Store per request.
|
|
credentialReportEvery = 10 * time.Minute
|
|
)
|
|
|
|
// alertChecks raises the alerts that have no transition to hang on: the
|
|
// daemon's configuration and the release channel. Each check lists what it
|
|
// owns, so alerts from before a restart are still cleared once fixed.
|
|
type alertChecks struct {
|
|
store alert.Store
|
|
log func(msg string, args ...any)
|
|
cfg func() config.Config
|
|
// catalog is the models the providers currently offer.
|
|
catalog func() modelcatalog.Snapshot
|
|
// release is the newest release the update channel offers.
|
|
release func(context.Context) (releaseupdate.Snapshot, error)
|
|
}
|
|
|
|
func (c alertChecks) reporter() alert.Reporter {
|
|
return alert.Reporter{Raiser: c.store, By: "archied"}
|
|
}
|
|
|
|
// sync makes the open alerts of kind in orgID match active: it raises each
|
|
// input and resolves the alerts, snoozed or open, whose condition cleared.
|
|
func (c alertChecks) sync(ctx context.Context, orgID org.OrgID, kind string, active []alert.Input) {
|
|
reporter := c.reporter()
|
|
subjects := make([]string, 0, len(active))
|
|
for _, in := range active {
|
|
reporter.Raise(ctx, in)
|
|
subjects = append(subjects, in.Subject)
|
|
}
|
|
existing, err := c.store.ListAlerts(ctx, orgID, alert.Filter{States: []alert.State{alert.StateOpen, alert.StateDismissed}, Limit: 500})
|
|
if err != nil {
|
|
c.log("alerts not checked", "kind", kind, "err", err)
|
|
return
|
|
}
|
|
for _, a := range existing {
|
|
if a.Kind == kind && a.Condition && !slices.Contains(subjects, a.Subject) {
|
|
reporter.Resolve(ctx, a.Key)
|
|
}
|
|
}
|
|
}
|
|
|
|
// checkConfig raises the alerts the effective configuration implies: a
|
|
// missing or dangling model alias, and providers and granted services with no
|
|
// usable credential.
|
|
func (c alertChecks) checkConfig(ctx context.Context) {
|
|
cfg := c.cfg()
|
|
c.sync(ctx, org.DefaultOrgID, alert.KindModelAlias, modelAliasAlerts(cfg.Models, c.catalog()))
|
|
c.sync(ctx, org.DefaultOrgID, alert.KindProviderNoKey, providerAlerts(cfg.Providers))
|
|
unbound := unboundAlerts(cfg)
|
|
for _, orgID := range credentialOrgs(cfg) {
|
|
c.sync(ctx, orgID, alert.KindToolNoBinding, unbound[orgID])
|
|
}
|
|
}
|
|
|
|
func modelAliasAlerts(aliases map[string]string, catalog modelcatalog.Snapshot) []alert.Input {
|
|
var out []alert.Input
|
|
input := func(alias, title, body string) alert.Input {
|
|
return alert.Input{
|
|
Org: org.DefaultOrgID, Kind: alert.KindModelAlias, Subject: "alias:" + alias,
|
|
Severity: alert.SeverityAction, Condition: true, Title: title, Body: body, Link: "/settings/models",
|
|
}
|
|
}
|
|
if strings.TrimSpace(aliases[config.DefaultModelAlias]) == "" {
|
|
out = append(out, input(config.DefaultModelAlias, "No default model alias is set",
|
|
"Set it to a model."))
|
|
}
|
|
chat := config.ChatAliases(aliases)
|
|
for _, alias := range slices.Sorted(maps.Keys(aliases)) {
|
|
// The catalogue lists tool-calling chat models only, so it cannot say
|
|
// whether an embedding or transcription model still exists.
|
|
if !slices.Contains(chat, alias) {
|
|
continue
|
|
}
|
|
providerID, modelID, ok := strings.Cut(aliases[alias], "/")
|
|
if !ok {
|
|
continue
|
|
}
|
|
offered := catalogModels(catalog, providerID)
|
|
// A provider the catalogue does not list is a credential problem, not
|
|
// a retired model.
|
|
if len(offered) > 0 && !slices.Contains(offered, modelID) {
|
|
out = append(out, input(alias, fmt.Sprintf("Model alias %q points at a retired model", alias),
|
|
fmt.Sprintf("%s no longer offers %s. Choose another model.", providerID, modelID)))
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func catalogModels(catalog modelcatalog.Snapshot, providerID string) []string {
|
|
for _, provider := range catalog.Providers {
|
|
if provider.ID == providerID {
|
|
ids := make([]string, len(provider.Models))
|
|
for i, model := range provider.Models {
|
|
ids[i] = model.ID
|
|
}
|
|
return ids
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func providerAlerts(providers map[string]config.Provider) []alert.Input {
|
|
var out []alert.Input
|
|
for _, name := range slices.Sorted(maps.Keys(providers)) {
|
|
if !providers[name].Disabled {
|
|
continue
|
|
}
|
|
out = append(out, alert.Input{
|
|
Org: org.DefaultOrgID, Kind: alert.KindProviderNoKey, Subject: "provider:" + name,
|
|
Severity: alert.SeverityAction, Condition: true,
|
|
Title: fmt.Sprintf("Provider %q has no usable credential", name),
|
|
Body: "Key unresolved; its models cannot run. Check the binding.",
|
|
Link: "/settings/credential-bindings",
|
|
})
|
|
}
|
|
return out
|
|
}
|
|
|
|
// credentialOrgs are the orgs whose identities are granted credentials, the
|
|
// instance org always among them.
|
|
func credentialOrgs(cfg config.Config) []org.OrgID {
|
|
orgs := []org.OrgID{org.DefaultOrgID}
|
|
for _, name := range append([]string{""}, identityNames(cfg)...) {
|
|
orgName, _ := cfg.CredentialAccess(name)
|
|
if id := org.OrgID(credentialOrg(orgName)); !slices.Contains(orgs, id) {
|
|
orgs = append(orgs, id)
|
|
}
|
|
}
|
|
return orgs
|
|
}
|
|
|
|
func identityNames(cfg config.Config) []string {
|
|
names := make([]string, 0, len(cfg.Identities))
|
|
for _, id := range cfg.Identities {
|
|
names = append(names, id.Name)
|
|
}
|
|
return names
|
|
}
|
|
|
|
func credentialOrg(name string) string {
|
|
if name == "" {
|
|
return string(org.DefaultOrgID)
|
|
}
|
|
return name
|
|
}
|
|
|
|
// unboundAlerts are the granted services with no credential binding in the
|
|
// granting identity's org, by org. A model provider needs no binding of its
|
|
// own here: its key is the provider's.
|
|
func unboundAlerts(cfg config.Config) map[org.OrgID][]alert.Input {
|
|
out := map[org.OrgID][]alert.Input{}
|
|
seen := map[alert.Key]bool{}
|
|
for _, name := range append([]string{""}, identityNames(cfg)...) {
|
|
orgName, granted := cfg.CredentialAccess(name)
|
|
orgID := org.OrgID(credentialOrg(orgName))
|
|
bound := cfg.Containers.BoundCredentials(orgName, granted, granted)
|
|
for _, service := range granted {
|
|
key := alert.Key{Org: orgID, Kind: alert.KindToolNoBinding, Subject: "service:" + service}
|
|
if _, ok := cfg.Providers[service]; ok || bound[service].Service != "" || seen[key] {
|
|
continue
|
|
}
|
|
seen[key] = true
|
|
out[orgID] = append(out[orgID], alert.Input{
|
|
Key: key, Severity: alert.SeverityAction, Condition: true,
|
|
Title: fmt.Sprintf("%q is granted but has no credential binding", service),
|
|
Body: "Runs cannot authenticate. Bind a credential.",
|
|
Link: "/settings/credential-bindings",
|
|
})
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// checkRelease raises an info alert while the update channel offers a newer
|
|
// release. A failed check changes nothing: the next one decides.
|
|
func (c alertChecks) checkRelease(ctx context.Context) {
|
|
snapshot, err := c.release(ctx)
|
|
if err != nil {
|
|
c.log("release check failed", "err", err)
|
|
return
|
|
}
|
|
var active []alert.Input
|
|
if available := snapshot.Available(); len(available) > 0 {
|
|
parts := make([]string, len(available))
|
|
for i, component := range available {
|
|
parts[i] = fmt.Sprintf("%s %s", component.Label, component.Available)
|
|
}
|
|
active = append(active, alert.Input{
|
|
Org: org.DefaultOrgID, Kind: alert.KindReleaseAvailable, Subject: "release",
|
|
Severity: alert.SeverityInfo, Condition: true,
|
|
Title: "Newer release available", Body: strings.Join(parts, ", "),
|
|
Link: "/settings/status",
|
|
})
|
|
}
|
|
c.sync(ctx, org.DefaultOrgID, alert.KindReleaseAvailable, active)
|
|
}
|
|
|
|
// every runs fn now and then each interval until ctx ends.
|
|
func every(ctx context.Context, interval time.Duration, fn func(context.Context)) {
|
|
ticker := time.NewTicker(interval)
|
|
defer ticker.Stop()
|
|
for {
|
|
callCtx, cancel := context.WithTimeout(ctx, alertCallTimeout)
|
|
fn(callCtx)
|
|
cancel()
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
}
|
|
}
|
|
}
|
|
|
|
// startAlerts starts the daemon's alert checks and the webhook delivery loop.
|
|
// The State Store being away costs the checks a tick, nothing else.
|
|
func (b *boot) startAlerts(ctx context.Context) {
|
|
client, ok := b.stateStore.(alert.Store)
|
|
if !ok {
|
|
return
|
|
}
|
|
reader := controlplanerpc.NewRPCClient(b.controlPlaneRPC)
|
|
catalog := controlplanerpc.UpdateService(reader, "").Catalog
|
|
checks := alertChecks{
|
|
store: client, log: b.log.Warn, cfg: b.cfgHolder.Get, catalog: b.catalogState,
|
|
release: func(ctx context.Context) (releaseupdate.Snapshot, error) { return catalog.Check(ctx) },
|
|
}
|
|
b.alertChecks = &checks
|
|
go every(ctx, releaseCheckInterval, checks.checkRelease)
|
|
go every(ctx, deliveryInterval, b.deliverAlerts(client))
|
|
checks.checkConfig(ctx)
|
|
}
|
|
|
|
// recheckAlerts re-evaluates the configuration alerts after the configuration
|
|
// changed. It never blocks the change that caused it.
|
|
func (b *boot) recheckAlerts(ctx context.Context) {
|
|
checks := b.alertChecks
|
|
if checks == nil {
|
|
return
|
|
}
|
|
go func() {
|
|
ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), alertCallTimeout)
|
|
defer cancel()
|
|
checks.checkConfig(ctx)
|
|
}()
|
|
}
|
|
|
|
// deliverAlerts sends due webhook deliveries. A delivery the webhook is not
|
|
// configured for, or whose alert is below the severity threshold, is closed
|
|
// as skipped; one that fails is retried by the store on its schedule.
|
|
func (b *boot) deliverAlerts(store alert.Deliverer) func(context.Context) {
|
|
client := &http.Client{Timeout: 15 * time.Second}
|
|
return func(ctx context.Context) {
|
|
due, err := store.ClaimAlertDeliveries(ctx, 20)
|
|
if err != nil {
|
|
b.log.Warn("alert deliveries not claimed", "err", err)
|
|
return
|
|
}
|
|
notify := b.cfgHolder.Get().Notify
|
|
for _, d := range due {
|
|
outcome := alert.Outcome{DeliveryID: d.ID}
|
|
threshold := alert.Severity(notify.MinSeverity)
|
|
if !threshold.Valid() {
|
|
threshold = alert.SeverityWarning
|
|
}
|
|
if notify.Webhook == "" || d.Alert.Severity.Rank() < threshold.Rank() {
|
|
outcome.Skipped = true
|
|
} else {
|
|
secret := ""
|
|
if notify.Secret != (config.SecretRef{}) {
|
|
secret, err = b.secrets.Resolve(notify.Secret)
|
|
}
|
|
if err != nil {
|
|
outcome.Error = "notify secret could not be resolved"
|
|
} else {
|
|
outcome = alertwebhook.Send(ctx, client, notify.Webhook, secret, d)
|
|
}
|
|
}
|
|
if err := store.RecordAlertAttempt(ctx, outcome); err != nil {
|
|
b.log.Warn("alert delivery not recorded", "delivery", d.ID, "err", err)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// credentialWatch raises an alert when a bound credential cannot be resolved
|
|
// and clears it once it can. The egress path calls observe per request, so an
|
|
// unchanged state is reported at most every credentialReportEvery.
|
|
type credentialWatch struct {
|
|
reporter alert.Reporter
|
|
mu sync.Mutex
|
|
last map[alert.Key]credentialState
|
|
}
|
|
|
|
type credentialState struct {
|
|
rejected bool
|
|
at time.Time
|
|
}
|
|
|
|
func (w *credentialWatch) observe(ctx context.Context, orgName, service string, rejected bool) {
|
|
if w == nil {
|
|
return
|
|
}
|
|
key := alert.Key{Org: org.OrgID(credentialOrg(orgName)), Kind: alert.KindCredentialInvalid, Subject: "service:" + service}
|
|
w.mu.Lock()
|
|
if w.last == nil {
|
|
w.last = map[alert.Key]credentialState{}
|
|
}
|
|
previous, known := w.last[key]
|
|
if known && previous.rejected == rejected && time.Since(previous.at) < credentialReportEvery {
|
|
w.mu.Unlock()
|
|
return
|
|
}
|
|
w.last[key] = credentialState{rejected: rejected, at: time.Now()}
|
|
w.mu.Unlock()
|
|
go func() {
|
|
ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), alertCallTimeout)
|
|
defer cancel()
|
|
w.reporter.Sync(ctx, rejected, alert.Input{
|
|
Key: key, Severity: alert.SeverityAction, Condition: true,
|
|
Title: fmt.Sprintf("The credential bound to %q is not working", service),
|
|
Body: "Secret unresolved; runs cannot authenticate. Check the binding.",
|
|
Link: "/settings/credential-bindings",
|
|
})
|
|
}()
|
|
}
|