From bb0d7fa8d0f4d77476f6290c72778781b2ef2bf9 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Mon, 26 Oct 2026 19:48:09 +1100 Subject: [PATCH] feat(archied): raise an alert when container max uptime is under the task limit --- internal/app/archied/alerts.go | 17 +++++++++++++++++ internal/app/archied/control_plane.go | 3 --- internal/domain/alert/alert.go | 1 + 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/internal/app/archied/alerts.go b/internal/app/archied/alerts.go index 5a5031fe2..a2943455a 100644 --- a/internal/app/archied/alerts.go +++ b/internal/app/archied/alerts.go @@ -76,6 +76,7 @@ func (c alertChecks) checkConfig(ctx context.Context) { cfg := c.cfg() c.sync(ctx, org.DefaultOrgID, alert.KindModelAlias, modelAliasAlerts(cfg.Models, c.catalog())) c.sync(ctx, org.DefaultOrgID, alert.KindProviderNoKey, providerAlerts(cfg.Providers)) + c.sync(ctx, org.DefaultOrgID, alert.KindUptimeShort, uptimeAlerts(cfg)) unbound := unboundAlerts(cfg) for _, orgID := range credentialOrgs(cfg) { c.sync(ctx, orgID, alert.KindToolNoBinding, unbound[orgID]) @@ -129,6 +130,22 @@ func catalogModels(catalog modelcatalog.Snapshot, providerID string) []string { return nil } +// uptimeAlerts flags a container uptime shorter than the task time limit: the +// container is killed before the task can reach its own limit. +func uptimeAlerts(cfg config.Config) []alert.Input { + uptime, limit := cfg.Containers.MaxUptime.Std(), cfg.Budgets.TaskWallClock.Std() + if uptime <= 0 || (limit > 0 && limit <= uptime) { + return nil + } + return []alert.Input{{ + Org: org.DefaultOrgID, Kind: alert.KindUptimeShort, Subject: "containers.max_uptime", + Severity: alert.SeverityWarning, Condition: true, + Title: "Container max uptime is shorter than the task time limit", + Body: "Tasks are killed at the uptime before they reach their limit. Raise the max uptime.", + Link: "/settings/container-runtime", + }} +} + func providerAlerts(providers map[string]config.Provider) []alert.Input { var out []alert.Input for _, name := range slices.Sorted(maps.Keys(providers)) { diff --git a/internal/app/archied/control_plane.go b/internal/app/archied/control_plane.go index 1ed939470..0d113feb4 100644 --- a/internal/app/archied/control_plane.go +++ b/internal/app/archied/control_plane.go @@ -201,9 +201,6 @@ func (b *boot) applyWorkflowExecutionSettings(ctx context.Context, settings work b.publishConfig(ctx, candidate.cfg) b.applyStatus.Report(ctx, controlplanestore.WorkflowExecutionSettingsKind, version, nil) b.log.InfoContext(ctx, "workflow execution settings applied", "version", version) - if limit, uptime := settings.MaxTaskRuntime, candidate.cfg.Containers.MaxUptime.Std(); uptime > 0 && (limit <= 0 || limit > uptime) { - b.log.WarnContext(ctx, "containers.max_uptime is shorter than the task time limit; the container is killed first", "max_uptime", uptime, "max_task_runtime", limit) - } return nil } diff --git a/internal/domain/alert/alert.go b/internal/domain/alert/alert.go index 7436a551a..78ec1d290 100644 --- a/internal/domain/alert/alert.go +++ b/internal/domain/alert/alert.go @@ -72,6 +72,7 @@ const ( KindReleaseAvailable = "release.available" KindPolicyInvalid = "policy.invalid" KindScheduleFailing = "schedule.failing" + KindUptimeShort = "container.uptime_short" ) // Limits on what a raiser may store.