feat(archied): raise an alert when container max uptime is under the task limit

This commit is contained in:
sam committed 2026-10-26 19:48:09 +11:00
1 parent e72234bcb7
commit bb0d7fa8d0
3 files changed
+18 -3

No files matched your search

+17
View File
@@ -76,6 +76,7 @@ func (c alertChecks) checkConfig(ctx context.Context) {
cfg := c.cfg()
c.sync(ctx, org.DefaultOrgID, alert.KindModelAlias, modelAliasAlerts(cfg.Models, c.catalog()))
c.sync(ctx, org.DefaultOrgID, alert.KindProviderNoKey, providerAlerts(cfg.Providers))
c.sync(ctx, org.DefaultOrgID, alert.KindUptimeShort, uptimeAlerts(cfg))
unbound := unboundAlerts(cfg)
for _, orgID := range credentialOrgs(cfg) {
c.sync(ctx, orgID, alert.KindToolNoBinding, unbound[orgID])
@@ -129,6 +130,22 @@ func catalogModels(catalog modelcatalog.Snapshot, providerID string) []string {
return nil
}
// uptimeAlerts flags a container uptime shorter than the task time limit: the
// container is killed before the task can reach its own limit.
func uptimeAlerts(cfg config.Config) []alert.Input {
uptime, limit := cfg.Containers.MaxUptime.Std(), cfg.Budgets.TaskWallClock.Std()
if uptime <= 0 || (limit > 0 && limit <= uptime) {
return nil
}
return []alert.Input{{
Org: org.DefaultOrgID, Kind: alert.KindUptimeShort, Subject: "containers.max_uptime",
Severity: alert.SeverityWarning, Condition: true,
Title: "Container max uptime is shorter than the task time limit",
Body: "Tasks are killed at the uptime before they reach their limit. Raise the max uptime.",
Link: "/settings/container-runtime",
}}
}
func providerAlerts(providers map[string]config.Provider) []alert.Input {
var out []alert.Input
for _, name := range slices.Sorted(maps.Keys(providers)) {
-3
View File
@@ -201,9 +201,6 @@ func (b *boot) applyWorkflowExecutionSettings(ctx context.Context, settings work
b.publishConfig(ctx, candidate.cfg)
b.applyStatus.Report(ctx, controlplanestore.WorkflowExecutionSettingsKind, version, nil)
b.log.InfoContext(ctx, "workflow execution settings applied", "version", version)
if limit, uptime := settings.MaxTaskRuntime, candidate.cfg.Containers.MaxUptime.Std(); uptime > 0 && (limit <= 0 || limit > uptime) {
b.log.WarnContext(ctx, "containers.max_uptime is shorter than the task time limit; the container is killed first", "max_uptime", uptime, "max_task_runtime", limit)
}
return nil
}
+1
View File
@@ -72,6 +72,7 @@ const (
KindReleaseAvailable = "release.available"
KindPolicyInvalid = "policy.invalid"
KindScheduleFailing = "schedule.failing"
KindUptimeShort = "container.uptime_short"
)
// Limits on what a raiser may store.