feat(archied): raise an alert when container max uptime is under the task limit
This commit is contained in:
3 files changed
+18
-3
No files matched your search
@@ -76,6 +76,7 @@ func (c alertChecks) checkConfig(ctx context.Context) {
|
||||
cfg := c.cfg()
|
||||
c.sync(ctx, org.DefaultOrgID, alert.KindModelAlias, modelAliasAlerts(cfg.Models, c.catalog()))
|
||||
c.sync(ctx, org.DefaultOrgID, alert.KindProviderNoKey, providerAlerts(cfg.Providers))
|
||||
c.sync(ctx, org.DefaultOrgID, alert.KindUptimeShort, uptimeAlerts(cfg))
|
||||
unbound := unboundAlerts(cfg)
|
||||
for _, orgID := range credentialOrgs(cfg) {
|
||||
c.sync(ctx, orgID, alert.KindToolNoBinding, unbound[orgID])
|
||||
@@ -129,6 +130,22 @@ func catalogModels(catalog modelcatalog.Snapshot, providerID string) []string {
|
||||
return nil
|
||||
}
|
||||
|
||||
// uptimeAlerts flags a container uptime shorter than the task time limit: the
|
||||
// container is killed before the task can reach its own limit.
|
||||
func uptimeAlerts(cfg config.Config) []alert.Input {
|
||||
uptime, limit := cfg.Containers.MaxUptime.Std(), cfg.Budgets.TaskWallClock.Std()
|
||||
if uptime <= 0 || (limit > 0 && limit <= uptime) {
|
||||
return nil
|
||||
}
|
||||
return []alert.Input{{
|
||||
Org: org.DefaultOrgID, Kind: alert.KindUptimeShort, Subject: "containers.max_uptime",
|
||||
Severity: alert.SeverityWarning, Condition: true,
|
||||
Title: "Container max uptime is shorter than the task time limit",
|
||||
Body: "Tasks are killed at the uptime before they reach their limit. Raise the max uptime.",
|
||||
Link: "/settings/container-runtime",
|
||||
}}
|
||||
}
|
||||
|
||||
func providerAlerts(providers map[string]config.Provider) []alert.Input {
|
||||
var out []alert.Input
|
||||
for _, name := range slices.Sorted(maps.Keys(providers)) {
|
||||
|
||||
@@ -201,9 +201,6 @@ func (b *boot) applyWorkflowExecutionSettings(ctx context.Context, settings work
|
||||
b.publishConfig(ctx, candidate.cfg)
|
||||
b.applyStatus.Report(ctx, controlplanestore.WorkflowExecutionSettingsKind, version, nil)
|
||||
b.log.InfoContext(ctx, "workflow execution settings applied", "version", version)
|
||||
if limit, uptime := settings.MaxTaskRuntime, candidate.cfg.Containers.MaxUptime.Std(); uptime > 0 && (limit <= 0 || limit > uptime) {
|
||||
b.log.WarnContext(ctx, "containers.max_uptime is shorter than the task time limit; the container is killed first", "max_uptime", uptime, "max_task_runtime", limit)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -72,6 +72,7 @@ const (
|
||||
KindReleaseAvailable = "release.available"
|
||||
KindPolicyInvalid = "policy.invalid"
|
||||
KindScheduleFailing = "schedule.failing"
|
||||
KindUptimeShort = "container.uptime_short"
|
||||
)
|
||||
|
||||
// Limits on what a raiser may store.
|
||||
|
||||
Reference in new issue
Block a user