Files
archie/internal/releaseupdate/canary_dashboard.go
T
sam 618b22db15
deploy / plan (push) Successful in 6s
deploy / archied-image (push) Canceled after 0s
deploy / agent-image (push) Canceled after 0s
deploy / dist-zip (push) Canceled after 0s
deploy / gate (push) Canceled after 4m28s
fix(releaseupdate): canary ignores task alerts, which the park rate already judges
2026-11-09 08:56:22 +11:00

338 lines
10 KiB
Go

package releaseupdate
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"time"
)
// CanaryDashboard reaches the canary through its dashboard API, the same
// contract an operator uses: it reads the signals, edits the update settings
// through the control plane and installs through the dashboard updater.
type CanaryDashboard struct {
URL string
Token string
HTTP *http.Client
}
// Statuses a finished task can be in, and the subset that counts against a
// release.
var (
finishedStatuses = map[string]bool{"completed": true, "merged": true, "pr_open": true, "rejected": true, "closed_wont_do": true, "parked": true, "dead": true}
badStatuses = map[string]bool{"parked": true, "dead": true}
)
const updateSettingsPath = "/api/control-plane/resources/update-settings"
// Signals reads the canary's current state. Tasks and error lines are split
// into the window before the release was installed on the canary and the time
// since.
func (d CanaryDashboard) Signals(ctx context.Context, installedAt, now time.Time, window time.Duration) (CanarySignals, error) {
var s CanarySignals
var err error
if s.Installed, _, _, err = d.Installed(ctx); err != nil {
return s, err
}
if s.Ready, err = d.ready(ctx); err != nil {
return s, err
}
if s.NewActionAlerts, err = d.actionAlerts(ctx, installedAt); err != nil {
return s, err
}
if err := d.tasks(ctx, installedAt, window, &s); err != nil {
return s, err
}
return s, d.errors(ctx, installedAt, now, window, &s)
}
// Installed returns the daemon version the canary runs, when that release was
// installed on the canary host (zero when the host doesn't report it), and
// the raw update snapshot it offers, which an install request must send back
// unchanged.
func (d CanaryDashboard) Installed(ctx context.Context) (version string, installedAt time.Time, snapshot json.RawMessage, err error) {
var body struct {
Snapshot json.RawMessage `json:"snapshot"`
}
if _, err := d.call(ctx, http.MethodGet, "/api/chat/update", nil, &body); err != nil {
return "", time.Time{}, nil, err
}
var snap Snapshot
if err := json.Unmarshal(body.Snapshot, &snap); err != nil {
return "", time.Time{}, nil, fmt.Errorf("decode update snapshot: %w", err)
}
for _, c := range snap.Components {
if c.ID == "daemon" {
return c.Installed, snap.InstalledAt, body.Snapshot, nil
}
}
return "", time.Time{}, body.Snapshot, errors.New("canary reports no daemon version")
}
func (d CanaryDashboard) ready(ctx context.Context) (bool, error) {
var report struct {
Status string `json:"status"`
}
if _, err := d.call(ctx, http.MethodGet, "/health/detailed", nil, &report); err != nil {
return false, err
}
return report.Status == "ok", nil
}
func (d CanaryDashboard) actionAlerts(ctx context.Context, since time.Time) ([]string, error) {
var body struct {
Alerts []struct {
Kind string `json:"kind"`
Severity string `json:"severity"`
Title string `json:"title"`
FirstRaised time.Time `json:"first_raised"`
} `json:"alerts"`
}
if _, err := d.call(ctx, http.MethodGet, "/api/alerts", nil, &body); err != nil {
return nil, err
}
var titles []string
for _, a := range body.Alerts {
// A task needing a decision is the task's outcome, which the park
// rate already judges against the baseline; counting each one as a
// hard failure rolls back every release on a busy host.
if a.Severity == "action" && !strings.HasPrefix(a.Kind, "task.") && !a.FirstRaised.Before(since) {
titles = append(titles, a.Title)
}
}
return titles, nil
}
// tasks reads tasks, newest update first, until it reaches tasks older than
// the baseline window.
func (d CanaryDashboard) tasks(ctx context.Context, at time.Time, window time.Duration, s *CanarySignals) error {
from := at.Add(-window)
cursor := ""
for range 50 {
var page []struct {
Status string `json:"status"`
UpdatedAt time.Time `json:"updated_at"`
}
path := "/api/tasks?limit=500"
if cursor != "" {
path += "&cursor=" + url.QueryEscape(cursor)
}
header, err := d.call(ctx, http.MethodGet, path, nil, &page)
if err != nil {
return err
}
for _, t := range page {
if t.UpdatedAt.Before(from) || !finishedStatuses[t.Status] {
continue
}
w := &s.Canary
if t.UpdatedAt.Before(at) {
w = &s.Baseline
}
w.Finished++
if badStatuses[t.Status] {
w.Bad++
}
}
cursor = header.Get("X-Next-Cursor")
if cursor == "" || len(page) == 0 || page[len(page)-1].UpdatedAt.Before(from) {
return nil
}
}
return nil
}
// errors counts error-level lines in each window. A log read returns a
// limited number of lines, so a truncated read only reaches back as far as
// its oldest line.
func (d CanaryDashboard) errors(ctx context.Context, at, now time.Time, window time.Duration, s *CanarySignals) error {
var body struct {
Entries []struct {
Time time.Time `json:"time"`
} `json:"entries"`
Truncated bool `json:"truncated"`
}
if _, err := d.call(ctx, http.MethodGet, "/api/logs?level=error&limit=2000", nil, &body); err != nil {
return err
}
from := at.Add(-window)
if body.Truncated {
from = now
if len(body.Entries) > 0 {
from = later(body.Entries[0].Time, at.Add(-window))
}
}
for _, e := range body.Entries {
switch {
case e.Time.Before(from):
case e.Time.Before(at):
s.Baseline.Errors++
default:
s.Canary.Errors++
}
}
s.Baseline.Covered = max(at.Sub(from), 0)
s.Canary.Covered = now.Sub(later(from, at))
return nil
}
func later(a, b time.Time) time.Time {
if a.After(b) {
return a
}
return b
}
// Follow sets the canary's update channel, keeping its release source and
// key. A pin is used only by exact-pin.
func (d CanaryDashboard) Follow(ctx context.Context, channel, pin string) error {
var current struct {
Resource struct {
Version int64 `json:"version"`
Value json.RawMessage `json:"value"`
} `json:"resource"`
}
if _, err := d.call(ctx, http.MethodGet, updateSettingsPath, nil, &current); err != nil {
return err
}
value := map[string]any{}
if len(current.Resource.Value) > 0 && string(current.Resource.Value) != "null" {
if err := json.Unmarshal(current.Resource.Value, &value); err != nil {
return fmt.Errorf("decode update settings: %w", err)
}
}
value["channel"], value["pin"] = channel, pin
_, err := d.call(ctx, http.MethodPost, updateSettingsPath+"/commands/replace",
map[string]any{"value": value, "expected_version": current.Resource.Version}, nil)
return err
}
// Channel is the canary's update channel and pin.
func (d CanaryDashboard) Channel(ctx context.Context) (channel, pin string, err error) {
var current struct {
Resource struct {
Value struct {
Channel string `json:"channel"`
Pin string `json:"pin"`
} `json:"value"`
} `json:"resource"`
}
if _, err := d.call(ctx, http.MethodGet, updateSettingsPath, nil, &current); err != nil {
return "", "", err
}
if current.Resource.Value.Channel == "" {
return "stable", "", nil
}
return current.Resource.Value.Channel, current.Resource.Value.Pin, nil
}
// Install has the canary install version through its updater and waits until
// it reports running it. The install restarts the dashboard, so a dropped
// connection after the request is sent is expected.
func (d CanaryDashboard) Install(ctx context.Context, version string, wait time.Duration) error {
deadline := time.Now().Add(wait)
requested := false
for {
installed, _, snapshot, err := d.Installed(ctx)
switch {
case err != nil:
// The dashboard drops off while the install restarts it.
if !requested {
return err
}
case installed == version:
return nil
case !requested:
// The canary caches its registry lookup, so a release just
// published can take a few minutes to be offered.
offered, err := offers(snapshot, version)
if err != nil {
return err
}
if offered {
_, err := d.call(ctx, http.MethodPost, "/api/chat/update/install", map[string]json.RawMessage{"snapshot": snapshot}, nil)
if _, ok := errors.AsType[statusError](err); ok {
return err
}
requested = true
}
}
if !time.Now().Before(deadline) {
break
}
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(15 * time.Second):
}
}
if !requested {
return fmt.Errorf("canary didn't offer %s within %s", version, wait)
}
return fmt.Errorf("canary didn't report %s within %s", version, wait)
}
// offers reports whether an update snapshot offers version.
func offers(snapshot json.RawMessage, version string) (bool, error) {
var offered Snapshot
if err := json.Unmarshal(snapshot, &offered); err != nil {
return false, fmt.Errorf("decode update snapshot: %w", err)
}
available := offered.Available()
return len(available) > 0 && available[0].Available == version, nil
}
type statusError struct {
code int
body string
}
func (e statusError) Error() string { return fmt.Sprintf("dashboard answered %d: %s", e.code, e.body) }
func (d CanaryDashboard) call(ctx context.Context, method, path string, in, out any) (http.Header, error) {
var body io.Reader
if in != nil {
data, err := json.Marshal(in)
if err != nil {
return nil, err
}
body = bytes.NewReader(data)
}
ctx, cancel := context.WithTimeout(ctx, 2*time.Minute)
defer cancel()
req, err := http.NewRequestWithContext(ctx, method, strings.TrimRight(d.URL, "/")+path, body)
if err != nil {
return nil, err
}
req.Header.Set("Authorization", "Bearer "+d.Token)
if in != nil {
req.Header.Set("Content-Type", "application/json")
}
client := d.HTTP
if client == nil {
client = http.DefaultClient
}
resp, err := client.Do(req)
if err != nil {
return nil, fmt.Errorf("%s %s: %w", method, path, err)
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode/100 != 2 {
text, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
return nil, fmt.Errorf("%s %s: %w", method, path, statusError{resp.StatusCode, strings.TrimSpace(string(text))})
}
if out == nil {
return resp.Header, nil
}
if err := json.NewDecoder(resp.Body).Decode(out); err != nil {
return nil, fmt.Errorf("%s %s: decode: %w", method, path, err)
}
return resp.Header, nil
}