338 lines
10 KiB
Go
338 lines
10 KiB
Go
package releaseupdate
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// CanaryDashboard reaches the canary through its dashboard API, the same
|
|
// contract an operator uses: it reads the signals, edits the update settings
|
|
// through the control plane and installs through the dashboard updater.
|
|
type CanaryDashboard struct {
|
|
URL string
|
|
Token string
|
|
HTTP *http.Client
|
|
}
|
|
|
|
// Statuses a finished task can be in, and the subset that counts against a
|
|
// release.
|
|
var (
|
|
finishedStatuses = map[string]bool{"completed": true, "merged": true, "pr_open": true, "rejected": true, "closed_wont_do": true, "parked": true, "dead": true}
|
|
badStatuses = map[string]bool{"parked": true, "dead": true}
|
|
)
|
|
|
|
const updateSettingsPath = "/api/control-plane/resources/update-settings"
|
|
|
|
// Signals reads the canary's current state. Tasks and error lines are split
|
|
// into the window before the release was installed on the canary and the time
|
|
// since.
|
|
func (d CanaryDashboard) Signals(ctx context.Context, installedAt, now time.Time, window time.Duration) (CanarySignals, error) {
|
|
var s CanarySignals
|
|
var err error
|
|
if s.Installed, _, _, err = d.Installed(ctx); err != nil {
|
|
return s, err
|
|
}
|
|
if s.Ready, err = d.ready(ctx); err != nil {
|
|
return s, err
|
|
}
|
|
if s.NewActionAlerts, err = d.actionAlerts(ctx, installedAt); err != nil {
|
|
return s, err
|
|
}
|
|
if err := d.tasks(ctx, installedAt, window, &s); err != nil {
|
|
return s, err
|
|
}
|
|
return s, d.errors(ctx, installedAt, now, window, &s)
|
|
}
|
|
|
|
// Installed returns the daemon version the canary runs, when that release was
|
|
// installed on the canary host (zero when the host doesn't report it), and
|
|
// the raw update snapshot it offers, which an install request must send back
|
|
// unchanged.
|
|
func (d CanaryDashboard) Installed(ctx context.Context) (version string, installedAt time.Time, snapshot json.RawMessage, err error) {
|
|
var body struct {
|
|
Snapshot json.RawMessage `json:"snapshot"`
|
|
}
|
|
if _, err := d.call(ctx, http.MethodGet, "/api/chat/update", nil, &body); err != nil {
|
|
return "", time.Time{}, nil, err
|
|
}
|
|
var snap Snapshot
|
|
if err := json.Unmarshal(body.Snapshot, &snap); err != nil {
|
|
return "", time.Time{}, nil, fmt.Errorf("decode update snapshot: %w", err)
|
|
}
|
|
for _, c := range snap.Components {
|
|
if c.ID == "daemon" {
|
|
return c.Installed, snap.InstalledAt, body.Snapshot, nil
|
|
}
|
|
}
|
|
return "", time.Time{}, body.Snapshot, errors.New("canary reports no daemon version")
|
|
}
|
|
|
|
func (d CanaryDashboard) ready(ctx context.Context) (bool, error) {
|
|
var report struct {
|
|
Status string `json:"status"`
|
|
}
|
|
if _, err := d.call(ctx, http.MethodGet, "/health/detailed", nil, &report); err != nil {
|
|
return false, err
|
|
}
|
|
return report.Status == "ok", nil
|
|
}
|
|
|
|
func (d CanaryDashboard) actionAlerts(ctx context.Context, since time.Time) ([]string, error) {
|
|
var body struct {
|
|
Alerts []struct {
|
|
Kind string `json:"kind"`
|
|
Severity string `json:"severity"`
|
|
Title string `json:"title"`
|
|
FirstRaised time.Time `json:"first_raised"`
|
|
} `json:"alerts"`
|
|
}
|
|
if _, err := d.call(ctx, http.MethodGet, "/api/alerts", nil, &body); err != nil {
|
|
return nil, err
|
|
}
|
|
var titles []string
|
|
for _, a := range body.Alerts {
|
|
// A task needing a decision is the task's outcome, which the park
|
|
// rate already judges against the baseline; counting each one as a
|
|
// hard failure rolls back every release on a busy host.
|
|
if a.Severity == "action" && !strings.HasPrefix(a.Kind, "task.") && !a.FirstRaised.Before(since) {
|
|
titles = append(titles, a.Title)
|
|
}
|
|
}
|
|
return titles, nil
|
|
}
|
|
|
|
// tasks reads tasks, newest update first, until it reaches tasks older than
|
|
// the baseline window.
|
|
func (d CanaryDashboard) tasks(ctx context.Context, at time.Time, window time.Duration, s *CanarySignals) error {
|
|
from := at.Add(-window)
|
|
cursor := ""
|
|
for range 50 {
|
|
var page []struct {
|
|
Status string `json:"status"`
|
|
UpdatedAt time.Time `json:"updated_at"`
|
|
}
|
|
path := "/api/tasks?limit=500"
|
|
if cursor != "" {
|
|
path += "&cursor=" + url.QueryEscape(cursor)
|
|
}
|
|
header, err := d.call(ctx, http.MethodGet, path, nil, &page)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, t := range page {
|
|
if t.UpdatedAt.Before(from) || !finishedStatuses[t.Status] {
|
|
continue
|
|
}
|
|
w := &s.Canary
|
|
if t.UpdatedAt.Before(at) {
|
|
w = &s.Baseline
|
|
}
|
|
w.Finished++
|
|
if badStatuses[t.Status] {
|
|
w.Bad++
|
|
}
|
|
}
|
|
cursor = header.Get("X-Next-Cursor")
|
|
if cursor == "" || len(page) == 0 || page[len(page)-1].UpdatedAt.Before(from) {
|
|
return nil
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// errors counts error-level lines in each window. A log read returns a
|
|
// limited number of lines, so a truncated read only reaches back as far as
|
|
// its oldest line.
|
|
func (d CanaryDashboard) errors(ctx context.Context, at, now time.Time, window time.Duration, s *CanarySignals) error {
|
|
var body struct {
|
|
Entries []struct {
|
|
Time time.Time `json:"time"`
|
|
} `json:"entries"`
|
|
Truncated bool `json:"truncated"`
|
|
}
|
|
if _, err := d.call(ctx, http.MethodGet, "/api/logs?level=error&limit=2000", nil, &body); err != nil {
|
|
return err
|
|
}
|
|
from := at.Add(-window)
|
|
if body.Truncated {
|
|
from = now
|
|
if len(body.Entries) > 0 {
|
|
from = later(body.Entries[0].Time, at.Add(-window))
|
|
}
|
|
}
|
|
for _, e := range body.Entries {
|
|
switch {
|
|
case e.Time.Before(from):
|
|
case e.Time.Before(at):
|
|
s.Baseline.Errors++
|
|
default:
|
|
s.Canary.Errors++
|
|
}
|
|
}
|
|
s.Baseline.Covered = max(at.Sub(from), 0)
|
|
s.Canary.Covered = now.Sub(later(from, at))
|
|
return nil
|
|
}
|
|
|
|
func later(a, b time.Time) time.Time {
|
|
if a.After(b) {
|
|
return a
|
|
}
|
|
return b
|
|
}
|
|
|
|
// Follow sets the canary's update channel, keeping its release source and
|
|
// key. A pin is used only by exact-pin.
|
|
func (d CanaryDashboard) Follow(ctx context.Context, channel, pin string) error {
|
|
var current struct {
|
|
Resource struct {
|
|
Version int64 `json:"version"`
|
|
Value json.RawMessage `json:"value"`
|
|
} `json:"resource"`
|
|
}
|
|
if _, err := d.call(ctx, http.MethodGet, updateSettingsPath, nil, ¤t); err != nil {
|
|
return err
|
|
}
|
|
value := map[string]any{}
|
|
if len(current.Resource.Value) > 0 && string(current.Resource.Value) != "null" {
|
|
if err := json.Unmarshal(current.Resource.Value, &value); err != nil {
|
|
return fmt.Errorf("decode update settings: %w", err)
|
|
}
|
|
}
|
|
value["channel"], value["pin"] = channel, pin
|
|
_, err := d.call(ctx, http.MethodPost, updateSettingsPath+"/commands/replace",
|
|
map[string]any{"value": value, "expected_version": current.Resource.Version}, nil)
|
|
return err
|
|
}
|
|
|
|
// Channel is the canary's update channel and pin.
|
|
func (d CanaryDashboard) Channel(ctx context.Context) (channel, pin string, err error) {
|
|
var current struct {
|
|
Resource struct {
|
|
Value struct {
|
|
Channel string `json:"channel"`
|
|
Pin string `json:"pin"`
|
|
} `json:"value"`
|
|
} `json:"resource"`
|
|
}
|
|
if _, err := d.call(ctx, http.MethodGet, updateSettingsPath, nil, ¤t); err != nil {
|
|
return "", "", err
|
|
}
|
|
if current.Resource.Value.Channel == "" {
|
|
return "stable", "", nil
|
|
}
|
|
return current.Resource.Value.Channel, current.Resource.Value.Pin, nil
|
|
}
|
|
|
|
// Install has the canary install version through its updater and waits until
|
|
// it reports running it. The install restarts the dashboard, so a dropped
|
|
// connection after the request is sent is expected.
|
|
func (d CanaryDashboard) Install(ctx context.Context, version string, wait time.Duration) error {
|
|
deadline := time.Now().Add(wait)
|
|
requested := false
|
|
for {
|
|
installed, _, snapshot, err := d.Installed(ctx)
|
|
switch {
|
|
case err != nil:
|
|
// The dashboard drops off while the install restarts it.
|
|
if !requested {
|
|
return err
|
|
}
|
|
case installed == version:
|
|
return nil
|
|
case !requested:
|
|
// The canary caches its registry lookup, so a release just
|
|
// published can take a few minutes to be offered.
|
|
offered, err := offers(snapshot, version)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if offered {
|
|
_, err := d.call(ctx, http.MethodPost, "/api/chat/update/install", map[string]json.RawMessage{"snapshot": snapshot}, nil)
|
|
if _, ok := errors.AsType[statusError](err); ok {
|
|
return err
|
|
}
|
|
requested = true
|
|
}
|
|
}
|
|
if !time.Now().Before(deadline) {
|
|
break
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
case <-time.After(15 * time.Second):
|
|
}
|
|
}
|
|
if !requested {
|
|
return fmt.Errorf("canary didn't offer %s within %s", version, wait)
|
|
}
|
|
return fmt.Errorf("canary didn't report %s within %s", version, wait)
|
|
}
|
|
|
|
// offers reports whether an update snapshot offers version.
|
|
func offers(snapshot json.RawMessage, version string) (bool, error) {
|
|
var offered Snapshot
|
|
if err := json.Unmarshal(snapshot, &offered); err != nil {
|
|
return false, fmt.Errorf("decode update snapshot: %w", err)
|
|
}
|
|
available := offered.Available()
|
|
return len(available) > 0 && available[0].Available == version, nil
|
|
}
|
|
|
|
type statusError struct {
|
|
code int
|
|
body string
|
|
}
|
|
|
|
func (e statusError) Error() string { return fmt.Sprintf("dashboard answered %d: %s", e.code, e.body) }
|
|
|
|
func (d CanaryDashboard) call(ctx context.Context, method, path string, in, out any) (http.Header, error) {
|
|
var body io.Reader
|
|
if in != nil {
|
|
data, err := json.Marshal(in)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
body = bytes.NewReader(data)
|
|
}
|
|
ctx, cancel := context.WithTimeout(ctx, 2*time.Minute)
|
|
defer cancel()
|
|
req, err := http.NewRequestWithContext(ctx, method, strings.TrimRight(d.URL, "/")+path, body)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
req.Header.Set("Authorization", "Bearer "+d.Token)
|
|
if in != nil {
|
|
req.Header.Set("Content-Type", "application/json")
|
|
}
|
|
client := d.HTTP
|
|
if client == nil {
|
|
client = http.DefaultClient
|
|
}
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("%s %s: %w", method, path, err)
|
|
}
|
|
defer func() { _ = resp.Body.Close() }()
|
|
if resp.StatusCode/100 != 2 {
|
|
text, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
|
|
return nil, fmt.Errorf("%s %s: %w", method, path, statusError{resp.StatusCode, strings.TrimSpace(string(text))})
|
|
}
|
|
if out == nil {
|
|
return resp.Header, nil
|
|
}
|
|
if err := json.NewDecoder(resp.Body).Decode(out); err != nil {
|
|
return nil, fmt.Errorf("%s %s: decode: %w", method, path, err)
|
|
}
|
|
return resp.Header, nil
|
|
}
|