fix(ops): log cancelled contexts at debug, one reconnect line per outage (#1809)

This commit is contained in:
sam committed 2026-11-08 17:27:30 +11:00
1 parent 6ae7f319cb
commit 1d9eff6529
6 files changed
+46 -7

No files matched your search

+14
View File
@@ -740,9 +740,23 @@ func encodeRegistryAuth(credential string) (string, error) {
return base64.URLEncoding.EncodeToString(payload), nil
}
// inContainer reports whether this process runs in a container, where Docker
// and Podman each leave a marker file.
func inContainer() bool {
for _, marker := range []string{"/.dockerenv", "/run/.containerenv"} {
if _, err := os.Stat(marker); err == nil {
return true
}
}
return false
}
// selfNetwork returns the user-defined network this process's container is
// on, or "".
func selfNetwork(ctx context.Context, cli *client.Client, log *slog.Logger) string {
if !inContainer() {
return ""
}
hostname, err := os.Hostname()
if err != nil {
log.WarnContext(ctx, "container network auto-detect: os.Hostname failed, spawned containers will use the default bridge network and won't resolve sibling services by name — set containers.network explicitly", "err", err)
+7 -1
View File
@@ -4,6 +4,7 @@ package applystatus
import (
"context"
"errors"
"log/slog"
"maps"
"slices"
@@ -166,7 +167,12 @@ func (r *Reporter) restamp(ctx context.Context) {
}
func (r *Reporter) write(ctx context.Context, status storecontract.ApplyStatus) {
if err := r.store.PutApplyStatus(ctx, status); err != nil {
err := r.store.PutApplyStatus(ctx, status)
switch {
case err == nil:
case errors.Is(err, context.Canceled):
r.log.DebugContext(ctx, "apply status not reported; cancelled", "kind", status.Kind)
default:
r.log.WarnContext(ctx, "apply status not reported", "kind", status.Kind, "err", err)
}
}
@@ -247,6 +247,9 @@ func (s *server) identityStatus(ctx context.Context, err error) error {
return status.Error(codes.Aborted, err.Error())
case errors.Is(err, identity.ErrInvalid), errors.Is(err, identity.ErrIllegalTransition), errors.Is(err, identity.ErrSystemImmutable):
return status.Error(codes.FailedPrecondition, err.Error())
case errors.Is(err, context.Canceled):
s.deps.Log.DebugContext(ctx, "identity store call cancelled")
return status.Error(codes.Canceled, "request cancelled")
default:
s.deps.Log.ErrorContext(ctx, "identity store failed", "err", err)
return status.Error(codes.Internal, "internal state store error")
+1 -3
View File
@@ -35,9 +35,7 @@ func orgStatus(err error) error {
// orgErr logs and maps one org RPC's failure.
func (s *server) orgErr(rpc string, err error) error {
if err != nil {
s.deps.Log.Error("staterpc call failed", "rpc", rpc, "err", err)
}
s.logFailure(rpc, err)
return orgStatus(err)
}
+13 -2
View File
@@ -6,6 +6,7 @@ package staterpc
import (
"bytes"
"context"
"errors"
"log/slog"
"time"
@@ -210,10 +211,20 @@ func RegisterServer(registrar grpc.ServiceRegistrar, deps Deps) {
}
func (s *server) logErr(rpc string, err error) error {
if err != nil {
s.logFailure(rpc, err)
return mapError(err)
}
// logFailure records a failed call. A cancelled context is the caller going
// away, so it is not a fault.
func (s *server) logFailure(rpc string, err error) {
switch {
case err == nil:
case errors.Is(err, context.Canceled):
s.deps.Log.Debug("staterpc call cancelled", "rpc", rpc)
default:
s.deps.Log.Error("staterpc call failed", "rpc", rpc, "err", err)
}
return mapError(err)
}
// validationFailure maps a domain validation error to InvalidArgument.
+8 -1
View File
@@ -109,6 +109,7 @@ func (s *Server) watchResource(ctx context.Context, kind string) {
const minDelay, maxDelay = 250 * time.Millisecond, 30 * time.Second
var version int64
delay := minDelay
logged := false
for ctx.Err() == nil {
started := time.Now()
progressed := false
@@ -136,10 +137,16 @@ func (s *Server) watchResource(ctx context.Context, kind string) {
failedDelay := min(2*delay, maxDelay)
if progressed || time.Since(started) >= 2*failedDelay {
delay = minDelay
logged = false
} else {
delay = failedDelay
}
s.logLive("control-plane watch ended; reconnecting", "kind", kind, "after_version", version, "retry_in", delay, "err", err)
// One line per outage: a peer restart refuses every attempt until it
// is back.
if !logged {
s.logLive("control-plane watch ended; reconnecting", "kind", kind, "after_version", version, "retry_in", delay, "err", err)
logged = true
}
if !waitLive(ctx, delay) {
return
}