fix(ops): log cancelled contexts at debug, one reconnect line per outage (#1809)
This commit is contained in:
6 files changed
+46
-7
No files matched your search
@@ -740,9 +740,23 @@ func encodeRegistryAuth(credential string) (string, error) {
|
||||
return base64.URLEncoding.EncodeToString(payload), nil
|
||||
}
|
||||
|
||||
// inContainer reports whether this process runs in a container, where Docker
|
||||
// and Podman each leave a marker file.
|
||||
func inContainer() bool {
|
||||
for _, marker := range []string{"/.dockerenv", "/run/.containerenv"} {
|
||||
if _, err := os.Stat(marker); err == nil {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// selfNetwork returns the user-defined network this process's container is
|
||||
// on, or "".
|
||||
func selfNetwork(ctx context.Context, cli *client.Client, log *slog.Logger) string {
|
||||
if !inContainer() {
|
||||
return ""
|
||||
}
|
||||
hostname, err := os.Hostname()
|
||||
if err != nil {
|
||||
log.WarnContext(ctx, "container network auto-detect: os.Hostname failed, spawned containers will use the default bridge network and won't resolve sibling services by name — set containers.network explicitly", "err", err)
|
||||
|
||||
@@ -4,6 +4,7 @@ package applystatus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"maps"
|
||||
"slices"
|
||||
@@ -166,7 +167,12 @@ func (r *Reporter) restamp(ctx context.Context) {
|
||||
}
|
||||
|
||||
func (r *Reporter) write(ctx context.Context, status storecontract.ApplyStatus) {
|
||||
if err := r.store.PutApplyStatus(ctx, status); err != nil {
|
||||
err := r.store.PutApplyStatus(ctx, status)
|
||||
switch {
|
||||
case err == nil:
|
||||
case errors.Is(err, context.Canceled):
|
||||
r.log.DebugContext(ctx, "apply status not reported; cancelled", "kind", status.Kind)
|
||||
default:
|
||||
r.log.WarnContext(ctx, "apply status not reported", "kind", status.Kind, "err", err)
|
||||
}
|
||||
}
|
||||
@@ -247,6 +247,9 @@ func (s *server) identityStatus(ctx context.Context, err error) error {
|
||||
return status.Error(codes.Aborted, err.Error())
|
||||
case errors.Is(err, identity.ErrInvalid), errors.Is(err, identity.ErrIllegalTransition), errors.Is(err, identity.ErrSystemImmutable):
|
||||
return status.Error(codes.FailedPrecondition, err.Error())
|
||||
case errors.Is(err, context.Canceled):
|
||||
s.deps.Log.DebugContext(ctx, "identity store call cancelled")
|
||||
return status.Error(codes.Canceled, "request cancelled")
|
||||
default:
|
||||
s.deps.Log.ErrorContext(ctx, "identity store failed", "err", err)
|
||||
return status.Error(codes.Internal, "internal state store error")
|
||||
|
||||
@@ -35,9 +35,7 @@ func orgStatus(err error) error {
|
||||
|
||||
// orgErr logs and maps one org RPC's failure.
|
||||
func (s *server) orgErr(rpc string, err error) error {
|
||||
if err != nil {
|
||||
s.deps.Log.Error("staterpc call failed", "rpc", rpc, "err", err)
|
||||
}
|
||||
s.logFailure(rpc, err)
|
||||
return orgStatus(err)
|
||||
}
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ package staterpc
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
@@ -210,10 +211,20 @@ func RegisterServer(registrar grpc.ServiceRegistrar, deps Deps) {
|
||||
}
|
||||
|
||||
func (s *server) logErr(rpc string, err error) error {
|
||||
if err != nil {
|
||||
s.logFailure(rpc, err)
|
||||
return mapError(err)
|
||||
}
|
||||
|
||||
// logFailure records a failed call. A cancelled context is the caller going
|
||||
// away, so it is not a fault.
|
||||
func (s *server) logFailure(rpc string, err error) {
|
||||
switch {
|
||||
case err == nil:
|
||||
case errors.Is(err, context.Canceled):
|
||||
s.deps.Log.Debug("staterpc call cancelled", "rpc", rpc)
|
||||
default:
|
||||
s.deps.Log.Error("staterpc call failed", "rpc", rpc, "err", err)
|
||||
}
|
||||
return mapError(err)
|
||||
}
|
||||
|
||||
// validationFailure maps a domain validation error to InvalidArgument.
|
||||
|
||||
@@ -109,6 +109,7 @@ func (s *Server) watchResource(ctx context.Context, kind string) {
|
||||
const minDelay, maxDelay = 250 * time.Millisecond, 30 * time.Second
|
||||
var version int64
|
||||
delay := minDelay
|
||||
logged := false
|
||||
for ctx.Err() == nil {
|
||||
started := time.Now()
|
||||
progressed := false
|
||||
@@ -136,10 +137,16 @@ func (s *Server) watchResource(ctx context.Context, kind string) {
|
||||
failedDelay := min(2*delay, maxDelay)
|
||||
if progressed || time.Since(started) >= 2*failedDelay {
|
||||
delay = minDelay
|
||||
logged = false
|
||||
} else {
|
||||
delay = failedDelay
|
||||
}
|
||||
s.logLive("control-plane watch ended; reconnecting", "kind", kind, "after_version", version, "retry_in", delay, "err", err)
|
||||
// One line per outage: a peer restart refuses every attempt until it
|
||||
// is back.
|
||||
if !logged {
|
||||
s.logLive("control-plane watch ended; reconnecting", "kind", kind, "after_version", version, "retry_in", delay, "err", err)
|
||||
logged = true
|
||||
}
|
||||
if !waitLive(ctx, delay) {
|
||||
return
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user