fix(ops): log cancelled contexts at debug, one reconnect line per outage (#1809)
This commit is contained in:
6 files changed
+46
-7
No files matched your search
@@ -740,9 +740,23 @@ func encodeRegistryAuth(credential string) (string, error) {
|
|||||||
return base64.URLEncoding.EncodeToString(payload), nil
|
return base64.URLEncoding.EncodeToString(payload), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// inContainer reports whether this process runs in a container, where Docker
|
||||||
|
// and Podman each leave a marker file.
|
||||||
|
func inContainer() bool {
|
||||||
|
for _, marker := range []string{"/.dockerenv", "/run/.containerenv"} {
|
||||||
|
if _, err := os.Stat(marker); err == nil {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
// selfNetwork returns the user-defined network this process's container is
|
// selfNetwork returns the user-defined network this process's container is
|
||||||
// on, or "".
|
// on, or "".
|
||||||
func selfNetwork(ctx context.Context, cli *client.Client, log *slog.Logger) string {
|
func selfNetwork(ctx context.Context, cli *client.Client, log *slog.Logger) string {
|
||||||
|
if !inContainer() {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
hostname, err := os.Hostname()
|
hostname, err := os.Hostname()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.WarnContext(ctx, "container network auto-detect: os.Hostname failed, spawned containers will use the default bridge network and won't resolve sibling services by name — set containers.network explicitly", "err", err)
|
log.WarnContext(ctx, "container network auto-detect: os.Hostname failed, spawned containers will use the default bridge network and won't resolve sibling services by name — set containers.network explicitly", "err", err)
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ package applystatus
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"maps"
|
"maps"
|
||||||
"slices"
|
"slices"
|
||||||
@@ -166,7 +167,12 @@ func (r *Reporter) restamp(ctx context.Context) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (r *Reporter) write(ctx context.Context, status storecontract.ApplyStatus) {
|
func (r *Reporter) write(ctx context.Context, status storecontract.ApplyStatus) {
|
||||||
if err := r.store.PutApplyStatus(ctx, status); err != nil {
|
err := r.store.PutApplyStatus(ctx, status)
|
||||||
|
switch {
|
||||||
|
case err == nil:
|
||||||
|
case errors.Is(err, context.Canceled):
|
||||||
|
r.log.DebugContext(ctx, "apply status not reported; cancelled", "kind", status.Kind)
|
||||||
|
default:
|
||||||
r.log.WarnContext(ctx, "apply status not reported", "kind", status.Kind, "err", err)
|
r.log.WarnContext(ctx, "apply status not reported", "kind", status.Kind, "err", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -247,6 +247,9 @@ func (s *server) identityStatus(ctx context.Context, err error) error {
|
|||||||
return status.Error(codes.Aborted, err.Error())
|
return status.Error(codes.Aborted, err.Error())
|
||||||
case errors.Is(err, identity.ErrInvalid), errors.Is(err, identity.ErrIllegalTransition), errors.Is(err, identity.ErrSystemImmutable):
|
case errors.Is(err, identity.ErrInvalid), errors.Is(err, identity.ErrIllegalTransition), errors.Is(err, identity.ErrSystemImmutable):
|
||||||
return status.Error(codes.FailedPrecondition, err.Error())
|
return status.Error(codes.FailedPrecondition, err.Error())
|
||||||
|
case errors.Is(err, context.Canceled):
|
||||||
|
s.deps.Log.DebugContext(ctx, "identity store call cancelled")
|
||||||
|
return status.Error(codes.Canceled, "request cancelled")
|
||||||
default:
|
default:
|
||||||
s.deps.Log.ErrorContext(ctx, "identity store failed", "err", err)
|
s.deps.Log.ErrorContext(ctx, "identity store failed", "err", err)
|
||||||
return status.Error(codes.Internal, "internal state store error")
|
return status.Error(codes.Internal, "internal state store error")
|
||||||
|
|||||||
@@ -35,9 +35,7 @@ func orgStatus(err error) error {
|
|||||||
|
|
||||||
// orgErr logs and maps one org RPC's failure.
|
// orgErr logs and maps one org RPC's failure.
|
||||||
func (s *server) orgErr(rpc string, err error) error {
|
func (s *server) orgErr(rpc string, err error) error {
|
||||||
if err != nil {
|
s.logFailure(rpc, err)
|
||||||
s.deps.Log.Error("staterpc call failed", "rpc", rpc, "err", err)
|
|
||||||
}
|
|
||||||
return orgStatus(err)
|
return orgStatus(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ package staterpc
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -210,10 +211,20 @@ func RegisterServer(registrar grpc.ServiceRegistrar, deps Deps) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *server) logErr(rpc string, err error) error {
|
func (s *server) logErr(rpc string, err error) error {
|
||||||
if err != nil {
|
s.logFailure(rpc, err)
|
||||||
|
return mapError(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// logFailure records a failed call. A cancelled context is the caller going
|
||||||
|
// away, so it is not a fault.
|
||||||
|
func (s *server) logFailure(rpc string, err error) {
|
||||||
|
switch {
|
||||||
|
case err == nil:
|
||||||
|
case errors.Is(err, context.Canceled):
|
||||||
|
s.deps.Log.Debug("staterpc call cancelled", "rpc", rpc)
|
||||||
|
default:
|
||||||
s.deps.Log.Error("staterpc call failed", "rpc", rpc, "err", err)
|
s.deps.Log.Error("staterpc call failed", "rpc", rpc, "err", err)
|
||||||
}
|
}
|
||||||
return mapError(err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// validationFailure maps a domain validation error to InvalidArgument.
|
// validationFailure maps a domain validation error to InvalidArgument.
|
||||||
|
|||||||
@@ -109,6 +109,7 @@ func (s *Server) watchResource(ctx context.Context, kind string) {
|
|||||||
const minDelay, maxDelay = 250 * time.Millisecond, 30 * time.Second
|
const minDelay, maxDelay = 250 * time.Millisecond, 30 * time.Second
|
||||||
var version int64
|
var version int64
|
||||||
delay := minDelay
|
delay := minDelay
|
||||||
|
logged := false
|
||||||
for ctx.Err() == nil {
|
for ctx.Err() == nil {
|
||||||
started := time.Now()
|
started := time.Now()
|
||||||
progressed := false
|
progressed := false
|
||||||
@@ -136,10 +137,16 @@ func (s *Server) watchResource(ctx context.Context, kind string) {
|
|||||||
failedDelay := min(2*delay, maxDelay)
|
failedDelay := min(2*delay, maxDelay)
|
||||||
if progressed || time.Since(started) >= 2*failedDelay {
|
if progressed || time.Since(started) >= 2*failedDelay {
|
||||||
delay = minDelay
|
delay = minDelay
|
||||||
|
logged = false
|
||||||
} else {
|
} else {
|
||||||
delay = failedDelay
|
delay = failedDelay
|
||||||
}
|
}
|
||||||
s.logLive("control-plane watch ended; reconnecting", "kind", kind, "after_version", version, "retry_in", delay, "err", err)
|
// One line per outage: a peer restart refuses every attempt until it
|
||||||
|
// is back.
|
||||||
|
if !logged {
|
||||||
|
s.logLive("control-plane watch ended; reconnecting", "kind", kind, "after_version", version, "retry_in", delay, "err", err)
|
||||||
|
logged = true
|
||||||
|
}
|
||||||
if !waitLive(ctx, delay) {
|
if !waitLive(ctx, delay) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in new issue
Block a user