diff --git a/internal/app/archied/repository_access.go b/internal/app/archied/repository_access.go index 079c89569..43f891022 100644 --- a/internal/app/archied/repository_access.go +++ b/internal/app/archied/repository_access.go @@ -5,7 +5,10 @@ import ( "encoding/json" "errors" "fmt" + "io/fs" + "path/filepath" "strings" + "syscall" "github.com/go-git/go-git/v6" @@ -94,9 +97,9 @@ func (b *boot) openRepositoryConnection(ctx context.Context, principal access.Pr if !connection.Permitted(string(principal.IdentityID), operation) { return failure("operation_denied", fmt.Sprintf("Allow this agent to %s through %s before continuing.", operation, connection.Name)) } - token, err := b.secrets.Resolve(connection.Token) - if err != nil || token == "" { - return failure("credential_unavailable", fmt.Sprintf("The credential for %s is unavailable. Check its secret reference.", connection.Name)) + token, code, message := b.connectionToken(ctx, connection) + if code != "" { + return failure(code, message) } host := strings.TrimRight(connection.Host, "/") if !strings.Contains(host, "://") { @@ -106,7 +109,10 @@ func (b *boot) openRepositoryConnection(ctx context.Context, principal access.Pr cfg := b.cfgHolder.Get() username := connection.Account gitToken := token - var minted forge.GitCredential + var ( + minted forge.GitCredential + err error + ) if issuer, ok := out.Forge.(forge.GitCredentialIssuer); ok { minted, err = issuer.GitCredential(ctx, repository.Owner, repository.Name) switch { @@ -138,6 +144,33 @@ func (b *boot) openRepositoryConnection(ctx context.Context, principal access.Pr return out, nil } +// connectionToken resolves the connection's credential, or returns a failure +// code and message. A full volume is reported as such, not as a bad reference. +func (b *boot) connectionToken(ctx context.Context, connection config.ForgeConnection) (string, string, string) { + token, err := b.secrets.Resolve(connection.Token) + if volume, full := exhaustedVolume(err); full { + b.log.ErrorContext(ctx, "secret resolution failed: host resource exhausted", "connection", connection.ID, "volume", volume, "error", err) + return "", "environment_exhausted", fmt.Sprintf("The host has run out of space on %s, so the credential for %s cannot be read. Free space there.", volume, connection.Name) + } + if err != nil || token == "" { + return "", "credential_unavailable", fmt.Sprintf("The credential for %s is unavailable. Check its secret reference.", connection.Name) + } + return token, "", "" +} + +// exhaustedVolume reports whether err is a full volume or quota and names the +// volume, so the failure reads as an environment problem rather than a +// misconfigured secret. +func exhaustedVolume(err error) (string, bool) { + if !errors.Is(err, syscall.ENOSPC) && !errors.Is(err, syscall.EDQUOT) { + return "", false + } + if pathErr, ok := errors.AsType[*fs.PathError](err); ok { + return filepath.Dir(pathErr.Path), true + } + return "the host volume", true +} + // commitIdentity is the author of commits made through a connection: its // configured commit identity, else the identity the forge minted with the // credential, else the root bot fields, else its account. diff --git a/internal/infrastructure/extension/gate.go b/internal/infrastructure/extension/gate.go index b911c22ca..045c0b53a 100644 --- a/internal/infrastructure/extension/gate.go +++ b/internal/infrastructure/extension/gate.go @@ -3,6 +3,7 @@ package extension import ( "context" "errors" + "fmt" "net" "net/http" "os" @@ -24,7 +25,7 @@ type gate struct { func openGate(ctx context.Context, allow []string) (*gate, error) { dir, err := os.MkdirTemp("", "archie-ext-") if err != nil { - return nil, err + return nil, fmt.Errorf("create egress directory in %s: %w", os.TempDir(), err) } g := &gate{dir: dir, socket: filepath.Join(dir, "egress.sock")} ln, err := (&net.ListenConfig{}).Listen(ctx, "unix", g.socket) diff --git a/internal/infrastructure/secretengine/runner.go b/internal/infrastructure/secretengine/runner.go index b33830765..6795831c8 100644 --- a/internal/infrastructure/secretengine/runner.go +++ b/internal/infrastructure/secretengine/runner.go @@ -11,6 +11,7 @@ import ( "path/filepath" "slices" "sync" + "syscall" "time" "github.com/samcharles93/archie-core/internal/domain/applystatus" @@ -177,6 +178,13 @@ func (r *Runner) ensure(ctx context.Context, pkg storepkg.Installed, setting con spec := extension.Spec{Name: pkg.Name, Path: binary, SHA256: sum, Env: slices.Clone(pkg.AcceptedAuthority.Env), Egress: slices.Clone(pkg.AcceptedAuthority.EgressHosts)} engine, err := Start(ctx, r.host, spec, setting.Settings) if err != nil { + if errors.Is(err, syscall.ENOSPC) || errors.Is(err, syscall.EDQUOT) { + // Registered so references report the exhausted resource instead + // of an engine that is not registered. The empty digest makes the + // next sync start it again. + r.registry.Register(&failedEngine{name: pkg.Name, err: err}) + r.running[pkg.Name] = runningEngine{} + } return err } engine.version = pkg.Descriptor.Version @@ -208,3 +216,14 @@ func Supervise(ctx context.Context, registry *secret.Registry, source Source, pr go runner.Run(ctx, applystatus.RestampInterval) return runner } + +// failedEngine answers every resolution with the error that kept its +// extension from starting. +type failedEngine struct { + name string + err error +} + +func (e *failedEngine) Name() string { return e.name } + +func (e *failedEngine) Resolve(string) (string, error) { return "", e.err }