Merge pull request #1814 from samcharles93/fix/egress-provider-secret-ref
fix(egress): bind a task's model providers from their api_key_ref
This commit is contained in:
10 files changed
+634
-38
No files matched your search
@@ -79,7 +79,7 @@ func (b *boot) setupKitLauncher(ctx context.Context) {
|
||||
}()
|
||||
// Native tasks need only the proxy for model access, so it serves them
|
||||
// even when a Kit-only piece below fails.
|
||||
b.modelEgress = modelEgress{proxy: proxy, addr: ln.Addr().String(), caFile: egress.CACertPath(caDir), config: b.cfgHolder}
|
||||
b.modelEgress = modelEgress{proxy: proxy, addr: ln.Addr().String(), caFile: egress.CACertPath(caDir), config: b.cfgHolder, secrets: b.secrets}
|
||||
fetcher, err := fetch.New(fetch.WithDockerCredentials())
|
||||
if err != nil {
|
||||
b.addCleanup(func() { _ = srv.Close() })
|
||||
|
||||
@@ -5,58 +5,70 @@ import (
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/docker/sandbox-kit-spec/v3/spec"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/daemon"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/egress"
|
||||
"github.com/samcharles93/archie-core/internal/secret"
|
||||
"github.com/samcharles93/archie-core/internal/storage"
|
||||
)
|
||||
|
||||
const modelProxyCAPath = "/opt/archie/egress-ca.pem"
|
||||
|
||||
// modelEgress opens native task containers' model sessions on the egress
|
||||
// proxy. Each keyed provider's key is the org's credential binding named
|
||||
// after the provider.
|
||||
// proxy. A provider's key is the one its api_key_ref resolves to, carried by
|
||||
// the session, or the org's credential binding named after the provider for a
|
||||
// provider that only declares an env var.
|
||||
type modelEgress struct {
|
||||
proxy *egress.Proxy
|
||||
addr string
|
||||
caFile string
|
||||
config *config.Holder
|
||||
proxy *egress.Proxy
|
||||
addr string
|
||||
caFile string
|
||||
config *config.Holder
|
||||
secrets *secret.Registry
|
||||
}
|
||||
|
||||
func (m modelEgress) Open(token, org string, providers map[string]config.Provider) (daemon.ModelSession, error) {
|
||||
var keyed []string
|
||||
for _, name := range slices.Sorted(maps.Keys(providers)) {
|
||||
if providers[name].APIKeyEnv != "" {
|
||||
keyed = append(keyed, name)
|
||||
// Each keyed provider's credential variable: the operator's, or the name
|
||||
// derived for a provider that carries only a secret reference.
|
||||
keyed := make(map[string]string, len(providers))
|
||||
for name, p := range providers {
|
||||
if keyEnv := p.KeyEnv(name); keyEnv != "" {
|
||||
keyed[name] = keyEnv
|
||||
}
|
||||
}
|
||||
if len(keyed) == 0 {
|
||||
names := slices.Sorted(maps.Keys(keyed))
|
||||
if len(names) == 0 {
|
||||
return daemon.ModelSession{}, nil
|
||||
}
|
||||
bound := m.config.Get().Containers.BoundCredentials(org, keyed, keyed)
|
||||
bound := m.config.Get().Containers.BoundCredentials(org, names, names)
|
||||
secrets := make(map[string]string, len(keyed))
|
||||
var (
|
||||
subs []egress.Substitution
|
||||
hosts []string
|
||||
env []string
|
||||
)
|
||||
for _, name := range keyed {
|
||||
for _, name := range names {
|
||||
p := providers[name]
|
||||
if _, ok := bound[name]; !ok {
|
||||
return daemon.ModelSession{}, fmt.Errorf("provider %q has no credential binding named %q in org %s", name, name, org)
|
||||
}
|
||||
host, err := egress.ProviderHost(p.Class, p.BaseURL)
|
||||
if err != nil {
|
||||
return daemon.ModelSession{}, fmt.Errorf("provider %q: %w", name, err)
|
||||
}
|
||||
key, err := m.providerKey(name, p, org, bound)
|
||||
if err != nil {
|
||||
return daemon.ModelSession{}, err
|
||||
}
|
||||
if key != "" {
|
||||
secrets[name] = key
|
||||
}
|
||||
subs = append(subs, egress.Substitution{Service: name, Host: host})
|
||||
hosts = append(hosts, host)
|
||||
env = append(env, p.APIKeyEnv+"="+egress.SentinelFor(name))
|
||||
env = append(env, keyed[name]+"="+egress.SentinelFor(name))
|
||||
}
|
||||
session, err := m.proxy.Register(egress.SessionOptions{
|
||||
Token: token, Org: org, Substitutions: subs,
|
||||
Token: token, Org: org, Substitutions: subs, Secrets: secrets,
|
||||
Network: &spec.PhasedNetwork{Runtime: &spec.NetworkRules{Allow: hosts}},
|
||||
})
|
||||
if err != nil {
|
||||
@@ -69,4 +81,27 @@ func (m modelEgress) Open(token, org string, providers map[string]config.Provide
|
||||
}, nil
|
||||
}
|
||||
|
||||
// providerKey is the key this session carries for p: its secret reference
|
||||
// resolved. A provider with no reference keeps the org's binding path, which
|
||||
// the proxy resolves when a request arrives, so its key stays only there.
|
||||
func (m modelEgress) providerKey(name string, p config.Provider, org string, bound map[string]config.CredentialBinding) (string, error) {
|
||||
if p.APIKey == (config.SecretRef{}) {
|
||||
if _, ok := bound[name]; !ok {
|
||||
return "", fmt.Errorf("provider %q has no credential binding named %q in org %s", name, name, org)
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
if m.secrets == nil {
|
||||
return "", fmt.Errorf("provider %q: no secret engine is available to resolve its api_key_ref", name)
|
||||
}
|
||||
value, err := m.secrets.Resolve(p.APIKey)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("provider %q: resolve api_key_ref: %w", name, err)
|
||||
}
|
||||
if value = strings.TrimSpace(value); value == "" {
|
||||
return "", fmt.Errorf("provider %q: api_key_ref resolved empty", name)
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func (m modelEgress) Close(ctx context.Context, token string) { m.proxy.Revoke(ctx, token) }
|
||||
@@ -0,0 +1,131 @@
|
||||
package archied
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/egress"
|
||||
"github.com/samcharles93/archie-core/internal/secret"
|
||||
)
|
||||
|
||||
// unboundResolver refuses every credential a session does not carry itself.
|
||||
type unboundResolver struct{}
|
||||
|
||||
func (unboundResolver) Resolve(context.Context, string, string) (string, error) {
|
||||
return "", egress.ErrUnbound
|
||||
}
|
||||
|
||||
// A provider credential recorded as a secret reference needs no credential
|
||||
// binding: modelEgress resolves it for the session while the container gets
|
||||
// only the sentinel. A provider that names only an api_key_env keeps resolving
|
||||
// through its org binding, and one with no binding is still refused.
|
||||
func TestModelEgressResolvesProviderSecretRefs(t *testing.T) {
|
||||
const realKey = "resolved-provider-key"
|
||||
t.Setenv("PROVIDER_KEY", realKey)
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
provider config.Provider
|
||||
bindings []config.CredentialBinding
|
||||
wantErr string
|
||||
wantEnv string
|
||||
}{
|
||||
{
|
||||
name: "a secret reference needs no binding",
|
||||
provider: config.Provider{Class: "openai", APIKey: config.SecretRef{Engine: "env", Key: "PROVIDER_KEY"}},
|
||||
wantEnv: config.ProviderKeyEnvName("used"),
|
||||
},
|
||||
{
|
||||
name: "an env-only provider keeps its binding",
|
||||
provider: config.Provider{Class: "openai", APIKeyEnv: "PROVIDER_ENV_KEY"},
|
||||
bindings: []config.CredentialBinding{{Service: "used", Org: "org-test", Secret: config.SecretRef{Engine: "env", Key: "PROVIDER_KEY"}}},
|
||||
wantEnv: "PROVIDER_ENV_KEY",
|
||||
},
|
||||
{
|
||||
name: "an env-only provider without a binding is refused",
|
||||
provider: config.Provider{Class: "openai", APIKeyEnv: "PROVIDER_ENV_KEY"},
|
||||
wantErr: "no credential binding",
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
proxy := egress.NewProxy(nil, egress.ProxyOptions{
|
||||
Resolver: unboundResolver{},
|
||||
})
|
||||
m := modelEgress{
|
||||
proxy: proxy, secrets: secret.NewRegistry(),
|
||||
config: config.NewHolder(config.Config{Containers: config.ContainerConfig{Credentials: tc.bindings}}),
|
||||
}
|
||||
session, err := m.Open("run-credential", "org-test", map[string]config.Provider{"used": tc.provider})
|
||||
if tc.wantErr != "" {
|
||||
if err == nil || !strings.Contains(err.Error(), tc.wantErr) {
|
||||
t.Fatalf("err = %v, want one naming %q", err, tc.wantErr)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("open model egress: %v", err)
|
||||
}
|
||||
env := strings.Join(session.Env, "\n")
|
||||
if want := tc.wantEnv + "=" + egress.SentinelFor("used"); !strings.Contains(env, want) {
|
||||
t.Fatalf("session env %q does not carry %q", env, want)
|
||||
}
|
||||
if strings.Contains(env, realKey) {
|
||||
t.Fatalf("the provider key reached the container: %q", env)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The key a provider's api_key_ref resolves to lives in the session, never in
|
||||
// the container: a request carrying the sentinel reaches the provider's host
|
||||
// with the real key, and the environment the container holds has only the
|
||||
// placeholder.
|
||||
func TestModelEgressSubstitutesTheProviderKey(t *testing.T) {
|
||||
t.Setenv("PROVIDER_KEY", "resolved-provider-key")
|
||||
|
||||
var seen *http.Request
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { seen = r }))
|
||||
defer upstream.Close()
|
||||
|
||||
// The resolver answers nothing: a session key is what the substitution uses.
|
||||
proxy := egress.NewProxy(nil, egress.ProxyOptions{
|
||||
Resolver: unboundResolver{},
|
||||
Dial: func(ctx context.Context, _, _ string) (net.Conn, error) {
|
||||
return (&net.Dialer{}).DialContext(ctx, "tcp", upstream.Listener.Addr().String())
|
||||
},
|
||||
})
|
||||
m := modelEgress{proxy: proxy, secrets: secret.NewRegistry(), config: config.NewHolder(config.Config{})}
|
||||
providers := config.Config{
|
||||
Models: map[string]string{"default": "alpha/model-one"},
|
||||
Providers: map[string]config.Provider{
|
||||
"alpha": {Class: "openai", BaseURL: "http://api.alpha.example", APIKey: config.SecretRef{Engine: "env", Key: "PROVIDER_KEY"}},
|
||||
},
|
||||
}.ModelProviders()
|
||||
session, err := m.Open("run-credential", "org-test", providers)
|
||||
if err != nil {
|
||||
t.Fatalf("open model egress: %v", err)
|
||||
}
|
||||
if env := strings.Join(session.Env, "\n"); !strings.Contains(env, egress.SentinelFor("alpha")) {
|
||||
t.Fatalf("session env %q does not place the sentinel", env)
|
||||
}
|
||||
|
||||
req := httptest.NewRequestWithContext(context.Background(), http.MethodPost, "http://api.alpha.example/v1/chat", nil)
|
||||
req.Header.Set("Authorization", "Bearer "+egress.SentinelFor("alpha"))
|
||||
req.Header.Set("Proxy-Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte("archie:run-credential")))
|
||||
rec := httptest.NewRecorder()
|
||||
proxy.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK || seen == nil {
|
||||
t.Fatalf("request not forwarded: %d %q", rec.Code, rec.Body.String())
|
||||
}
|
||||
if got := seen.Header.Get("Authorization"); got != "Bearer resolved-provider-key" {
|
||||
t.Fatalf("upstream saw %q, want the resolved key", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ProviderKeyEnvName is the variable a run reads provider id's credential from
|
||||
// when the provider names no api_key_env. It is derived from the id alone, so
|
||||
// the daemon that resolves the credential and the container that holds only its
|
||||
// sentinel agree on the name without the key travelling.
|
||||
func ProviderKeyEnvName(id string) string {
|
||||
sum := sha256.Sum256([]byte("provider\x00" + id))
|
||||
return fmt.Sprintf("ARCHIE_PROVIDER_%X_API_KEY", sum[:8])
|
||||
}
|
||||
|
||||
// KeyEnv is the variable a run reads provider id's credential from: its
|
||||
// api_key_env, or the name derived for a provider that carries only a secret
|
||||
// reference. A provider with neither holds no credential and names no variable.
|
||||
func (p Provider) KeyEnv(id string) string {
|
||||
switch {
|
||||
case p.APIKeyEnv != "":
|
||||
return p.APIKeyEnv
|
||||
case p.APIKey == (SecretRef{}):
|
||||
return ""
|
||||
default:
|
||||
return ProviderKeyEnvName(id)
|
||||
}
|
||||
}
|
||||
|
||||
// ModelProviders returns c's providers that one of its model aliases resolves
|
||||
// to, each carrying the variable its credential is read from: the operator's
|
||||
// api_key_env, or a name derived from the provider when its credential is a
|
||||
// secret reference. A provider no alias names is left out, so nothing requires
|
||||
// a credential for a provider a run never reaches.
|
||||
func (c Config) ModelProviders() map[string]Provider {
|
||||
out := make(map[string]Provider, len(c.Providers))
|
||||
for _, ref := range c.Models {
|
||||
id, _, ok := strings.Cut(ref, "/")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
provider, ok := c.Providers[id]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
provider.APIKeyEnv = provider.KeyEnv(id)
|
||||
out[id] = provider
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package config
|
||||
|
||||
import "testing"
|
||||
|
||||
// A task's run carries only the providers its model aliases reach, and every
|
||||
// credential-bearing provider names the variable a run reads its credential
|
||||
// from: the operator's api_key_env, or a name derived from the provider so a
|
||||
// container that holds only a sentinel still knows where to put it.
|
||||
func TestModelProviders(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ref := SecretRef{Engine: "env", Key: "PROVIDER_KEY"}
|
||||
cases := []struct {
|
||||
name string
|
||||
models map[string]string
|
||||
providers map[string]Provider
|
||||
want []string
|
||||
wantEnv map[string]string // "" means no credential variable
|
||||
}{
|
||||
{
|
||||
name: "an alias's provider keeps the operator's variable",
|
||||
models: map[string]string{"default": "alpha/model-one", "fast": "beta/model-two"},
|
||||
providers: map[string]Provider{"alpha": {Class: "openai", APIKeyEnv: "ALPHA_KEY"}, "beta": {Class: "gemini", APIKeyEnv: "BETA_KEY"}},
|
||||
want: []string{"alpha", "beta"},
|
||||
wantEnv: map[string]string{"alpha": "ALPHA_KEY", "beta": "BETA_KEY"},
|
||||
},
|
||||
{
|
||||
name: "a provider no alias names is left out",
|
||||
models: map[string]string{"default": "alpha/model-one"},
|
||||
providers: map[string]Provider{"alpha": {Class: "openai", APIKeyEnv: "ALPHA_KEY"}, "unused": {Class: "gemini", APIKeyEnv: "UNUSED_KEY"}},
|
||||
want: []string{"alpha"},
|
||||
wantEnv: map[string]string{"alpha": "ALPHA_KEY"},
|
||||
},
|
||||
{
|
||||
name: "a secret reference names a derived variable",
|
||||
models: map[string]string{"default": "alpha/model-one", "fast": "beta/model-two"},
|
||||
providers: map[string]Provider{"alpha": {Class: "openai", APIKey: ref}, "beta": {Class: "deepseek", APIKey: ref}},
|
||||
want: []string{"alpha", "beta"},
|
||||
wantEnv: map[string]string{"alpha": "<derived>", "beta": "<derived>"},
|
||||
},
|
||||
{
|
||||
name: "a provider with no credential carries no variable",
|
||||
models: map[string]string{"default": "alpha/model-one"},
|
||||
providers: map[string]Provider{"alpha": {Class: "openai", BaseURL: "http://models.internal:11434"}},
|
||||
want: []string{"alpha"},
|
||||
wantEnv: map[string]string{"alpha": ""},
|
||||
},
|
||||
{
|
||||
name: "an alias naming no configured provider carries nothing",
|
||||
models: map[string]string{"default": "missing/model-one", "fast": "no-slash"},
|
||||
providers: map[string]Provider{"alpha": {Class: "openai", APIKeyEnv: "ALPHA_KEY"}},
|
||||
want: nil,
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
got := Config{Models: tc.models, Providers: tc.providers}.ModelProviders()
|
||||
if len(got) != len(tc.want) {
|
||||
t.Fatalf("providers = %v, want %v", keys(got), tc.want)
|
||||
}
|
||||
for _, name := range tc.want {
|
||||
provider, ok := got[name]
|
||||
if !ok {
|
||||
t.Fatalf("provider %q missing from %v", name, keys(got))
|
||||
}
|
||||
wantEnv := tc.wantEnv[name]
|
||||
switch wantEnv {
|
||||
case "<derived>":
|
||||
if provider.APIKeyEnv == "" {
|
||||
t.Fatalf("provider %q with a secret reference has no credential variable", name)
|
||||
}
|
||||
default:
|
||||
if provider.APIKeyEnv != wantEnv {
|
||||
t.Fatalf("provider %q variable = %q, want %q", name, provider.APIKeyEnv, wantEnv)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The derived name belongs to the provider it was derived for: two providers
|
||||
// never share one, and the same provider always gets the same name, so the
|
||||
// daemon and the container agree without the key travelling.
|
||||
func TestProviderKeyEnvNameIsPerProvider(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
alpha, beta := ProviderKeyEnvName("alpha"), ProviderKeyEnvName("beta")
|
||||
if alpha == "" {
|
||||
t.Fatal("ProviderKeyEnvName(alpha) is empty")
|
||||
}
|
||||
if alpha != ProviderKeyEnvName("alpha") {
|
||||
t.Fatal("ProviderKeyEnvName is not stable for one provider")
|
||||
}
|
||||
if alpha == beta {
|
||||
t.Fatalf("both providers share the variable %q", alpha)
|
||||
}
|
||||
}
|
||||
|
||||
func keys(providers map[string]Provider) []string {
|
||||
out := make([]string, 0, len(providers))
|
||||
for name := range providers {
|
||||
out = append(out, name)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -135,17 +135,21 @@ type TaskModels interface {
|
||||
}
|
||||
|
||||
// taskConfig is task's configuration with its org's model aliases and own
|
||||
// providers.
|
||||
// providers, narrowed to the providers those aliases reach, each carrying the
|
||||
// variable its credential is read from. A provider no alias names is not part
|
||||
// of the run at all: requiring its credential would park a task whose models
|
||||
// never touch it.
|
||||
func (d *Daemon) taskConfig(ctx context.Context, task *workflow.Task) (config.Config, error) {
|
||||
cfg := d.configFor(task)
|
||||
if d.TaskModels == nil {
|
||||
return cfg, nil
|
||||
if d.TaskModels != nil {
|
||||
models, err := d.TaskModels.TaskModels(ctx, task.ID)
|
||||
if err != nil {
|
||||
return config.Config{}, fmt.Errorf("resolve the task's models: %w", err)
|
||||
}
|
||||
cfg = cfg.WithOrgModels(models)
|
||||
}
|
||||
models, err := d.TaskModels.TaskModels(ctx, task.ID)
|
||||
if err != nil {
|
||||
return config.Config{}, fmt.Errorf("resolve the task's models: %w", err)
|
||||
}
|
||||
return cfg.WithOrgModels(models), nil
|
||||
cfg.Providers = cfg.ModelProviders()
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// ModelSession is what a native container needs for its model proxy session.
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log/slog"
|
||||
"testing"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/agentexec/modelloop"
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
||||
"github.com/samcharles93/archie-core/internal/eventbus"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/egress"
|
||||
"github.com/samcharles93/archie-core/internal/taskrun"
|
||||
"github.com/samcharles93/archie-core/internal/taskstate"
|
||||
)
|
||||
|
||||
// recordingModelEgress keeps the providers a task's model session was opened
|
||||
// with.
|
||||
type recordingModelEgress struct {
|
||||
providers map[string]config.Provider
|
||||
}
|
||||
|
||||
func (r *recordingModelEgress) Open(_, _ string, providers map[string]config.Provider) (ModelSession, error) {
|
||||
r.providers = providers
|
||||
return ModelSession{}, nil
|
||||
}
|
||||
|
||||
func (r *recordingModelEgress) Close(context.Context, string) {}
|
||||
|
||||
// A task's model session carries only the providers its models name: a
|
||||
// configured provider no alias reaches must never be required, and the
|
||||
// provider the task does use arrives with the variable its credential is read
|
||||
// from.
|
||||
func TestOpenModelEgressCarriesOnlyReferencedProviders(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
egressProxy := &recordingModelEgress{}
|
||||
d := &Daemon{
|
||||
Cfg: config.NewHolder(config.Config{
|
||||
Models: map[string]string{"default": "used/model-one"},
|
||||
Providers: map[string]config.Provider{
|
||||
"used": {Class: "deepseek", APIKey: config.SecretRef{Engine: "env", Key: "USED_KEY"}},
|
||||
"unused": {Class: "gemini", APIKeyEnv: "UNUSED_KEY"},
|
||||
},
|
||||
}),
|
||||
ModelEgress: egressProxy,
|
||||
}
|
||||
task := &workflow.Task{ID: 7, WorkflowDefinitionYAML: "id: w\nsteps:\n - type: agent.run\n"}
|
||||
|
||||
if _, _, err := d.openModelEgress(t.Context(), task, "run-credential"); err != nil {
|
||||
t.Fatalf("open model egress: %v", err)
|
||||
}
|
||||
if _, ok := egressProxy.providers["unused"]; ok {
|
||||
t.Fatalf("a provider no model names reached the session: %v", egressProxy.providers)
|
||||
}
|
||||
used, ok := egressProxy.providers["used"]
|
||||
if !ok {
|
||||
t.Fatalf("the provider the task's model names is missing: %v", egressProxy.providers)
|
||||
}
|
||||
if want := config.ProviderKeyEnvName("used"); used.APIKeyEnv != want {
|
||||
t.Fatalf("provider variable = %q, want %q", used.APIKeyEnv, want)
|
||||
}
|
||||
}
|
||||
|
||||
// capturingBus records the taskrun request the daemon sent.
|
||||
type capturingBus struct {
|
||||
request []byte
|
||||
reply []byte
|
||||
}
|
||||
|
||||
func (b *capturingBus) PublishUnique(context.Context, string, string, []byte) error { return nil }
|
||||
|
||||
func (b *capturingBus) Fetch(context.Context) (eventbus.Message, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (b *capturingBus) Request(_ context.Context, _ string, payload []byte) ([]byte, error) {
|
||||
b.request = payload
|
||||
return b.reply, nil
|
||||
}
|
||||
|
||||
// The provider a task's model runs on reaches the container with a variable the
|
||||
// agent can read, so the runtime authenticates through the egress proxy instead
|
||||
// of running the provider unauthenticated.
|
||||
func TestRunViaAgentCarriesProviderCredentialVariable(t *testing.T) {
|
||||
reply, err := json.Marshal(taskrun.Response{Status: taskstate.Completed})
|
||||
if err != nil {
|
||||
t.Fatalf("marshal response: %v", err)
|
||||
}
|
||||
bus := &capturingBus{reply: reply}
|
||||
d := &Daemon{
|
||||
Cfg: config.NewHolder(config.Config{
|
||||
Models: map[string]string{"default": "used/model-one"},
|
||||
Providers: map[string]config.Provider{
|
||||
"used": {Class: "deepseek", APIKey: config.SecretRef{Engine: "env", Key: "USED_KEY"}},
|
||||
},
|
||||
}),
|
||||
Tasks: bus,
|
||||
Log: slog.New(slog.DiscardHandler),
|
||||
}
|
||||
task := &workflow.Task{ID: 7, Attempt: 1, Workflow: "implement", WorkflowDefinitionYAML: "id: w\nsteps:\n - type: agent.run\n"}
|
||||
|
||||
d.runViaAgent(t.Context(), task, config.Repo{}, config.AgentProfile{}, nil, "run-credential")
|
||||
|
||||
if bus.request == nil {
|
||||
t.Fatal("the daemon sent no taskrun request")
|
||||
}
|
||||
var req taskrun.Request
|
||||
if err := json.Unmarshal(bus.request, &req); err != nil {
|
||||
t.Fatalf("decode taskrun request: %v", err)
|
||||
}
|
||||
provider, ok := req.Providers["used"]
|
||||
if !ok {
|
||||
t.Fatalf("the task's provider is missing from the request: %v", req.Providers)
|
||||
}
|
||||
if provider.APIKeyEnv == "" {
|
||||
t.Fatalf("the provider reached the container without a credential variable: %+v", provider)
|
||||
}
|
||||
sentinel := egress.SentinelFor("used")
|
||||
t.Setenv(provider.APIKeyEnv, sentinel)
|
||||
|
||||
rt := modelloop.NewRuntime(req.Providers, nil)
|
||||
if rt == nil {
|
||||
t.Fatal("no runtime built from the request's providers")
|
||||
}
|
||||
auth, err := rt.ResolveAuth(t.Context(), "used")
|
||||
if err != nil {
|
||||
t.Fatalf("resolve auth: %v", err)
|
||||
}
|
||||
if auth.Token != sentinel {
|
||||
t.Fatalf("the runtime resolves %q, so the egress sentinel is never sent", auth.Token)
|
||||
}
|
||||
}
|
||||
@@ -50,6 +50,7 @@ type Session struct {
|
||||
runtime rules
|
||||
injections []injection
|
||||
substitutions []substitution
|
||||
secrets map[string]string
|
||||
oauth []oauthRule
|
||||
atRun atomic.Bool
|
||||
}
|
||||
@@ -109,6 +110,10 @@ type SessionOptions struct {
|
||||
// Substitutions swap a service's sentinel for its key on requests to the
|
||||
// service's host.
|
||||
Substitutions []Substitution
|
||||
// Secrets are credentials the run itself carries, keyed by service. A
|
||||
// substitution prefers them over the Resolver, so a run-supplied key needs
|
||||
// no credential binding.
|
||||
Secrets map[string]string
|
||||
}
|
||||
|
||||
// Proxy is the egress proxy sandbox containers reach through their relay.
|
||||
@@ -149,7 +154,7 @@ func (p *Proxy) Register(opts SessionOptions) (*Session, error) {
|
||||
s := &Session{
|
||||
token: opts.Token, org: opts.Org, runCredential: opts.RunCredential, principal: opts.Principal,
|
||||
injections: compileInjections(opts.Credentials, opts.Bound), oauth: compileOAuthRules(opts.Credentials, opts.Bound),
|
||||
substitutions: compileSubstitutions(opts.Substitutions),
|
||||
substitutions: compileSubstitutions(opts.Substitutions), secrets: opts.Secrets,
|
||||
}
|
||||
if opts.RunCredential != "" && p.taskTools != nil {
|
||||
var token [32]byte
|
||||
|
||||
@@ -48,26 +48,40 @@ func (p *Proxy) substitute(ctx context.Context, s *Session, r *http.Request, hos
|
||||
if !requestCarries(r, sentinel) {
|
||||
continue
|
||||
}
|
||||
if p.resolver == nil {
|
||||
return fmt.Errorf("credential %q has no resolver", sub.service)
|
||||
}
|
||||
secret, err := p.resolver.Resolve(ctx, s.credential(), sub.service)
|
||||
if errors.Is(err, ErrUnbound) {
|
||||
return fmt.Errorf("credential %q is not bound for this run", sub.service)
|
||||
}
|
||||
key, err := p.keyFor(ctx, s, sub.service)
|
||||
if err != nil {
|
||||
return fmt.Errorf("resolve credential %q: %w", sub.service, err)
|
||||
return err
|
||||
}
|
||||
for _, values := range r.Header {
|
||||
for i, v := range values {
|
||||
values[i] = strings.ReplaceAll(v, sentinel, secret)
|
||||
values[i] = strings.ReplaceAll(v, sentinel, key)
|
||||
}
|
||||
}
|
||||
r.URL.RawQuery = strings.ReplaceAll(r.URL.RawQuery, sentinel, secret)
|
||||
r.URL.RawQuery = strings.ReplaceAll(r.URL.RawQuery, sentinel, key)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// keyFor is the key the run itself carries for service, or the one its
|
||||
// credential binding resolves. A run-supplied key needs no binding; a service
|
||||
// the session holds no key for keeps the binding's ErrUnbound semantics.
|
||||
func (p *Proxy) keyFor(ctx context.Context, s *Session, service string) (string, error) {
|
||||
if key, ok := s.secrets[service]; ok {
|
||||
return key, nil
|
||||
}
|
||||
if p.resolver == nil {
|
||||
return "", fmt.Errorf("credential %q has no resolver", service)
|
||||
}
|
||||
key, err := p.resolver.Resolve(ctx, s.credential(), service)
|
||||
if errors.Is(err, ErrUnbound) {
|
||||
return "", fmt.Errorf("credential %q is not bound for this run", service)
|
||||
}
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("resolve credential %q: %w", service, err)
|
||||
}
|
||||
return key, nil
|
||||
}
|
||||
|
||||
func requestCarries(r *http.Request, sentinel string) bool {
|
||||
if strings.Contains(r.URL.RawQuery, sentinel) {
|
||||
return true
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
package egress
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A session's own resolved keys are substituted before the binding resolver is
|
||||
// consulted: a run that carries a provider credential needs no credential
|
||||
// binding for it. A service the session holds no key for still resolves through
|
||||
// its binding, and an unbound one still stops the request.
|
||||
func TestSubstitutionPrefersSessionKeys(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
host string
|
||||
secrets map[string]string
|
||||
resolve func() (string, error)
|
||||
wantAuth string
|
||||
wantStatus int
|
||||
wantResolve int
|
||||
}{
|
||||
{
|
||||
name: "the session key replaces the sentinel on the service host",
|
||||
host: "api.alpha.example", secrets: map[string]string{"alpha": "session-key"},
|
||||
wantAuth: "Bearer session-key", wantStatus: http.StatusOK,
|
||||
},
|
||||
{
|
||||
name: "the session key never reaches another host",
|
||||
host: "attacker.example", secrets: map[string]string{"alpha": "session-key"},
|
||||
wantAuth: "Bearer " + SentinelFor("alpha"), wantStatus: http.StatusOK,
|
||||
},
|
||||
{
|
||||
name: "a service with no session key resolves from its binding",
|
||||
host: "api.beta.example", secrets: map[string]string{"alpha": "session-key"},
|
||||
resolve: func() (string, error) { return "bound-key", nil },
|
||||
wantAuth: "Bearer bound-key", wantStatus: http.StatusOK, wantResolve: 1,
|
||||
},
|
||||
{
|
||||
name: "an unbound service with no session key stops the request",
|
||||
host: "api.beta.example",
|
||||
resolve: func() (string, error) { return "", ErrUnbound },
|
||||
wantStatus: http.StatusBadGateway, wantResolve: 1,
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var seen *http.Request
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { seen = r }))
|
||||
defer upstream.Close()
|
||||
|
||||
resolved := 0
|
||||
proxy := NewProxy(nil, ProxyOptions{
|
||||
Resolver: ResolverFunc(func(_ context.Context, credential, _ string) (string, error) {
|
||||
if credential != "host-run-grant" {
|
||||
t.Error("resolver received the proxy password")
|
||||
return "", ErrUnbound
|
||||
}
|
||||
resolved++
|
||||
if tc.resolve == nil {
|
||||
return "", ErrUnbound
|
||||
}
|
||||
return tc.resolve()
|
||||
}),
|
||||
Dial: func(ctx context.Context, _, _ string) (net.Conn, error) {
|
||||
return (&net.Dialer{}).DialContext(ctx, "tcp", upstream.Listener.Addr().String())
|
||||
},
|
||||
})
|
||||
service := "alpha"
|
||||
if tc.host == "api.beta.example" {
|
||||
service = "beta"
|
||||
}
|
||||
session, err := proxy.Register(SessionOptions{
|
||||
Token: "proxy-password", RunCredential: "host-run-grant",
|
||||
Substitutions: []Substitution{{Service: "alpha", Host: "api.alpha.example"}, {Service: "beta", Host: "api.beta.example"}},
|
||||
Secrets: tc.secrets,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("register session: %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequestWithContext(context.Background(), http.MethodPost, "http://"+tc.host+"/v1/chat", nil)
|
||||
req.Header.Set("Authorization", "Bearer "+SentinelFor(service))
|
||||
rec := httptest.NewRecorder()
|
||||
proxy.forward(context.Background(), rec, req, session, "http", tc.host, 80)
|
||||
|
||||
if resolved != tc.wantResolve {
|
||||
t.Fatalf("resolver called %d times, want %d", resolved, tc.wantResolve)
|
||||
}
|
||||
if rec.Code != tc.wantStatus {
|
||||
t.Fatalf("status = %d, want %d (%s)", rec.Code, tc.wantStatus, rec.Body.String())
|
||||
}
|
||||
if tc.wantAuth == "" {
|
||||
if seen != nil {
|
||||
t.Fatal("a refused request was forwarded upstream")
|
||||
}
|
||||
return
|
||||
}
|
||||
if seen == nil {
|
||||
t.Fatalf("request not forwarded: %d %q", rec.Code, rec.Body.String())
|
||||
}
|
||||
if got := seen.Header.Get("Authorization"); got != tc.wantAuth {
|
||||
t.Fatalf("upstream saw %q, want %q", got, tc.wantAuth)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user