fix(extension): confine each extension's network to its accepted egress hosts

This commit is contained in:
sam committed 2026-10-05 23:30:52 +11:00
1 parent 143209493a
commit 209ef6f71e
9 files changed
+516 -25

No files matched your search

+1 -1
View File
@@ -85,7 +85,7 @@ func (e *forgeExtensions) start(ctx context.Context, pkg storepkg.Installed, fil
if err != nil {
return nil, err
}
spec := extension.Spec{Name: "forge/" + instance, Path: binary, SHA256: sum, Env: slices.Clone(pkg.AcceptedAuthority.Env)}
spec := extension.Spec{Name: "forge/" + instance, Path: binary, SHA256: sum, Env: slices.Clone(pkg.AcceptedAuthority.Env), Egress: slices.Clone(pkg.AcceptedAuthority.EgressHosts)}
return forgeext.Open(ctx, e.host, forgeext.Instance{Spec: spec, Host: host, Token: token, Settings: settings}, e.log)
}
@@ -93,6 +93,6 @@ func buildExtensionChannel(ctx context.Context, host *extension.Host, source ext
}
resolved[name] = secretValue
}
spec := extension.Spec{Name: "channel/" + pkg.Name, Path: binary, SHA256: sum, Env: slices.Clone(pkg.AcceptedAuthority.Env)}
spec := extension.Spec{Name: "channel/" + pkg.Name, Path: binary, SHA256: sum, Env: slices.Clone(pkg.AcceptedAuthority.Env), Egress: slices.Clone(pkg.AcceptedAuthority.EgressHosts)}
return channelext.New(host, pkg.Name, spec, plain, resolved), nil
}
+138
View File
@@ -0,0 +1,138 @@
package egress
import (
"context"
"fmt"
"io"
"net"
"net/http"
"net/http/httputil"
"strconv"
"sync"
)
// Tunnel is an allow-list forward proxy that never terminates TLS: CONNECT is
// spliced byte for byte to an admitted host, and absolute-form http requests
// are forwarded. Extensions get it rather than Proxy because they carry their
// own credentials and never trust the daemon CA.
type Tunnel struct {
rules rules
dial func(ctx context.Context, network, addr string) (net.Conn, error)
}
// NewTunnel admits the hosts allow names, in network-policy@1 form. A host
// named exactly may resolve to a private address, since the operator named
// it (a LAN forge); a wildcard match may not. Loopback is refused whatever
// is named: it would reach archie's own services.
func NewTunnel(allow []string) *Tunnel {
t := &Tunnel{rules: rules{allow: compilePatterns(allow)}}
t.dial = t.dialGuarded
return t
}
func (t *Tunnel) ServeHTTP(w http.ResponseWriter, r *http.Request) {
defaultPort, hostport := 443, r.Host
if r.Method != http.MethodConnect {
if !r.URL.IsAbs() || r.URL.Scheme != "http" {
http.Error(w, "only CONNECT and absolute-form http requests are proxied", http.StatusBadRequest)
return
}
defaultPort, hostport = 80, r.URL.Host
}
host, port, err := splitTarget(hostport, defaultPort)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
target := net.JoinHostPort(host, strconv.Itoa(port))
if !t.rules.allows(host, port) {
http.Error(w, "egress to "+target+" is not allowed", http.StatusForbidden)
return
}
if r.Method == http.MethodConnect {
t.splice(w, r, target)
return
}
rp := &httputil.ReverseProxy{
Transport: &http.Transport{DialContext: t.dial},
FlushInterval: -1,
Rewrite: func(pr *httputil.ProxyRequest) {
pr.Out.URL.Host = target
pr.Out.Header.Del("Proxy-Connection")
},
ErrorHandler: func(w http.ResponseWriter, _ *http.Request, err error) {
http.Error(w, "upstream "+target+": "+err.Error(), http.StatusBadGateway)
},
}
rp.ServeHTTP(w, r)
}
func (t *Tunnel) splice(w http.ResponseWriter, r *http.Request, target string) {
upstream, err := t.dial(r.Context(), "tcp", target)
if err != nil {
http.Error(w, "upstream "+target+": "+err.Error(), http.StatusBadGateway)
return
}
hijacker, ok := w.(http.Hijacker)
if !ok {
_ = upstream.Close()
http.Error(w, "connection cannot be tunnelled", http.StatusInternalServerError)
return
}
client, buffered, err := hijacker.Hijack()
if err != nil {
_ = upstream.Close()
return
}
defer func() { _ = client.Close(); _ = upstream.Close() }()
if _, err := client.Write([]byte("HTTP/1.1 200 Connection Established\r\n\r\n")); err != nil {
return
}
var wg sync.WaitGroup
wg.Go(func() {
_, _ = io.Copy(upstream, buffered)
closeWrite(upstream)
})
_, _ = io.Copy(client, upstream)
closeWrite(client)
wg.Wait()
}
func closeWrite(c net.Conn) {
if cw, ok := c.(interface{ CloseWrite() error }); ok {
_ = cw.CloseWrite()
return
}
_ = c.Close()
}
// dialGuarded applies the Proxy's upstream guard, counting a host the allow
// list names exactly as naming every address it resolves to.
func (t *Tunnel) dialGuarded(ctx context.Context, network, addr string) (net.Conn, error) {
host, port, err := net.SplitHostPort(addr)
if err != nil {
return nil, err
}
named := t.namesHost(normalizeHost(host))
addrs, err := net.DefaultResolver.LookupIPAddr(ctx, host)
if err != nil {
return nil, err
}
guardErr := fmt.Errorf("%s resolved to no address", host)
for _, a := range addrs {
if guardErr = guardUpstream(a.IP, named); guardErr != nil {
continue
}
return (&net.Dialer{Timeout: handshakeTimeout}).DialContext(ctx, network, net.JoinHostPort(a.IP.String(), port))
}
return nil, guardErr
}
func (t *Tunnel) namesHost(host string) bool {
for _, p := range t.rules.allow {
if !p.everything && p.suffix == "" && p.host == host {
return true
}
}
return false
}
@@ -0,0 +1,159 @@
package extension
import (
"context"
"errors"
"fmt"
"io"
"net"
"os"
"os/exec"
"os/signal"
"runtime"
"syscall"
"golang.org/x/sys/unix"
)
// launcherArg makes any binary importing this package the confined
// launcher: the extension's parent inside its network namespace.
const launcherArg = "__extension-egress"
// proxyAddr is where the launcher serves the extension's only way out.
const proxyAddr = "127.0.0.1:3128"
func init() {
if len(os.Args) > 2 && os.Args[1] == launcherArg {
os.Exit(launch(os.Args[2]))
}
}
// confinedCommand re-execs this binary as the launcher in a fresh user and
// network namespace. The namespace holds only loopback, so the extension
// cannot reach anything except through the launcher's relay to socket. Where
// the kernel refuses the namespaces, the start fails: there is no unconfined
// fallback.
func confinedCommand(ctx context.Context, binary *os.File, socket string) (*exec.Cmd, error) {
self, err := os.Executable()
if err != nil {
return nil, err
}
uid, gid := os.Getuid(), os.Getgid()
cmd := exec.CommandContext(ctx, self, launcherArg, socket)
cmd.ExtraFiles = []*os.File{binary}
cmd.SysProcAttr = &syscall.SysProcAttr{
Cloneflags: syscall.CLONE_NEWUSER | syscall.CLONE_NEWNET,
UidMappings: []syscall.SysProcIDMap{{ContainerID: uid, HostID: uid, Size: 1}},
GidMappings: []syscall.SysProcIDMap{{ContainerID: gid, HostID: gid, Size: 1}},
// Only the launcher holds this, to bring loopback up; it clears it
// before the extension starts.
AmbientCaps: []uintptr{unix.CAP_NET_ADMIN},
Pdeathsig: syscall.SIGKILL,
}
return cmd, nil
}
// launch runs inside the namespace: loopback up, relay listening, then the
// extension from descriptor 3 with every capability gone.
func launch(socket string) int {
ctx := context.Background()
if err := loopbackUp(); err != nil {
fmt.Fprintln(os.Stderr, "extension egress: loopback:", err)
return 1
}
ln, err := (&net.ListenConfig{}).Listen(ctx, "tcp", proxyAddr)
if err != nil {
fmt.Fprintln(os.Stderr, "extension egress: relay:", err)
return 1
}
go relay(ctx, ln, socket)
if err := unix.Prctl(unix.PR_CAP_AMBIENT, unix.PR_CAP_AMBIENT_CLEAR_ALL, 0, 0, 0); err != nil {
fmt.Fprintln(os.Stderr, "extension egress: drop capabilities:", err)
return 1
}
if err := unix.Prctl(unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); err != nil {
fmt.Fprintln(os.Stderr, "extension egress: no_new_privs:", err)
return 1
}
syscall.CloseOnExec(3)
return runExtension(ctx, fmt.Sprintf("/proc/%d/fd/3", os.Getpid()))
}
// runExtension starts the extension as the launcher's child and mirrors its
// exit. Pdeathsig ties the child to this thread, so killing the launcher
// kills the extension.
func runExtension(ctx context.Context, path string) int {
runtime.LockOSThread()
proxy := "http://" + proxyAddr
cmd := exec.CommandContext(ctx, path) //nolint:gosec // the host-verified binary, by descriptor
cmd.Stdin, cmd.Stdout, cmd.Stderr = os.Stdin, os.Stdout, os.Stderr
cmd.Env = append(os.Environ(),
"HTTP_PROXY="+proxy, "HTTPS_PROXY="+proxy, "ALL_PROXY="+proxy,
"http_proxy="+proxy, "https_proxy="+proxy, "all_proxy="+proxy, "NO_PROXY=", "no_proxy=")
cmd.SysProcAttr = &syscall.SysProcAttr{Pdeathsig: syscall.SIGKILL}
if err := cmd.Start(); err != nil {
fmt.Fprintln(os.Stderr, "extension egress: start:", err)
return 1
}
signals := make(chan os.Signal, 1)
signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
go func() {
for s := range signals {
_ = cmd.Process.Signal(s)
}
}()
err := cmd.Wait()
if exit, ok := errors.AsType[*exec.ExitError](err); ok {
return exit.ExitCode()
}
if err != nil {
return 1
}
return 0
}
func loopbackUp() error {
fd, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM|unix.SOCK_CLOEXEC, 0)
if err != nil {
return err
}
defer func() { _ = unix.Close(fd) }()
ifr, err := unix.NewIfreq("lo")
if err != nil {
return err
}
if err := unix.IoctlIfreq(fd, unix.SIOCGIFFLAGS, ifr); err != nil {
return err
}
ifr.SetUint16(ifr.Uint16() | unix.IFF_UP)
return unix.IoctlIfreq(fd, unix.SIOCSIFFLAGS, ifr)
}
// relay carries each proxy connection to the host's gate socket.
func relay(ctx context.Context, ln net.Listener, socket string) {
for {
conn, err := ln.Accept()
if err != nil {
return
}
go func() {
defer func() { _ = conn.Close() }()
upstream, err := (&net.Dialer{}).DialContext(ctx, "unix", socket)
if err != nil {
return
}
defer func() { _ = upstream.Close() }()
go func() {
_, _ = io.Copy(upstream, conn)
closeWrite(upstream)
}()
_, _ = io.Copy(conn, upstream)
}()
}
}
func closeWrite(c net.Conn) {
if cw, ok := c.(interface{ CloseWrite() error }); ok {
_ = cw.CloseWrite()
}
}
@@ -0,0 +1,16 @@
//go:build !linux
package extension
import (
"context"
"errors"
"os"
"os/exec"
)
// confinedCommand refuses: extensions run only where their egress can be
// confined to the hosts the operator accepted.
func confinedCommand(context.Context, *os.File, string) (*exec.Cmd, error) {
return nil, errors.New("extensions need Linux network namespaces to confine their egress")
}
+47
View File
@@ -0,0 +1,47 @@
package extension
import (
"context"
"errors"
"net"
"net/http"
"os"
"path/filepath"
"time"
"github.com/samcharles93/archie-core/internal/infrastructure/egress"
)
// gate is the host end of one extension's egress: a Unix socket serving an
// allow-list proxy. A path socket is reachable from the extension's network
// namespace, where nothing else outside it is.
type gate struct {
dir string
socket string
server *http.Server
}
func openGate(ctx context.Context, allow []string) (*gate, error) {
dir, err := os.MkdirTemp("", "archie-ext-")
if err != nil {
return nil, err
}
g := &gate{dir: dir, socket: filepath.Join(dir, "egress.sock")}
ln, err := (&net.ListenConfig{}).Listen(ctx, "unix", g.socket)
if err != nil {
_ = os.RemoveAll(dir)
return nil, err
}
g.server = &http.Server{Handler: egress.NewTunnel(allow), ReadHeaderTimeout: 30 * time.Second}
go func() {
if err := g.server.Serve(ln); err != nil && !errors.Is(err, http.ErrServerClosed) {
_ = ln.Close()
}
}()
return g, nil
}
func (g *gate) close() {
_ = g.server.Close()
_ = os.RemoveAll(g.dir)
}
+69 -20
View File
@@ -3,12 +3,14 @@
package extension
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"os"
"os/exec"
"sync"
"github.com/hashicorp/go-hclog"
@@ -34,6 +36,10 @@ type Spec struct {
// Env names the host environment variables the extension may see. PATH and
// HOME are always passed so a plugin can find the CLI it wraps.
Env []string
// Egress is the accepted egress host list. The extension runs in its own
// network namespace and reaches only these hosts, through a proxy; an
// empty list reaches nothing.
Egress []string
}
// Host owns the running extension processes, keyed by Spec.Name.
@@ -41,12 +47,23 @@ type Host struct {
log hclog.Logger
mu sync.Mutex
procs map[string]*goplugin.Client
procs map[string]*proc
}
// proc is one running extension and the egress gate it alone can reach.
type proc struct {
client *goplugin.Client
gate *gate
}
func (p *proc) kill() {
p.client.Kill()
p.gate.close()
}
// NewHost returns a Host that reports plugin lifecycle problems to log.
func NewHost(log hclog.Logger) *Host {
return &Host{log: log, procs: make(map[string]*goplugin.Client)}
return &Host{log: log, procs: make(map[string]*proc)}
}
// Start launches spec serving surface and returns the dispensed client. An
@@ -57,33 +74,47 @@ func (h *Host) Start(ctx context.Context, spec Spec, surface string, impl goplug
if err != nil || len(sum) != sha256.Size {
return nil, fmt.Errorf("extension %q: sha256 must be a hex digest", spec.Name)
}
cmd := exec.CommandContext(ctx, spec.Path)
// The verified file is what runs: the confined launcher execs this open
// descriptor, so swapping the path after the check changes nothing.
binary, err := openVerified(spec.Path, sum)
if err != nil {
return nil, fmt.Errorf("extension %q: %w", spec.Name, err)
}
defer func() { _ = binary.Close() }()
g, err := openGate(ctx, spec.Egress)
if err != nil {
return nil, fmt.Errorf("extension %q: egress: %w", spec.Name, err)
}
cmd, err := confinedCommand(ctx, binary, g.socket)
if err != nil {
g.close()
return nil, fmt.Errorf("extension %q: %w", spec.Name, err)
}
cmd.Env = passEnv(spec.Env)
client := goplugin.NewClient(&goplugin.ClientConfig{
p := &proc{gate: g, client: goplugin.NewClient(&goplugin.ClientConfig{
HandshakeConfig: Handshake,
Plugins: goplugin.PluginSet{surface: impl},
Cmd: cmd,
AllowedProtocols: []goplugin.Protocol{goplugin.ProtocolGRPC},
SecureConfig: &goplugin.SecureConfig{Checksum: sum, Hash: sha256.New()},
SkipHostEnv: true,
Logger: h.log.Named(spec.Name),
})
rpc, err := client.Client()
})}
rpc, err := p.client.Client()
if err != nil {
client.Kill()
p.kill()
return nil, fmt.Errorf("extension %q: connect: %w", spec.Name, err)
}
raw, err := rpc.Dispense(surface)
if err != nil {
client.Kill()
p.kill()
return nil, fmt.Errorf("extension %q: dispense %s: %w", spec.Name, surface, err)
}
h.mu.Lock()
old := h.procs[spec.Name]
h.procs[spec.Name] = client
h.procs[spec.Name] = p
h.mu.Unlock()
if old != nil {
old.Kill()
old.kill()
}
return raw, nil
}
@@ -92,11 +123,11 @@ func (h *Host) Start(ctx context.Context, spec Spec, surface string, impl goplug
// error.
func (h *Host) Stop(name string) {
h.mu.Lock()
client := h.procs[name]
p := h.procs[name]
delete(h.procs, name)
h.mu.Unlock()
if client != nil {
client.Kill()
if p != nil {
p.kill()
}
}
@@ -104,21 +135,39 @@ func (h *Host) Stop(name string) {
func (h *Host) Alive(name string) bool {
h.mu.Lock()
defer h.mu.Unlock()
client := h.procs[name]
return client != nil && !client.Exited()
p := h.procs[name]
return p != nil && !p.client.Exited()
}
// Close stops every extension.
func (h *Host) Close() {
h.mu.Lock()
procs := h.procs
h.procs = make(map[string]*goplugin.Client)
h.procs = make(map[string]*proc)
h.mu.Unlock()
for _, client := range procs {
client.Kill()
for _, p := range procs {
p.kill()
}
}
// openVerified opens the extension binary and checks it against sum.
func openVerified(path string, sum []byte) (*os.File, error) {
f, err := os.Open(path) //nolint:gosec // the path is the installed package's binary
if err != nil {
return nil, err
}
h := sha256.New()
if _, err := io.Copy(h, f); err != nil {
_ = f.Close()
return nil, err
}
if !bytes.Equal(h.Sum(nil), sum) {
_ = f.Close()
return nil, errors.New("binary does not match its checksum")
}
return f, nil
}
func passEnv(names []string) []string {
env := make([]string, 0, len(names)+2)
for _, name := range append([]string{"PATH", "HOME"}, names...) {
@@ -149,7 +149,7 @@ func (r *Runner) ensure(ctx context.Context, pkg storepkg.Installed, setting con
if err != nil {
return err
}
spec := extension.Spec{Name: pkg.Name, Path: binary, SHA256: sum, Env: slices.Clone(pkg.AcceptedAuthority.Env)}
spec := extension.Spec{Name: pkg.Name, Path: binary, SHA256: sum, Env: slices.Clone(pkg.AcceptedAuthority.Env), Egress: slices.Clone(pkg.AcceptedAuthority.EgressHosts)}
engine, err := Start(ctx, r.host, spec, setting.Settings)
if err != nil {
return err
@@ -7,10 +7,16 @@ import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"io"
"log/slog"
"net"
"net/http"
"net/http/httptest"
"net/url"
"os"
"strings"
"testing"
"time"
"github.com/hashicorp/go-hclog"
goplugin "github.com/hashicorp/go-plugin"
@@ -37,16 +43,53 @@ func (e *echoEngine) Configure(_ context.Context, r *secretenginev1.ConfigureReq
return &secretenginev1.ConfigureResponse{}, nil
}
func (e *echoEngine) Resolve(_ context.Context, r *secretenginev1.ResolveRequest) (*secretenginev1.ResolveResponse, error) {
func (e *echoEngine) Resolve(ctx context.Context, r *secretenginev1.ResolveRequest) (*secretenginev1.ResolveResponse, error) {
if name, ok := strings.CutPrefix(r.GetKey(), "ENV:"); ok {
return &secretenginev1.ResolveResponse{Value: os.Getenv(name)}, nil
}
if url, ok := strings.CutPrefix(r.GetKey(), "GET:"); ok {
return &secretenginev1.ResolveResponse{Value: fetch(url)}, nil
}
if addr, ok := strings.CutPrefix(r.GetKey(), "DIAL:"); ok {
return &secretenginev1.ResolveResponse{Value: dial(ctx, addr)}, nil
}
if v, ok := e.settings[r.GetKey()]; ok {
return &secretenginev1.ResolveResponse{Value: v}, nil
}
return nil, status.Error(codes.NotFound, "no such key")
}
// dial reports whether a raw TCP connection to addr succeeds.
func dial(ctx context.Context, addr string) string {
conn, err := (&net.Dialer{Timeout: 2 * time.Second}).DialContext(ctx, "tcp", addr)
if err != nil {
return "dial failed"
}
_ = conn.Close()
return "connected"
}
// fetch GETs target through the proxy HTTP_PROXY names, reporting the body or
// the failing status. The proxy is set explicitly because the environment
// lookup never proxies loopback, which is where the test servers listen.
func fetch(target string) string {
proxy, err := url.Parse(os.Getenv("HTTP_PROXY"))
if err != nil || proxy.Host == "" {
return "no proxy"
}
client := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxy)}, Timeout: 5 * time.Second}
resp, err := client.Get(target) //nolint:noctx // test plugin
if err != nil {
return "error: " + err.Error()
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
return resp.Status
}
body, _ := io.ReadAll(resp.Body)
return string(body)
}
func TestMain(m *testing.M) {
if os.Getenv(extension.Handshake.MagicCookieKey) != "" {
goplugin.Serve(&goplugin.ServeConfig{
@@ -69,10 +112,45 @@ func selfDigest(t *testing.T) string {
return hex.EncodeToString(sum[:])
}
// lanAddress is a non-loopback IPv4 address of this machine. Without one the
// egress cases cannot run, and they fail rather than skip.
func lanAddress(t *testing.T) string {
t.Helper()
addrs, err := net.InterfaceAddrs()
if err != nil {
t.Fatal(err)
}
for _, a := range addrs {
if ip, ok := a.(*net.IPNet); ok && ip.IP.To4() != nil && !ip.IP.IsLoopback() {
return ip.IP.String()
}
}
t.Fatal("no non-loopback IPv4 address to stand in for an egress host")
return ""
}
func TestHostLaunch(t *testing.T) {
t.Setenv("ARCHIE_TEST_SECRET", "host-only")
t.Setenv("ARCHIE_TEST_PASSED", "granted")
good := selfDigest(t)
// The egress gate refuses loopback, so the servers listen on this
// machine's network address, standing in for a private-network host.
lan := lanAddress(t)
serve := func() string {
ln, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", net.JoinHostPort(lan, "0"))
if err != nil {
t.Fatal(err)
}
srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = io.WriteString(w, "reached") }))
srv.Listener = ln
srv.Start()
t.Cleanup(srv.Close)
return ln.Addr().String()
}
accepted, other := serve(), serve()
loopbackServer := httptest.NewServer(http.NotFoundHandler())
t.Cleanup(loopbackServer.Close)
loopback := loopbackServer.Listener.Addr().String()
tests := []struct {
name string
@@ -81,6 +159,10 @@ func TestHostLaunch(t *testing.T) {
want string
wantErr string
}{
{name: "an accepted egress host is reachable through the proxy", sha: good, key: "GET:http://" + accepted, want: "reached"},
{name: "a host that was not accepted is refused", sha: good, key: "GET:http://" + other, want: "403 Forbidden"},
{name: "a direct connection bypassing the proxy fails", sha: good, key: "DIAL:" + accepted, want: "dial failed"},
{name: "loopback is refused even when accepted", sha: good, key: "GET:http://" + loopback, want: "502 Bad Gateway"},
{name: "settings reach the engine", sha: good, key: "color", want: "blue"},
{name: "an unknown key is not found", sha: good, key: "missing", wantErr: "not found"},
{name: "an env var outside the allowlist is invisible", sha: good, key: "ENV:ARCHIE_TEST_SECRET", want: ""},
@@ -91,7 +173,7 @@ func TestHostLaunch(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
host := extension.NewHost(hclog.NewNullLogger())
t.Cleanup(host.Close)
spec := extension.Spec{Name: "echo", Path: os.Args[0], SHA256: tt.sha, Env: []string{"ARCHIE_TEST_PASSED"}}
spec := extension.Spec{Name: "echo", Path: os.Args[0], SHA256: tt.sha, Env: []string{"ARCHIE_TEST_PASSED"}, Egress: []string{accepted, loopback}}
engine, err := Start(context.Background(), host, spec, map[string]string{"color": "blue"})
if tt.sha != good {
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {