fix(sdk,semaphore): an action that may have run is an unknown outcome, not an error; retry a failed task status read (archie #2001)
publish / publish (push) Successful in 1m12s

This commit is contained in:
sam committed 2026-11-16 03:57:20 +11:00
1 parent 8df55d3225
commit 53ba618b0b
4 files changed
+81 -9

No files matched your search

+12 -6
View File
@@ -296,21 +296,27 @@ func (s *semaphore) launch(ctx context.Context, t Template, in actionInput, mess
return tk, nil
}
// wait polls the task until it ends. A failed status read is retried until
// the deadline: Semaphore can stall reads while a task starts, and the task
// runs on regardless.
func (s *semaphore) wait(ctx context.Context, id int) (task, error) {
ctx, cancel := context.WithTimeout(ctx, s.deadline)
defer cancel()
var last string
for {
var tk task
if err := s.do(ctx, http.MethodGet, fmt.Sprintf("/api/project/%d/tasks/%d", s.project, id), nil, &tk); err != nil {
return task{}, fmt.Errorf("task %d: %w", id, err)
}
switch tk.Status {
case "success", "error", "stopped", "rejected":
return tk, nil
last = err.Error()
} else {
switch tk.Status {
case "success", "error", "stopped", "rejected":
return tk, nil
}
last = tk.Status
}
select {
case <-ctx.Done():
return task{}, fmt.Errorf("task %d still %s: %w", id, tk.Status, ctx.Err())
return task{}, fmt.Errorf("task %d launched, last seen %s: %w", id, last, ctx.Err())
case <-time.After(s.pollEvery):
}
}
+10 -1
View File
@@ -22,6 +22,8 @@ type fakeSemaphore struct {
tasks []task
launches int
finalStatus string
// failReads is how many status reads fail before one succeeds.
failReads int
// limitFixed is a template that ignores a task's limit.
limitFixed bool
lastLaunch launchBody
@@ -59,6 +61,11 @@ func (f *fakeSemaphore) ServeHTTP(w http.ResponseWriter, r *http.Request) {
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/output"):
_ = json.MarshalWrite(w, []outputLine{{Output: "removed 3 files"}})
case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, "/api/project/3/tasks/"):
if f.failReads > 0 {
f.failReads--
w.WriteHeader(http.StatusBadGateway)
return
}
id, _ := strconv.Atoi(strings.TrimPrefix(r.URL.Path, "/api/project/3/tasks/"))
if f.finalStatus != "" {
f.tasks[id-1].Status = f.finalStatus
@@ -104,6 +111,7 @@ func TestLaunchHappensOncePerKey(t *testing.T) {
tests := []struct {
name string
final string
failReads int
invokes int
wantLaunches int
wantErr bool
@@ -111,12 +119,13 @@ func TestLaunchHappensOncePerKey(t *testing.T) {
}{
{name: "one invoke launches once", final: "success", invokes: 1, wantLaunches: 1},
{name: "invoking again after a restart doesn't launch again", final: "success", invokes: 2, wantLaunches: 1},
{name: "a status read that fails is retried", final: "success", failReads: 3, invokes: 1, wantLaunches: 1},
{name: "a task that fails is a known failure", final: "error", invokes: 1, wantLaunches: 1, wantErr: true},
{name: "a task still running at the deadline is unknown", final: "", invokes: 2, wantLaunches: 1, wantErr: true, wantUnknown: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
fake := &fakeSemaphore{finalStatus: tt.final}
fake := &fakeSemaphore{finalStatus: tt.final, failReads: tt.failReads}
srv := httptest.NewServer(fake)
defer srv.Close()
var (
+6 -2
View File
@@ -89,14 +89,18 @@ func (s *server) Resolve(ctx context.Context, req *connectorv1.ResolveRequest) (
}
// Invoke runs the action and reports the digest of the input it ran, so the
// host can check it ran the judged action.
// host can check it ran the judged action. An action that may have taken
// effect is reported as an unknown outcome, with what went wrong as its
// output, never as an error: the host must not treat it as not having run.
func (s *server) Invoke(ctx context.Context, req *connectorv1.InvokeRequest) (*connectorv1.InvokeResponse, error) {
a, err := s.actor()
if err != nil {
return nil, err
}
out, unknown, err := a.Invoke(ctx, req.GetAction(), req.GetInput(), req.GetIdempotencyKey())
if err != nil {
if err != nil && unknown {
out = []byte(err.Error())
} else if err != nil {
return nil, status.Errorf(codes.Unknown, "invoke: %v", err)
}
return &connectorv1.InvokeResponse{
+53
View File
@@ -0,0 +1,53 @@
package connector
import (
"context"
"encoding/json/jsontext"
"errors"
"testing"
connectorv1 "github.com/samcharles93/archipelago/sdk/connector/v1"
)
type stubActor struct {
unknown bool
err error
}
func (stubActor) Describe() Description { return Description{} }
func (stubActor) Resolve(context.Context, string, jsontext.Value) (*connectorv1.ResolveResponse, error) {
return &connectorv1.ResolveResponse{}, nil
}
func (a stubActor) Invoke(context.Context, string, jsontext.Value, string) (jsontext.Value, bool, error) {
return nil, a.unknown, a.err
}
// TestAnActionThatMayHaveRunIsNotAFailure guards the unknown outcome: an
// action that fails after it may have taken effect must reach the host as
// outcome_unknown, not as an error, or the host treats it as not having run
// and may run it again.
func TestAnActionThatMayHaveRunIsNotAFailure(t *testing.T) {
tests := []struct {
name string
actor stubActor
wantErr bool
wantUnknown bool
}{
{name: "a failure before any effect is an error", actor: stubActor{err: errors.New("bad input")}, wantErr: true},
{name: "a failure after a possible effect is unknown", actor: stubActor{unknown: true, err: errors.New("lost track of task 9")}, wantUnknown: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := &server{build: func(Config) (Connector, error) { return tt.actor, nil }}
if _, err := s.Configure(t.Context(), &connectorv1.ConfigureRequest{}); err != nil {
t.Fatal(err)
}
resp, err := s.Invoke(t.Context(), &connectorv1.InvokeRequest{Action: "a", Input: []byte(`{}`)})
if (err != nil) != tt.wantErr || resp.GetOutcomeUnknown() != tt.wantUnknown {
t.Fatalf("err = %v, unknown = %v; want error %v, unknown %v", err, resp.GetOutcomeUnknown(), tt.wantErr, tt.wantUnknown)
}
})
}
}