From 53ba618b0bb56bf15bd2ee298f6bba005f7c76da Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Mon, 16 Nov 2026 03:57:20 +1100 Subject: [PATCH] fix(sdk,semaphore): an action that may have run is an unknown outcome, not an error; retry a failed task status read (archie #2001) --- connectors/semaphore/semaphore.go | 18 ++++++--- connectors/semaphore/semaphore_test.go | 11 +++++- sdk/connector/server.go | 8 +++- sdk/connector/server_test.go | 53 ++++++++++++++++++++++++++ 4 files changed, 81 insertions(+), 9 deletions(-) create mode 100644 sdk/connector/server_test.go diff --git a/connectors/semaphore/semaphore.go b/connectors/semaphore/semaphore.go index ec6c1ef..abfe240 100644 --- a/connectors/semaphore/semaphore.go +++ b/connectors/semaphore/semaphore.go @@ -296,21 +296,27 @@ func (s *semaphore) launch(ctx context.Context, t Template, in actionInput, mess return tk, nil } +// wait polls the task until it ends. A failed status read is retried until +// the deadline: Semaphore can stall reads while a task starts, and the task +// runs on regardless. func (s *semaphore) wait(ctx context.Context, id int) (task, error) { ctx, cancel := context.WithTimeout(ctx, s.deadline) defer cancel() + var last string for { var tk task if err := s.do(ctx, http.MethodGet, fmt.Sprintf("/api/project/%d/tasks/%d", s.project, id), nil, &tk); err != nil { - return task{}, fmt.Errorf("task %d: %w", id, err) - } - switch tk.Status { - case "success", "error", "stopped", "rejected": - return tk, nil + last = err.Error() + } else { + switch tk.Status { + case "success", "error", "stopped", "rejected": + return tk, nil + } + last = tk.Status } select { case <-ctx.Done(): - return task{}, fmt.Errorf("task %d still %s: %w", id, tk.Status, ctx.Err()) + return task{}, fmt.Errorf("task %d launched, last seen %s: %w", id, last, ctx.Err()) case <-time.After(s.pollEvery): } } diff --git a/connectors/semaphore/semaphore_test.go b/connectors/semaphore/semaphore_test.go index 65dc806..bf5a76a 100644 --- a/connectors/semaphore/semaphore_test.go +++ b/connectors/semaphore/semaphore_test.go @@ -22,6 +22,8 @@ type fakeSemaphore struct { tasks []task launches int finalStatus string + // failReads is how many status reads fail before one succeeds. + failReads int // limitFixed is a template that ignores a task's limit. limitFixed bool lastLaunch launchBody @@ -59,6 +61,11 @@ func (f *fakeSemaphore) ServeHTTP(w http.ResponseWriter, r *http.Request) { case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/output"): _ = json.MarshalWrite(w, []outputLine{{Output: "removed 3 files"}}) case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, "/api/project/3/tasks/"): + if f.failReads > 0 { + f.failReads-- + w.WriteHeader(http.StatusBadGateway) + return + } id, _ := strconv.Atoi(strings.TrimPrefix(r.URL.Path, "/api/project/3/tasks/")) if f.finalStatus != "" { f.tasks[id-1].Status = f.finalStatus @@ -104,6 +111,7 @@ func TestLaunchHappensOncePerKey(t *testing.T) { tests := []struct { name string final string + failReads int invokes int wantLaunches int wantErr bool @@ -111,12 +119,13 @@ func TestLaunchHappensOncePerKey(t *testing.T) { }{ {name: "one invoke launches once", final: "success", invokes: 1, wantLaunches: 1}, {name: "invoking again after a restart doesn't launch again", final: "success", invokes: 2, wantLaunches: 1}, + {name: "a status read that fails is retried", final: "success", failReads: 3, invokes: 1, wantLaunches: 1}, {name: "a task that fails is a known failure", final: "error", invokes: 1, wantLaunches: 1, wantErr: true}, {name: "a task still running at the deadline is unknown", final: "", invokes: 2, wantLaunches: 1, wantErr: true, wantUnknown: true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - fake := &fakeSemaphore{finalStatus: tt.final} + fake := &fakeSemaphore{finalStatus: tt.final, failReads: tt.failReads} srv := httptest.NewServer(fake) defer srv.Close() var ( diff --git a/sdk/connector/server.go b/sdk/connector/server.go index efb7e0b..d9c6397 100644 --- a/sdk/connector/server.go +++ b/sdk/connector/server.go @@ -89,14 +89,18 @@ func (s *server) Resolve(ctx context.Context, req *connectorv1.ResolveRequest) ( } // Invoke runs the action and reports the digest of the input it ran, so the -// host can check it ran the judged action. +// host can check it ran the judged action. An action that may have taken +// effect is reported as an unknown outcome, with what went wrong as its +// output, never as an error: the host must not treat it as not having run. func (s *server) Invoke(ctx context.Context, req *connectorv1.InvokeRequest) (*connectorv1.InvokeResponse, error) { a, err := s.actor() if err != nil { return nil, err } out, unknown, err := a.Invoke(ctx, req.GetAction(), req.GetInput(), req.GetIdempotencyKey()) - if err != nil { + if err != nil && unknown { + out = []byte(err.Error()) + } else if err != nil { return nil, status.Errorf(codes.Unknown, "invoke: %v", err) } return &connectorv1.InvokeResponse{ diff --git a/sdk/connector/server_test.go b/sdk/connector/server_test.go new file mode 100644 index 0000000..4bf9991 --- /dev/null +++ b/sdk/connector/server_test.go @@ -0,0 +1,53 @@ +package connector + +import ( + "context" + "encoding/json/jsontext" + "errors" + "testing" + + connectorv1 "github.com/samcharles93/archipelago/sdk/connector/v1" +) + +type stubActor struct { + unknown bool + err error +} + +func (stubActor) Describe() Description { return Description{} } + +func (stubActor) Resolve(context.Context, string, jsontext.Value) (*connectorv1.ResolveResponse, error) { + return &connectorv1.ResolveResponse{}, nil +} + +func (a stubActor) Invoke(context.Context, string, jsontext.Value, string) (jsontext.Value, bool, error) { + return nil, a.unknown, a.err +} + +// TestAnActionThatMayHaveRunIsNotAFailure guards the unknown outcome: an +// action that fails after it may have taken effect must reach the host as +// outcome_unknown, not as an error, or the host treats it as not having run +// and may run it again. +func TestAnActionThatMayHaveRunIsNotAFailure(t *testing.T) { + tests := []struct { + name string + actor stubActor + wantErr bool + wantUnknown bool + }{ + {name: "a failure before any effect is an error", actor: stubActor{err: errors.New("bad input")}, wantErr: true}, + {name: "a failure after a possible effect is unknown", actor: stubActor{unknown: true, err: errors.New("lost track of task 9")}, wantUnknown: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := &server{build: func(Config) (Connector, error) { return tt.actor, nil }} + if _, err := s.Configure(t.Context(), &connectorv1.ConfigureRequest{}); err != nil { + t.Fatal(err) + } + resp, err := s.Invoke(t.Context(), &connectorv1.InvokeRequest{Action: "a", Input: []byte(`{}`)}) + if (err != nil) != tt.wantErr || resp.GetOutcomeUnknown() != tt.wantUnknown { + t.Fatalf("err = %v, unknown = %v; want error %v, unknown %v", err, resp.GetOutcomeUnknown(), tt.wantErr, tt.wantUnknown) + } + }) + } +}