fix(sdk,semaphore): an action that may have run is an unknown outcome, not an error; retry a failed task status read (archie #2001)
publish / publish (push) Successful in 1m12s
publish / publish (push) Successful in 1m12s
This commit is contained in:
4 files changed
+81
-9
No files matched your search
@@ -296,21 +296,27 @@ func (s *semaphore) launch(ctx context.Context, t Template, in actionInput, mess
|
||||
return tk, nil
|
||||
}
|
||||
|
||||
// wait polls the task until it ends. A failed status read is retried until
|
||||
// the deadline: Semaphore can stall reads while a task starts, and the task
|
||||
// runs on regardless.
|
||||
func (s *semaphore) wait(ctx context.Context, id int) (task, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, s.deadline)
|
||||
defer cancel()
|
||||
var last string
|
||||
for {
|
||||
var tk task
|
||||
if err := s.do(ctx, http.MethodGet, fmt.Sprintf("/api/project/%d/tasks/%d", s.project, id), nil, &tk); err != nil {
|
||||
return task{}, fmt.Errorf("task %d: %w", id, err)
|
||||
}
|
||||
switch tk.Status {
|
||||
case "success", "error", "stopped", "rejected":
|
||||
return tk, nil
|
||||
last = err.Error()
|
||||
} else {
|
||||
switch tk.Status {
|
||||
case "success", "error", "stopped", "rejected":
|
||||
return tk, nil
|
||||
}
|
||||
last = tk.Status
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return task{}, fmt.Errorf("task %d still %s: %w", id, tk.Status, ctx.Err())
|
||||
return task{}, fmt.Errorf("task %d launched, last seen %s: %w", id, last, ctx.Err())
|
||||
case <-time.After(s.pollEvery):
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,6 +22,8 @@ type fakeSemaphore struct {
|
||||
tasks []task
|
||||
launches int
|
||||
finalStatus string
|
||||
// failReads is how many status reads fail before one succeeds.
|
||||
failReads int
|
||||
// limitFixed is a template that ignores a task's limit.
|
||||
limitFixed bool
|
||||
lastLaunch launchBody
|
||||
@@ -59,6 +61,11 @@ func (f *fakeSemaphore) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/output"):
|
||||
_ = json.MarshalWrite(w, []outputLine{{Output: "removed 3 files"}})
|
||||
case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, "/api/project/3/tasks/"):
|
||||
if f.failReads > 0 {
|
||||
f.failReads--
|
||||
w.WriteHeader(http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
id, _ := strconv.Atoi(strings.TrimPrefix(r.URL.Path, "/api/project/3/tasks/"))
|
||||
if f.finalStatus != "" {
|
||||
f.tasks[id-1].Status = f.finalStatus
|
||||
@@ -104,6 +111,7 @@ func TestLaunchHappensOncePerKey(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
final string
|
||||
failReads int
|
||||
invokes int
|
||||
wantLaunches int
|
||||
wantErr bool
|
||||
@@ -111,12 +119,13 @@ func TestLaunchHappensOncePerKey(t *testing.T) {
|
||||
}{
|
||||
{name: "one invoke launches once", final: "success", invokes: 1, wantLaunches: 1},
|
||||
{name: "invoking again after a restart doesn't launch again", final: "success", invokes: 2, wantLaunches: 1},
|
||||
{name: "a status read that fails is retried", final: "success", failReads: 3, invokes: 1, wantLaunches: 1},
|
||||
{name: "a task that fails is a known failure", final: "error", invokes: 1, wantLaunches: 1, wantErr: true},
|
||||
{name: "a task still running at the deadline is unknown", final: "", invokes: 2, wantLaunches: 1, wantErr: true, wantUnknown: true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
fake := &fakeSemaphore{finalStatus: tt.final}
|
||||
fake := &fakeSemaphore{finalStatus: tt.final, failReads: tt.failReads}
|
||||
srv := httptest.NewServer(fake)
|
||||
defer srv.Close()
|
||||
var (
|
||||
|
||||
@@ -89,14 +89,18 @@ func (s *server) Resolve(ctx context.Context, req *connectorv1.ResolveRequest) (
|
||||
}
|
||||
|
||||
// Invoke runs the action and reports the digest of the input it ran, so the
|
||||
// host can check it ran the judged action.
|
||||
// host can check it ran the judged action. An action that may have taken
|
||||
// effect is reported as an unknown outcome, with what went wrong as its
|
||||
// output, never as an error: the host must not treat it as not having run.
|
||||
func (s *server) Invoke(ctx context.Context, req *connectorv1.InvokeRequest) (*connectorv1.InvokeResponse, error) {
|
||||
a, err := s.actor()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out, unknown, err := a.Invoke(ctx, req.GetAction(), req.GetInput(), req.GetIdempotencyKey())
|
||||
if err != nil {
|
||||
if err != nil && unknown {
|
||||
out = []byte(err.Error())
|
||||
} else if err != nil {
|
||||
return nil, status.Errorf(codes.Unknown, "invoke: %v", err)
|
||||
}
|
||||
return &connectorv1.InvokeResponse{
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
package connector
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json/jsontext"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
connectorv1 "github.com/samcharles93/archipelago/sdk/connector/v1"
|
||||
)
|
||||
|
||||
type stubActor struct {
|
||||
unknown bool
|
||||
err error
|
||||
}
|
||||
|
||||
func (stubActor) Describe() Description { return Description{} }
|
||||
|
||||
func (stubActor) Resolve(context.Context, string, jsontext.Value) (*connectorv1.ResolveResponse, error) {
|
||||
return &connectorv1.ResolveResponse{}, nil
|
||||
}
|
||||
|
||||
func (a stubActor) Invoke(context.Context, string, jsontext.Value, string) (jsontext.Value, bool, error) {
|
||||
return nil, a.unknown, a.err
|
||||
}
|
||||
|
||||
// TestAnActionThatMayHaveRunIsNotAFailure guards the unknown outcome: an
|
||||
// action that fails after it may have taken effect must reach the host as
|
||||
// outcome_unknown, not as an error, or the host treats it as not having run
|
||||
// and may run it again.
|
||||
func TestAnActionThatMayHaveRunIsNotAFailure(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
actor stubActor
|
||||
wantErr bool
|
||||
wantUnknown bool
|
||||
}{
|
||||
{name: "a failure before any effect is an error", actor: stubActor{err: errors.New("bad input")}, wantErr: true},
|
||||
{name: "a failure after a possible effect is unknown", actor: stubActor{unknown: true, err: errors.New("lost track of task 9")}, wantUnknown: true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := &server{build: func(Config) (Connector, error) { return tt.actor, nil }}
|
||||
if _, err := s.Configure(t.Context(), &connectorv1.ConfigureRequest{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resp, err := s.Invoke(t.Context(), &connectorv1.InvokeRequest{Action: "a", Input: []byte(`{}`)})
|
||||
if (err != nil) != tt.wantErr || resp.GetOutcomeUnknown() != tt.wantUnknown {
|
||||
t.Fatalf("err = %v, unknown = %v; want error %v, unknown %v", err, resp.GetOutcomeUnknown(), tt.wantErr, tt.wantUnknown)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user