feat(connectors): Semaphore connector, launching declared job templates once per idempotency key (archie #2001)
publish / publish (push) Successful in 1m6s
publish / publish (push) Successful in 1m6s
Each template the operator declares becomes an action with the effect declared for it, so judgement decides on the declaration. A launch carries Archie's idempotency key in the task message; invoking again finds that task instead of launching another, so a resumed run never launches twice. A task that ends in error is a known failure; one still running at the deadline is unknown.
This commit is contained in:
4 files changed
+463
No files matched your search
@@ -0,0 +1,14 @@
|
||||
// Command semaphore is the Semaphore connector: each job template an operator
|
||||
// declares becomes one action, with the effect the operator declares for it,
|
||||
// so judgement decides on the template's effect rather than on its name. A
|
||||
// launch carries Archie's idempotency key, so a resumed run finds the task it
|
||||
// already started instead of launching another.
|
||||
package main
|
||||
|
||||
import "github.com/samcharles93/archipelago/sdk/connector"
|
||||
|
||||
func main() {
|
||||
connector.Serve(func(cfg connector.Config) (connector.Connector, error) {
|
||||
return newSemaphore(cfg)
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
apiVersion: dev.archie.package.v1
|
||||
displayName: Semaphore
|
||||
description: Launches the Semaphore job templates an operator declares, each judged on the effect declared for it. Settings name the base URL, project and templates; the api credential slot holds a bearer token.
|
||||
version: 0.1.0
|
||||
contributes:
|
||||
extensions:
|
||||
- surface: connector
|
||||
path: bin/semaphore
|
||||
files:
|
||||
- path: bin/semaphore
|
||||
mode: 493
|
||||
@@ -0,0 +1,320 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json/jsontext"
|
||||
json "encoding/json/v2"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archipelago/sdk/connector"
|
||||
connectorv1 "github.com/samcharles93/archipelago/sdk/connector/v1"
|
||||
)
|
||||
|
||||
const (
|
||||
pollEvery = 2 * time.Second
|
||||
taskDeadline = 15 * time.Minute
|
||||
outputLines = 60
|
||||
recentTasks = 200
|
||||
messagePrefix = "archie "
|
||||
)
|
||||
|
||||
// Template is one job template the operator lets Archie launch, declared in
|
||||
// the templates setting.
|
||||
type Template struct {
|
||||
Action string `json:"action"`
|
||||
TemplateID int `json:"template_id"`
|
||||
Description string `json:"description"`
|
||||
Effect struct {
|
||||
ReadOnly bool `json:"read_only"`
|
||||
// Reversibility is "undone_automatically", "recoverable" or
|
||||
// "irreversible".
|
||||
Reversibility string `json:"reversibility"`
|
||||
Summary string `json:"summary"`
|
||||
} `json:"effect"`
|
||||
Target struct {
|
||||
Kind string `json:"kind"`
|
||||
Environment string `json:"environment"`
|
||||
Disposable bool `json:"disposable"`
|
||||
} `json:"target"`
|
||||
// Vars are the extra variables the template takes, each with a
|
||||
// description. Every one is required, so the judged action is complete.
|
||||
Vars map[string]string `json:"vars"`
|
||||
}
|
||||
|
||||
var validAction = regexp.MustCompile(`^[a-z][a-z0-9_]{0,62}$`)
|
||||
|
||||
type semaphore struct {
|
||||
base *url.URL
|
||||
project int
|
||||
token string
|
||||
templates map[string]Template
|
||||
client *http.Client
|
||||
pollEvery time.Duration
|
||||
deadline time.Duration
|
||||
}
|
||||
|
||||
func newSemaphore(cfg connector.Config) (*semaphore, error) {
|
||||
base, err := url.Parse(cfg.Settings["base_url"])
|
||||
if err != nil || (base.Scheme != "https" && base.Scheme != "http") || base.Host == "" {
|
||||
return nil, fmt.Errorf("base_url %q must be an http or https URL", cfg.Settings["base_url"])
|
||||
}
|
||||
project, err := strconv.Atoi(cfg.Settings["project_id"])
|
||||
if err != nil || project <= 0 {
|
||||
return nil, fmt.Errorf("project_id %q must be a positive number", cfg.Settings["project_id"])
|
||||
}
|
||||
token := cfg.Secrets["api.token"]
|
||||
if token == "" {
|
||||
return nil, errors.New("the api credential slot needs a bearer token")
|
||||
}
|
||||
var list []Template
|
||||
if err := json.Unmarshal([]byte(cfg.Settings["templates"]), &list); err != nil {
|
||||
return nil, fmt.Errorf("templates setting: %w", err)
|
||||
}
|
||||
s := &semaphore{
|
||||
base: base, project: project, token: token, templates: map[string]Template{}, client: &http.Client{Timeout: 30 * time.Second},
|
||||
pollEvery: pollEvery, deadline: taskDeadline,
|
||||
}
|
||||
for _, t := range list {
|
||||
if !validAction.MatchString(t.Action) || t.TemplateID <= 0 {
|
||||
return nil, fmt.Errorf("template %q needs a lowercase action name and a template_id", t.Action)
|
||||
}
|
||||
if _, err := reversibility(t.Effect.Reversibility, t.Effect.ReadOnly); err != nil {
|
||||
return nil, fmt.Errorf("template %q: %w", t.Action, err)
|
||||
}
|
||||
s.templates[t.Action] = t
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func reversibility(name string, readOnly bool) (connectorv1.Reversibility, error) {
|
||||
switch name {
|
||||
case "undone_automatically":
|
||||
return connectorv1.Reversibility_REVERSIBILITY_UNDONE_AUTOMATICALLY, nil
|
||||
case "recoverable":
|
||||
return connectorv1.Reversibility_REVERSIBILITY_RECOVERABLE, nil
|
||||
case "irreversible":
|
||||
return connectorv1.Reversibility_REVERSIBILITY_IRREVERSIBLE, nil
|
||||
case "":
|
||||
if readOnly {
|
||||
return connectorv1.Reversibility_REVERSIBILITY_UNSPECIFIED, nil
|
||||
}
|
||||
}
|
||||
return 0, fmt.Errorf("reversibility %q must be undone_automatically, recoverable or irreversible", name)
|
||||
}
|
||||
|
||||
func (s *semaphore) effect(t Template) *connectorv1.Effect {
|
||||
r, _ := reversibility(t.Effect.Reversibility, t.Effect.ReadOnly)
|
||||
return &connectorv1.Effect{ReadOnly: t.Effect.ReadOnly, Reversibility: r, Summary: t.Effect.Summary}
|
||||
}
|
||||
|
||||
func (s *semaphore) Describe() connector.Description {
|
||||
var d connector.Description
|
||||
for _, name := range slices.Sorted(maps.Keys(s.templates)) {
|
||||
t := s.templates[name]
|
||||
d.Actions = append(d.Actions, &connectorv1.ActionType{
|
||||
Name: t.Action, Description: t.Description, InputSchema: inputSchema(t),
|
||||
Effect: s.effect(t), Idempotent: true,
|
||||
})
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// inputSchema is the host to limit the run to, and every declared variable.
|
||||
func inputSchema(t Template) []byte {
|
||||
props := map[string]any{"host": map[string]any{"type": "string", "description": "The inventory host to run against."}}
|
||||
required := []string{"host"}
|
||||
for name, desc := range t.Vars {
|
||||
props[name] = map[string]any{"type": "string", "description": desc}
|
||||
required = append(required, name)
|
||||
}
|
||||
slices.Sort(required)
|
||||
b, _ := json.Marshal(map[string]any{"type": "object", "additionalProperties": false, "required": required, "properties": props})
|
||||
return b
|
||||
}
|
||||
|
||||
// actionInput is an action's host and the template's variables.
|
||||
type actionInput struct {
|
||||
Host string
|
||||
Vars map[string]string
|
||||
}
|
||||
|
||||
func (s *semaphore) parse(action string, raw jsontext.Value) (Template, actionInput, error) {
|
||||
t, ok := s.templates[action]
|
||||
if !ok {
|
||||
return Template{}, actionInput{}, fmt.Errorf("no action named %q", action)
|
||||
}
|
||||
var in map[string]string
|
||||
if err := json.Unmarshal(raw, &in); err != nil {
|
||||
return Template{}, actionInput{}, fmt.Errorf("input: %w", err)
|
||||
}
|
||||
host := in["host"]
|
||||
if host == "" {
|
||||
return Template{}, actionInput{}, errors.New("input needs a host")
|
||||
}
|
||||
delete(in, "host")
|
||||
return t, actionInput{Host: host, Vars: in}, nil
|
||||
}
|
||||
|
||||
func (s *semaphore) Resolve(_ context.Context, action string, raw jsontext.Value) (*connectorv1.ResolveResponse, error) {
|
||||
t, in, err := s.parse(action, raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &connectorv1.ResolveResponse{
|
||||
Target: &connectorv1.Target{Kind: t.Target.Kind, Id: in.Host, Environment: t.Target.Environment, Disposable: t.Target.Disposable},
|
||||
Effect: s.effect(t),
|
||||
}, nil
|
||||
}
|
||||
|
||||
type task struct {
|
||||
ID int `json:"id"`
|
||||
TemplateID int `json:"template_id"`
|
||||
Status string `json:"status"`
|
||||
Message string `json:"message,omitempty"`
|
||||
}
|
||||
|
||||
// Invoke launches the template, unless a task carrying this idempotency key
|
||||
// already exists, and waits for it to finish. A task still running at the
|
||||
// deadline is reported as an unknown outcome, never launched again.
|
||||
func (s *semaphore) Invoke(ctx context.Context, action string, raw jsontext.Value, key string) (jsontext.Value, bool, error) {
|
||||
t, in, err := s.parse(action, raw)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if key == "" {
|
||||
return nil, false, errors.New("an idempotency key is required")
|
||||
}
|
||||
message := messagePrefix + key
|
||||
tk, found, err := s.findTask(ctx, t.TemplateID, message)
|
||||
if err != nil {
|
||||
return nil, true, err
|
||||
}
|
||||
if !found {
|
||||
if tk, err = s.launch(ctx, t, in, message); err != nil {
|
||||
return nil, true, err
|
||||
}
|
||||
}
|
||||
done, err := s.wait(ctx, tk.ID)
|
||||
if err != nil {
|
||||
return nil, true, err
|
||||
}
|
||||
out, err := s.result(ctx, done)
|
||||
return out, false, err
|
||||
}
|
||||
|
||||
// findTask looks through the project's recent tasks for one this key
|
||||
// started.
|
||||
func (s *semaphore) findTask(ctx context.Context, templateID int, message string) (task, bool, error) {
|
||||
var tasks []task
|
||||
if err := s.do(ctx, http.MethodGet, fmt.Sprintf("/api/project/%d/tasks/last?count=%d", s.project, recentTasks), nil, &tasks); err != nil {
|
||||
return task{}, false, fmt.Errorf("list tasks: %w", err)
|
||||
}
|
||||
for _, tk := range tasks {
|
||||
if tk.TemplateID == templateID && tk.Message == message {
|
||||
return tk, true, nil
|
||||
}
|
||||
}
|
||||
return task{}, false, nil
|
||||
}
|
||||
|
||||
func (s *semaphore) launch(ctx context.Context, t Template, in actionInput, message string) (task, error) {
|
||||
env, err := json.Marshal(in.Vars, json.FormatNilMapAsNull(false))
|
||||
if err != nil {
|
||||
return task{}, err
|
||||
}
|
||||
body := map[string]any{"template_id": t.TemplateID, "environment": string(env), "limit": in.Host, "message": message}
|
||||
var tk task
|
||||
if err := s.do(ctx, http.MethodPost, fmt.Sprintf("/api/project/%d/tasks", s.project), body, &tk); err != nil {
|
||||
return task{}, fmt.Errorf("launch template %d: %w", t.TemplateID, err)
|
||||
}
|
||||
return tk, nil
|
||||
}
|
||||
|
||||
func (s *semaphore) wait(ctx context.Context, id int) (task, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, s.deadline)
|
||||
defer cancel()
|
||||
for {
|
||||
var tk task
|
||||
if err := s.do(ctx, http.MethodGet, fmt.Sprintf("/api/project/%d/tasks/%d", s.project, id), nil, &tk); err != nil {
|
||||
return task{}, fmt.Errorf("task %d: %w", id, err)
|
||||
}
|
||||
switch tk.Status {
|
||||
case "success", "error", "stopped", "rejected":
|
||||
return tk, nil
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return task{}, fmt.Errorf("task %d still %s: %w", id, tk.Status, ctx.Err())
|
||||
case <-time.After(s.pollEvery):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type outputLine struct {
|
||||
Output string `json:"output"`
|
||||
}
|
||||
|
||||
// result reports how the task ended with the tail of its log. A task that
|
||||
// ended in error is a known failure.
|
||||
func (s *semaphore) result(ctx context.Context, tk task) (jsontext.Value, error) {
|
||||
var lines []outputLine
|
||||
if err := s.do(ctx, http.MethodGet, fmt.Sprintf("/api/project/%d/tasks/%d/output", s.project, tk.ID), nil, &lines); err != nil {
|
||||
return nil, fmt.Errorf("task %d output: %w", tk.ID, err)
|
||||
}
|
||||
tail := make([]string, 0, outputLines)
|
||||
for _, l := range lines[max(0, len(lines)-outputLines):] {
|
||||
tail = append(tail, l.Output)
|
||||
}
|
||||
out, err := json.Marshal(map[string]any{"task_id": tk.ID, "status": tk.Status, "output": strings.Join(tail, "\n")})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if tk.Status != "success" {
|
||||
return nil, fmt.Errorf("task %d ended %s: %s", tk.ID, tk.Status, strings.Join(tail, "\n"))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// do calls one API path (which may carry a query) and decodes the reply.
|
||||
func (s *semaphore) do(ctx context.Context, method, path string, body, out any) error {
|
||||
ref, err := url.Parse(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
target := s.base.JoinPath(ref.Path)
|
||||
target.RawQuery = ref.RawQuery
|
||||
payload := []byte{}
|
||||
if body != nil {
|
||||
if payload, err = json.Marshal(body); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, method, target.String(), bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+s.token)
|
||||
req.Header.Set("Accept", "application/json")
|
||||
if body != nil {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
resp, err := s.client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if resp.StatusCode < 200 || resp.StatusCode > 299 {
|
||||
return fmt.Errorf("%s %s: %s", method, ref.Path, resp.Status)
|
||||
}
|
||||
return json.UnmarshalRead(resp.Body, out)
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json/jsontext"
|
||||
json "encoding/json/v2"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archipelago/sdk/connector"
|
||||
)
|
||||
|
||||
// fakeSemaphore keeps tasks in memory. A launched task ends with finalStatus
|
||||
// on its first status read, unless finalStatus is empty, when it runs for
|
||||
// ever.
|
||||
type fakeSemaphore struct {
|
||||
mu sync.Mutex
|
||||
tasks []task
|
||||
launches int
|
||||
finalStatus string
|
||||
}
|
||||
|
||||
func (f *fakeSemaphore) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if r.Header.Get("Authorization") != "Bearer plainly-fake-token" {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
switch {
|
||||
case r.Method == http.MethodGet && r.URL.Path == "/api/project/3/tasks/last":
|
||||
_ = json.MarshalWrite(w, f.tasks)
|
||||
case r.Method == http.MethodPost && r.URL.Path == "/api/project/3/tasks":
|
||||
var body struct {
|
||||
TemplateID int `json:"template_id"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
_ = json.UnmarshalRead(r.Body, &body)
|
||||
f.launches++
|
||||
t := task{ID: len(f.tasks) + 1, TemplateID: body.TemplateID, Status: "waiting", Message: body.Message}
|
||||
f.tasks = append(f.tasks, t)
|
||||
_ = json.MarshalWrite(w, t)
|
||||
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/output"):
|
||||
_ = json.MarshalWrite(w, []outputLine{{Output: "removed 3 files"}})
|
||||
case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, "/api/project/3/tasks/"):
|
||||
id, _ := strconv.Atoi(strings.TrimPrefix(r.URL.Path, "/api/project/3/tasks/"))
|
||||
if f.finalStatus != "" {
|
||||
f.tasks[id-1].Status = f.finalStatus
|
||||
}
|
||||
_ = json.MarshalWrite(w, f.tasks[id-1])
|
||||
default:
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}
|
||||
|
||||
func newTestConnector(t *testing.T, url string) *semaphore {
|
||||
t.Helper()
|
||||
s, err := newSemaphore(connector.Config{
|
||||
Settings: map[string]string{
|
||||
"base_url": url,
|
||||
"project_id": "3",
|
||||
"templates": `[{"action":"clean_tmp","template_id":7,"effect":{"reversibility":"irreversible","summary":"delete old files"},"target":{"kind":"host","environment":"non-production"},"vars":{"older_than_hours":"Age of files to delete"}}]`,
|
||||
},
|
||||
Secrets: map[string]string{"api.token": "plainly-fake-token"}, //nolint:gosec // a fake token the fake server checks
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s.pollEvery, s.deadline = time.Millisecond, 50*time.Millisecond
|
||||
return s
|
||||
}
|
||||
|
||||
// TestLaunchHappensOncePerKey guards #2001's idempotency: a launch carries
|
||||
// Archie's idempotency key, so invoking it again, from a connector that has
|
||||
// restarted, finds the task already started instead of launching another. A
|
||||
// task that ends in error is a known failure; one still running at the
|
||||
// deadline is an unknown outcome.
|
||||
func TestLaunchHappensOncePerKey(t *testing.T) {
|
||||
input := jsontext.Value(`{"host":"web-1","older_than_hours":"24"}`)
|
||||
tests := []struct {
|
||||
name string
|
||||
final string
|
||||
invokes int
|
||||
wantLaunches int
|
||||
wantErr bool
|
||||
wantUnknown bool
|
||||
}{
|
||||
{name: "one invoke launches once", final: "success", invokes: 1, wantLaunches: 1},
|
||||
{name: "invoking again after a restart doesn't launch again", final: "success", invokes: 2, wantLaunches: 1},
|
||||
{name: "a task that fails is a known failure", final: "error", invokes: 1, wantLaunches: 1, wantErr: true},
|
||||
{name: "a task still running at the deadline is unknown", final: "", invokes: 2, wantLaunches: 1, wantErr: true, wantUnknown: true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
fake := &fakeSemaphore{finalStatus: tt.final}
|
||||
srv := httptest.NewServer(fake)
|
||||
defer srv.Close()
|
||||
var (
|
||||
err error
|
||||
unknown bool
|
||||
)
|
||||
for range tt.invokes {
|
||||
// A fresh connector each time, as after a restart.
|
||||
_, unknown, err = newTestConnector(t, srv.URL).Invoke(t.Context(), "clean_tmp", input, "run-1:investigate:4")
|
||||
}
|
||||
if fake.launches != tt.wantLaunches {
|
||||
t.Fatalf("launched %d times, want %d", fake.launches, tt.wantLaunches)
|
||||
}
|
||||
if (err != nil) != tt.wantErr || unknown != tt.wantUnknown {
|
||||
t.Fatalf("err = %v, unknown = %v; want error %v, unknown %v", err, unknown, tt.wantErr, tt.wantUnknown)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user