diff --git a/connectors/servicenow/README.md b/connectors/servicenow/README.md index ee6caf2..46670e8 100644 --- a/connectors/servicenow/README.md +++ b/connectors/servicenow/README.md @@ -12,7 +12,7 @@ Each area of ServiceNow is its own file, so the file list is the map: | `incident.go` | Incident delivery and `incident.updated` event | `incident` | | `change.go` | Change request deliveries and `change.approved` / `change.rejected` events | `change_request` | | `cmdb.go` | Owner, environment and data-class lookups | `cmdb_ci_server`, `cmdb_ci_appl`, `cmdb_ci` | -| `calendar.go` | Blackouts lookup for the Timing guard | `cmn_blackout` | +| `calendar.go` | Blackouts lookup for the Timing guard | `cmn_schedule_span` (spans of `schedule.type=blackout`) | | `listen.go` | Event polling, cursor over both tables | `change_request`, `incident` | | `servicenow.go` | Wiring: config, `Describe`, delivery and lookup dispatch | | @@ -45,9 +45,12 @@ delivery honours it. ## Events -`listen.go` polls `change_request` and `incident` by `sys_id`, with the cursor -`chg:;inc:` stored per emitted batch, so a restart resumes after the -last stored batch. +`listen.go` polls `change_request` and `incident` by `sys_updated_on`, bounded +above by the current time so future-dated sample rows cannot push the cursor +past the present. The cursor is `chg:|;inc:...`, stored +per emitted batch, so a restart resumes after the last stored batch and an +in-place update (an approver's decision moves `sys_updated_on`, not `sys_id`) +is still caught. - `change.approved`, `change.rejected` — the change's `approval` field. Fields: `number`, `approval`, `state`, `short_description`. `LinkKey` is the change @@ -57,17 +60,20 @@ last stored batch. ## Lookups All CMDB lookups take `{"target": ""}` and resolve -through `cmdb_ci_server`, then `cmdb_ci_appl`, then `cmdb_ci`, with reference -fields read as display names. `found=false` means no CI matched, which core -treats as an unknown-owner finding. +through `cmdb_ci_server`, then `cmdb_ci_appl`, then `cmdb_ci`. The whole CI row +is read with display values and the fields are picked locally, because which of +`owner`, `delegate`, `environment`, `data_class` exist depends on the +instance's CMDB schema. `found=false` means no CI matched, which core treats as +an unknown-owner finding. - `cmdb_owner` — `owner`, `delegate`, `support_group`, plus `source` (which CI answered). - `cmdb_environment` — `value` from the CI's `environment` (or `u_environment`). - `cmdb_data_class` — `value` from the CI's `data_class` (or `u_data_class`). - `blackouts` — takes `{"at": ""}`, empty for now; returns the active - `cmn_blackout` windows as `{name, type, start, end}`. The Timing guard reads - this as a freeze. + blackout spans as `{name, schedule, start, end}`, read from + `cmn_schedule_span` where `schedule.type=blackout` and the span covers the + time. The Timing guard reads this as a freeze. ## Done when (archie #1983) diff --git a/connectors/servicenow/calendar.go b/connectors/servicenow/calendar.go index 76a7c3b..1696758 100644 --- a/connectors/servicenow/calendar.go +++ b/connectors/servicenow/calendar.go @@ -15,7 +15,9 @@ import ( // guard reads to hold changing actions during a freeze. const ( - blackoutTable = "cmn_blackout" + // spanTable holds the schedule spans; a span of a blackout schedule is one + // blackout window. Dot-walking schedule.type keeps this one query. + spanTable = "cmn_schedule_span" blackoutsLookup = "blackouts" ) @@ -35,10 +37,10 @@ type blackoutsInput struct { } type blackout struct { - Name string `json:"name"` - Type string `json:"type"` - Start string `json:"start"` - End string `json:"end"` + Name string `json:"name"` + Schedule string `json:"schedule"` + Start string `json:"start"` + End string `json:"end"` } // lookupBlackouts lists the blackouts in force at the given time, or now when @@ -59,20 +61,21 @@ func (s *servicenow) lookupBlackouts(ctx context.Context, raw jsontext.Value) (j } stamp := at.Format("2006-01-02 15:04:05") q := url.Values{ - "sysparm_query": {"start<=" + stamp + "^end>=" + stamp}, - "sysparm_fields": {"name,type,start,end"}, + "sysparm_query": {"schedule.type=blackout^start<=" + stamp + "^end>=" + stamp}, + "sysparm_fields": {"name,start,end,schedule"}, + "sysparm_display_value": {"true"}, } - rows, err := s.client.rows(ctx, blackoutTable, q) + rows, err := s.client.rows(ctx, spanTable, q) if err != nil { return nil, false, err } out := []blackout{} for _, row := range rows { out = append(out, blackout{ - Name: str(row["name"]), - Type: str(row["type"]), - Start: str(row["start"]), - End: str(row["end"]), + Name: str(row["name"]), + Schedule: display(row["schedule"]), + Start: str(row["start"]), + End: str(row["end"]), }) } res, err := marshalFields(map[string]any{"blackouts": out}) diff --git a/connectors/servicenow/calendar_test.go b/connectors/servicenow/calendar_test.go index c8c24ca..e21da7c 100644 --- a/connectors/servicenow/calendar_test.go +++ b/connectors/servicenow/calendar_test.go @@ -8,9 +8,9 @@ import ( func TestLookupBlackoutsReturnsActiveWindow(t *testing.T) { s, fake := newTestConnector(t) - fake.tables[blackoutTable] = []map[string]any{ - {"name": "release freeze", "type": "blackout", "start": "2026-11-16 08:00:00", "end": "2026-11-16 18:00:00"}, - {"name": "old freeze", "type": "blackout", "start": "2026-11-15 08:00:00", "end": "2026-11-15 18:00:00"}, + fake.tables[spanTable] = []map[string]any{ + {"name": "release freeze", "schedule.type": "blackout", "schedule": "sched-1", "start": "2026-11-16 08:00:00", "end": "2026-11-16 18:00:00"}, + {"name": "old freeze", "schedule.type": "blackout", "schedule": "sched-2", "start": "2026-11-15 08:00:00", "end": "2026-11-15 18:00:00"}, } out, found, err := s.Lookup(context.Background(), blackoutsLookup, raw(map[string]any{"at": "2026-11-16T12:00:00Z"})) if err != nil || !found { @@ -22,7 +22,7 @@ func TestLookupBlackoutsReturnsActiveWindow(t *testing.T) { if err := json.Unmarshal(out, &got); err != nil { t.Fatalf("unmarshal: %v", err) } - if len(got.Blackouts) != 1 || got.Blackouts[0].Name != "release freeze" { + if len(got.Blackouts) != 1 || got.Blackouts[0].Name != "release freeze" || got.Blackouts[0].Schedule != "sched-1" { t.Fatalf("blackouts = %+v, want only the active one", got.Blackouts) } } diff --git a/connectors/servicenow/change.go b/connectors/servicenow/change.go index a37dc43..413f744 100644 --- a/connectors/servicenow/change.go +++ b/connectors/servicenow/change.go @@ -6,6 +6,8 @@ import ( json "encoding/json/v2" "fmt" "net/url" + "strings" + "time" "github.com/samcharles93/archipelago/sdk/connector" connectorv1 "github.com/samcharles93/archipelago/sdk/connector/v1" @@ -145,12 +147,19 @@ func (s *servicenow) deliverChangeWithdraw(ctx context.Context, raw jsontext.Val return str(updated["number"]), nil } -// pollChanges reads change requests newer than cursor and turns an approval -// or rejection into an event. It returns the events and the new cursor. +// pollChanges reads change requests updated at or after the cursor and turns an +// approval or rejection into an event. The cursor is "|", +// because an approver's decision updates the row without changing its sys_id. +// Rows at the cursor's second are filtered client-side, so nothing is missed. func (s *servicenow) pollChanges(ctx context.Context, cursor string) ([]connector.Event, string, error) { + ts, id := splitCursor(cursor) + // The upper bound keeps future-dated sample rows from pushing the cursor + // past the present, which would hide later in-place updates. + now := time.Now().UTC().Format("2006-01-02 15:04:05") q := url.Values{ - "sysparm_query": {"sys_id>" + cursor + "^ORDERBYsys_id"}, - "sysparm_fields": {"sys_id,number,approval,state,short_description,sys_updated_on"}, + "sysparm_query": {"sys_updated_on>=" + ts + "^sys_updated_on<=" + now + "^ORDERBYsys_updated_on^ORDERBYsys_id"}, + "sysparm_fields": {"sys_id,number,approval,state,short_description,sys_updated_on"}, + "sysparm_display_value": {"true"}, } rows, err := s.client.rows(ctx, changeTable, q) if err != nil { @@ -158,34 +167,35 @@ func (s *servicenow) pollChanges(ctx context.Context, cursor string) ([]connecto } var events []connector.Event for _, row := range rows { - approval := str(row["approval"]) - var eventType string - switch approval { - case "Approved": - eventType = changeApproved - case "Rejected": - eventType = changeRejected - default: - cursor = str(row["sys_id"]) + rts, rid := str(row["sys_updated_on"]), str(row["sys_id"]) + if !afterCursor(rts, rid, ts, id) { continue } - fields, err := json.Marshal(map[string]any{ - "number": str(row["number"]), - "approval": approval, - "state": str(row["state"]), - "short_description": str(row["short_description"]), - }) - if err != nil { - return nil, cursor, err + approval := display(row["approval"]) + if strings.EqualFold(approval, "approved") || strings.EqualFold(approval, "rejected") { + fields, err := json.Marshal(map[string]any{ + "number": str(row["number"]), + "approval": approval, + "state": str(row["state"]), + "short_description": str(row["short_description"]), + }) + if err != nil { + return nil, cursor, err + } + eventType := changeApproved + if strings.EqualFold(approval, "rejected") { + eventType = changeRejected + } + events = append(events, connector.Event{ + Type: eventType, + SourceEventID: rid, + Fields: fields, + LinkKey: str(row["number"]), + OccurredAt: parseTime(rts), + }) } - events = append(events, connector.Event{ - Type: eventType, - SourceEventID: str(row["sys_id"]), - Fields: fields, - LinkKey: str(row["number"]), - OccurredAt: parseTime(str(row["sys_updated_on"])), - }) - cursor = str(row["sys_id"]) + ts, id = rts, rid + cursor = formatTableCursor(ts, id) } return events, cursor, nil } diff --git a/connectors/servicenow/change_test.go b/connectors/servicenow/change_test.go index 2052598..dd1008c 100644 --- a/connectors/servicenow/change_test.go +++ b/connectors/servicenow/change_test.go @@ -70,7 +70,7 @@ func TestPollChangesEmitsApprovalEvents(t *testing.T) { {"sys_id": "0000000000000011", "number": "CHG0000011", "approval": "Approved", "state": "-3", "sys_updated_on": "2026-11-16 10:01:00"}, {"sys_id": "0000000000000012", "number": "CHG0000012", "approval": "Rejected", "state": "7", "sys_updated_on": "2026-11-16 10:02:00"}, } - events, cursor, err := s.pollChanges(context.Background(), "0") + events, cursor, err := s.pollChanges(context.Background(), "") if err != nil { t.Fatalf("poll: %v", err) } @@ -83,7 +83,34 @@ func TestPollChangesEmitsApprovalEvents(t *testing.T) { if events[1].Type != changeRejected || events[1].LinkKey != "CHG0000012" { t.Fatalf("events[1] = %+v", events[1]) } - if cursor != "0000000000000012" { + if cursor != "2026-11-16 10:02:00|0000000000000012" { + t.Fatalf("cursor = %q", cursor) + } +} + +// An approver's decision updates the change in place, so its sys_id does not +// move. The cursor is sys_updated_on, and a re-poll must still emit the event. +func TestPollChangesCatchesInPlaceApproval(t *testing.T) { + s, fake := newTestConnector(t) + fake.tables[changeTable] = []map[string]any{ + {"sys_id": "0000000000000011", "number": "CHG0000011", "approval": "Not Yet Requested", "state": "-5", "sys_updated_on": "2026-11-16 10:00:00"}, + } + ctx := context.Background() + if _, cursor, err := s.pollChanges(ctx, ""); err != nil { + t.Fatalf("first poll: %v", err) + } else if cursor != "2026-11-16 10:00:00|0000000000000011" { + t.Fatalf("cursor after first poll = %q", cursor) + } + fake.tables[changeTable][0]["approval"] = "Approved" + fake.tables[changeTable][0]["sys_updated_on"] = "2026-11-16 10:05:00" + events, cursor, err := s.pollChanges(ctx, "2026-11-16 10:00:00|0000000000000011") + if err != nil { + t.Fatalf("second poll: %v", err) + } + if len(events) != 1 || events[0].Type != changeApproved || events[0].LinkKey != "CHG0000011" { + t.Fatalf("events = %+v, want one change.approved for CHG0000011", events) + } + if cursor != "2026-11-16 10:05:00|0000000000000011" { t.Fatalf("cursor = %q", cursor) } } diff --git a/connectors/servicenow/client.go b/connectors/servicenow/client.go index 8c98d2a..f741f76 100644 --- a/connectors/servicenow/client.go +++ b/connectors/servicenow/client.go @@ -61,22 +61,27 @@ func (c *client) one(ctx context.Context, table string, query url.Values) (map[s return rows[0], nil } +// singleResult is the envelope the Table API wraps one written row in. +type singleResult struct { + Result map[string]any `json:"result"` +} + // create inserts one row and returns it. func (c *client) create(ctx context.Context, table string, body map[string]any) (map[string]any, error) { - var out map[string]any + var out singleResult if err := c.do(ctx, http.MethodPost, c.base+"/api/now/table/"+table, body, &out); err != nil { return nil, err } - return out, nil + return out.Result, nil } // update changes one row, addressed by sys_id, and returns it. func (c *client) update(ctx context.Context, table, sysID string, body map[string]any) (map[string]any, error) { - var out map[string]any + var out singleResult if err := c.do(ctx, http.MethodPatch, c.base+"/api/now/table/"+table+"/"+url.PathEscape(sysID), body, &out); err != nil { return nil, err } - return out, nil + return out.Result, nil } func (c *client) do(ctx context.Context, method, u string, body map[string]any, out any) error { diff --git a/connectors/servicenow/cmdb.go b/connectors/servicenow/cmdb.go index 1468b2e..3eb0ba6 100644 --- a/connectors/servicenow/cmdb.go +++ b/connectors/servicenow/cmdb.go @@ -53,12 +53,13 @@ type cmdbInput struct { } // findCI returns the first CI matching the target, and which table it came -// from. Reference fields are requested as display values. -func (s *servicenow) findCI(ctx context.Context, target string, fields ...string) (map[string]any, string, error) { +// from. The whole row is read with display values, because which of the +// candidate fields exist depends on the instance's CMDB schema; the caller +// picks the fields it wants from what is there. +func (s *servicenow) findCI(ctx context.Context, target string) (map[string]any, string, error) { for _, table := range ciTables { q := url.Values{ "sysparm_query": {"name=" + target + "^ORhost_name=" + target}, - "sysparm_fields": {"sys_id,name,host_name," + joinComma(fields)}, "sysparm_display_value": {"true"}, "sysparm_limit": {"1"}, } @@ -84,7 +85,7 @@ func (s *servicenow) lookupOwner(ctx context.Context, raw jsontext.Value) (jsont if in.Target == "" { return nil, false, fmt.Errorf("cmdb_owner: target is required") } - row, table, err := s.findCI(ctx, in.Target, "owner", "delegate", "support_group") + row, table, err := s.findCI(ctx, in.Target) if err != nil { return nil, false, err } @@ -92,9 +93,9 @@ func (s *servicenow) lookupOwner(ctx context.Context, raw jsontext.Value) (jsont return nil, false, nil } out, err := marshalFields(map[string]any{ - "owner": str(row["owner"]), - "delegate": str(row["delegate"]), - "support_group": str(row["support_group"]), + "owner": display(row["owner"]), + "delegate": display(row["delegate"]), + "support_group": display(row["support_group"]), "source": sourceName(table), }) if err != nil { @@ -113,7 +114,7 @@ func (s *servicenow) lookupCIField(ctx context.Context, raw jsontext.Value, fiel if in.Target == "" { return nil, false, fmt.Errorf("target is required") } - row, table, err := s.findCI(ctx, in.Target, fields...) + row, table, err := s.findCI(ctx, in.Target) if err != nil { return nil, false, err } @@ -121,7 +122,7 @@ func (s *servicenow) lookupCIField(ctx context.Context, raw jsontext.Value, fiel return nil, false, nil } for _, f := range fields { - if v := str(row[f]); v != "" { + if v := display(row[f]); v != "" { out, err := marshalFields(map[string]any{"value": v, "source": sourceName(table)}) if err != nil { return nil, false, err @@ -143,14 +144,3 @@ func sourceName(table string) string { return "ci" } } - -func joinComma(ss []string) string { - out := "" - for i, s := range ss { - if i > 0 { - out += "," - } - out += s - } - return out -} diff --git a/connectors/servicenow/incident.go b/connectors/servicenow/incident.go index 0da2ff5..3a412ca 100644 --- a/connectors/servicenow/incident.go +++ b/connectors/servicenow/incident.go @@ -123,11 +123,13 @@ func (s *servicenow) deliverIncident(ctx context.Context, raw jsontext.Value, ke return str(created["number"]), nil } -// pollIncidents reads incidents newer than cursor and turns each into an -// incident.updated event. It returns the events and the new cursor. +// pollIncidents reads incidents updated at or after the cursor and turns each +// into an incident.updated event. The cursor works like pollChanges'. func (s *servicenow) pollIncidents(ctx context.Context, cursor string) ([]connector.Event, string, error) { + ts, id := splitCursor(cursor) + now := time.Now().UTC().Format("2006-01-02 15:04:05") q := url.Values{ - "sysparm_query": {"sys_id>" + cursor + "^ORDERBYsys_id"}, + "sysparm_query": {"sys_updated_on>=" + ts + "^sys_updated_on<=" + now + "^ORDERBYsys_updated_on^ORDERBYsys_id"}, "sysparm_fields": {"sys_id,number,state,short_description,sys_updated_on"}, } rows, err := s.client.rows(ctx, incidentTable, q) @@ -136,6 +138,10 @@ func (s *servicenow) pollIncidents(ctx context.Context, cursor string) ([]connec } var events []connector.Event for _, row := range rows { + rts, rid := str(row["sys_updated_on"]), str(row["sys_id"]) + if !afterCursor(rts, rid, ts, id) { + continue + } fields, err := json.Marshal(map[string]any{ "number": str(row["number"]), "state": str(row["state"]), @@ -146,12 +152,13 @@ func (s *servicenow) pollIncidents(ctx context.Context, cursor string) ([]connec } events = append(events, connector.Event{ Type: incidentUpdated, - SourceEventID: str(row["sys_id"]), + SourceEventID: rid, Fields: fields, LinkKey: str(row["number"]), - OccurredAt: parseTime(str(row["sys_updated_on"])), + OccurredAt: parseTime(rts), }) - cursor = str(row["sys_id"]) + ts, id = rts, rid + cursor = formatTableCursor(ts, id) } return events, cursor, nil } @@ -168,6 +175,15 @@ func str(v any) string { return s } +// display reads a field read with sysparm_display_value=true: a reference comes +// back as {"display_value": ..., "link": ...}, a plain field as a string. +func display(v any) string { + if m, ok := v.(map[string]any); ok { + return str(m["display_value"]) + } + return str(v) +} + func parseTime(s string) time.Time { t, err := time.Parse("2006-01-02 15:04:05", s) if err != nil { diff --git a/connectors/servicenow/incident_test.go b/connectors/servicenow/incident_test.go index c325b91..53be719 100644 --- a/connectors/servicenow/incident_test.go +++ b/connectors/servicenow/incident_test.go @@ -65,14 +65,14 @@ func TestPollIncidentsEmitsUpdates(t *testing.T) { fake.tables[incidentTable] = []map[string]any{ {"sys_id": "0000000000000001", "number": "INC0000001", "state": "2", "short_description": "demo", "sys_updated_on": "2026-11-16 10:00:00"}, } - events, cursor, err := s.pollIncidents(context.Background(), "0") + events, cursor, err := s.pollIncidents(context.Background(), "") if err != nil { t.Fatalf("poll: %v", err) } if len(events) != 1 || events[0].Type != incidentUpdated || events[0].LinkKey != "INC0000001" { t.Fatalf("events = %+v", events) } - if cursor != "0000000000000001" { + if cursor != "2026-11-16 10:00:00|0000000000000001" { t.Fatalf("cursor = %q", cursor) } } diff --git a/connectors/servicenow/listen.go b/connectors/servicenow/listen.go index fa69641..fb81128 100644 --- a/connectors/servicenow/listen.go +++ b/connectors/servicenow/listen.go @@ -9,8 +9,9 @@ import ( ) // listen.go is the event source: a poll over the change and incident tables. -// The cursor holds both tables' last seen sys_id, as "chg:;inc:", so a -// resumed listener picks up where the last batch ended. +// The cursor holds both tables' last seen "sys_updated_on|sys_id", as +// "chg:|;inc:|", so a resumed listener picks up where the last +// batch ended and an in-place update (an approval) is not missed. // Listen polls until ctx ends. Events are emitted in batches, and the cursor // only advances once a batch is stored, so a failed store replays the batch. @@ -57,3 +58,22 @@ func parseCursor(cursor string) (chg, inc string) { func formatCursor(chg, inc string) string { return "chg:" + chg + ";inc:" + inc } + +// splitCursor splits one table's "|" cursor. +func splitCursor(cursor string) (ts, id string) { + ts, id, _ = strings.Cut(cursor, "|") + return ts, id +} + +func formatTableCursor(ts, id string) string { + return ts + "|" + id +} + +// afterCursor reports whether a row is newer than the cursor, comparing +// sys_updated_on first and sys_id to break same-second ties. +func afterCursor(rts, rid, ts, id string) bool { + if rts != ts { + return rts > ts + } + return rid > id +} diff --git a/connectors/servicenow/listen_test.go b/connectors/servicenow/listen_test.go index 6a95d15..026f19f 100644 --- a/connectors/servicenow/listen_test.go +++ b/connectors/servicenow/listen_test.go @@ -40,7 +40,7 @@ func TestListenEmitsBatchedEventsAndAdvancesCursor(t *testing.T) { if len(got) != 2 { t.Fatalf("events = %d, want 2", len(got)) } - if cursor != "chg:0000000000000011;inc:0000000000000021" { + if cursor != "chg:2026-11-16 10:01:00|0000000000000011;inc:2026-11-16 10:02:00|0000000000000021" { t.Fatalf("cursor = %q", cursor) } } diff --git a/connectors/servicenow/servicenow_test.go b/connectors/servicenow/servicenow_test.go index 86251d3..8453053 100644 --- a/connectors/servicenow/servicenow_test.go +++ b/connectors/servicenow/servicenow_test.go @@ -64,7 +64,7 @@ func (f *fakeServiceNow) ServeHTTP(w http.ResponseWriter, r *http.Request) { } f.tables[table] = append(rows, body) w.WriteHeader(http.StatusCreated) - _ = json.MarshalWrite(w, body) + _ = json.MarshalWrite(w, map[string]any{"result": body}) case http.MethodPatch: sysID := parts[1] for i, row := range rows { @@ -75,7 +75,7 @@ func (f *fakeServiceNow) ServeHTTP(w http.ResponseWriter, r *http.Request) { row[k] = v } f.tables[table][i] = row - _ = json.MarshalWrite(w, row) + _ = json.MarshalWrite(w, map[string]any{"result": row}) return } }