fix(connectors/servicenow): match the real Table API, verified against a PDI

Live testing against dev483891 surfaced four gaps the fake hid: write
responses wrap the row in result, reference fields come back as
display_value objects without display_value=true, the blackout table is
cmn_schedule_span (spans of schedule.type=blackout), not cmn_blackout,
and an approval updates sys_updated_on without moving sys_id, so the
poll cursor is now sys_updated_on|sys_id bounded above by the current
time. Approval values are compared case-insensitively. Adds a regression
test for the in-place approval. All lookups, deliveries and the
approved/rejected event path verified live; test rows cleaned up.
This commit is contained in:
sam committed 2026-11-16 21:15:25 +11:00
1 parent 110b28e62a
commit 541b7a46e0
12 files changed
+170 -93

No files matched your search

+15 -9
View File
@@ -12,7 +12,7 @@ Each area of ServiceNow is its own file, so the file list is the map:
| `incident.go` | Incident delivery and `incident.updated` event | `incident` |
| `change.go` | Change request deliveries and `change.approved` / `change.rejected` events | `change_request` |
| `cmdb.go` | Owner, environment and data-class lookups | `cmdb_ci_server`, `cmdb_ci_appl`, `cmdb_ci` |
| `calendar.go` | Blackouts lookup for the Timing guard | `cmn_blackout` |
| `calendar.go` | Blackouts lookup for the Timing guard | `cmn_schedule_span` (spans of `schedule.type=blackout`) |
| `listen.go` | Event polling, cursor over both tables | `change_request`, `incident` |
| `servicenow.go` | Wiring: config, `Describe`, delivery and lookup dispatch | |
@@ -45,9 +45,12 @@ delivery honours it.
## Events
`listen.go` polls `change_request` and `incident` by `sys_id`, with the cursor
`chg:<id>;inc:<id>` stored per emitted batch, so a restart resumes after the
last stored batch.
`listen.go` polls `change_request` and `incident` by `sys_updated_on`, bounded
above by the current time so future-dated sample rows cannot push the cursor
past the present. The cursor is `chg:<sys_updated_on>|<sys_id>;inc:...`, stored
per emitted batch, so a restart resumes after the last stored batch and an
in-place update (an approver's decision moves `sys_updated_on`, not `sys_id`)
is still caught.
- `change.approved`, `change.rejected` — the change's `approval` field. Fields:
`number`, `approval`, `state`, `short_description`. `LinkKey` is the change
@@ -57,17 +60,20 @@ last stored batch.
## Lookups
All CMDB lookups take `{"target": "<host or application name>"}` and resolve
through `cmdb_ci_server`, then `cmdb_ci_appl`, then `cmdb_ci`, with reference
fields read as display names. `found=false` means no CI matched, which core
treats as an unknown-owner finding.
through `cmdb_ci_server`, then `cmdb_ci_appl`, then `cmdb_ci`. The whole CI row
is read with display values and the fields are picked locally, because which of
`owner`, `delegate`, `environment`, `data_class` exist depends on the
instance's CMDB schema. `found=false` means no CI matched, which core treats as
an unknown-owner finding.
- `cmdb_owner` — `owner`, `delegate`, `support_group`, plus `source` (which CI
answered).
- `cmdb_environment` — `value` from the CI's `environment` (or `u_environment`).
- `cmdb_data_class` — `value` from the CI's `data_class` (or `u_data_class`).
- `blackouts` — takes `{"at": "<RFC 3339>"}`, empty for now; returns the active
`cmn_blackout` windows as `{name, type, start, end}`. The Timing guard reads
this as a freeze.
blackout spans as `{name, schedule, start, end}`, read from
`cmn_schedule_span` where `schedule.type=blackout` and the span covers the
time. The Timing guard reads this as a freeze.
## Done when (archie #1983)
+15 -12
View File
@@ -15,7 +15,9 @@ import (
// guard reads to hold changing actions during a freeze.
const (
blackoutTable = "cmn_blackout"
// spanTable holds the schedule spans; a span of a blackout schedule is one
// blackout window. Dot-walking schedule.type keeps this one query.
spanTable = "cmn_schedule_span"
blackoutsLookup = "blackouts"
)
@@ -35,10 +37,10 @@ type blackoutsInput struct {
}
type blackout struct {
Name string `json:"name"`
Type string `json:"type"`
Start string `json:"start"`
End string `json:"end"`
Name string `json:"name"`
Schedule string `json:"schedule"`
Start string `json:"start"`
End string `json:"end"`
}
// lookupBlackouts lists the blackouts in force at the given time, or now when
@@ -59,20 +61,21 @@ func (s *servicenow) lookupBlackouts(ctx context.Context, raw jsontext.Value) (j
}
stamp := at.Format("2006-01-02 15:04:05")
q := url.Values{
"sysparm_query": {"start<=" + stamp + "^end>=" + stamp},
"sysparm_fields": {"name,type,start,end"},
"sysparm_query": {"schedule.type=blackout^start<=" + stamp + "^end>=" + stamp},
"sysparm_fields": {"name,start,end,schedule"},
"sysparm_display_value": {"true"},
}
rows, err := s.client.rows(ctx, blackoutTable, q)
rows, err := s.client.rows(ctx, spanTable, q)
if err != nil {
return nil, false, err
}
out := []blackout{}
for _, row := range rows {
out = append(out, blackout{
Name: str(row["name"]),
Type: str(row["type"]),
Start: str(row["start"]),
End: str(row["end"]),
Name: str(row["name"]),
Schedule: display(row["schedule"]),
Start: str(row["start"]),
End: str(row["end"]),
})
}
res, err := marshalFields(map[string]any{"blackouts": out})
+4 -4
View File
@@ -8,9 +8,9 @@ import (
func TestLookupBlackoutsReturnsActiveWindow(t *testing.T) {
s, fake := newTestConnector(t)
fake.tables[blackoutTable] = []map[string]any{
{"name": "release freeze", "type": "blackout", "start": "2026-11-16 08:00:00", "end": "2026-11-16 18:00:00"},
{"name": "old freeze", "type": "blackout", "start": "2026-11-15 08:00:00", "end": "2026-11-15 18:00:00"},
fake.tables[spanTable] = []map[string]any{
{"name": "release freeze", "schedule.type": "blackout", "schedule": "sched-1", "start": "2026-11-16 08:00:00", "end": "2026-11-16 18:00:00"},
{"name": "old freeze", "schedule.type": "blackout", "schedule": "sched-2", "start": "2026-11-15 08:00:00", "end": "2026-11-15 18:00:00"},
}
out, found, err := s.Lookup(context.Background(), blackoutsLookup, raw(map[string]any{"at": "2026-11-16T12:00:00Z"}))
if err != nil || !found {
@@ -22,7 +22,7 @@ func TestLookupBlackoutsReturnsActiveWindow(t *testing.T) {
if err := json.Unmarshal(out, &got); err != nil {
t.Fatalf("unmarshal: %v", err)
}
if len(got.Blackouts) != 1 || got.Blackouts[0].Name != "release freeze" {
if len(got.Blackouts) != 1 || got.Blackouts[0].Name != "release freeze" || got.Blackouts[0].Schedule != "sched-1" {
t.Fatalf("blackouts = %+v, want only the active one", got.Blackouts)
}
}
+39 -29
View File
@@ -6,6 +6,8 @@ import (
json "encoding/json/v2"
"fmt"
"net/url"
"strings"
"time"
"github.com/samcharles93/archipelago/sdk/connector"
connectorv1 "github.com/samcharles93/archipelago/sdk/connector/v1"
@@ -145,12 +147,19 @@ func (s *servicenow) deliverChangeWithdraw(ctx context.Context, raw jsontext.Val
return str(updated["number"]), nil
}
// pollChanges reads change requests newer than cursor and turns an approval
// or rejection into an event. It returns the events and the new cursor.
// pollChanges reads change requests updated at or after the cursor and turns an
// approval or rejection into an event. The cursor is "<sys_updated_on>|<sys_id>",
// because an approver's decision updates the row without changing its sys_id.
// Rows at the cursor's second are filtered client-side, so nothing is missed.
func (s *servicenow) pollChanges(ctx context.Context, cursor string) ([]connector.Event, string, error) {
ts, id := splitCursor(cursor)
// The upper bound keeps future-dated sample rows from pushing the cursor
// past the present, which would hide later in-place updates.
now := time.Now().UTC().Format("2006-01-02 15:04:05")
q := url.Values{
"sysparm_query": {"sys_id>" + cursor + "^ORDERBYsys_id"},
"sysparm_fields": {"sys_id,number,approval,state,short_description,sys_updated_on"},
"sysparm_query": {"sys_updated_on>=" + ts + "^sys_updated_on<=" + now + "^ORDERBYsys_updated_on^ORDERBYsys_id"},
"sysparm_fields": {"sys_id,number,approval,state,short_description,sys_updated_on"},
"sysparm_display_value": {"true"},
}
rows, err := s.client.rows(ctx, changeTable, q)
if err != nil {
@@ -158,34 +167,35 @@ func (s *servicenow) pollChanges(ctx context.Context, cursor string) ([]connecto
}
var events []connector.Event
for _, row := range rows {
approval := str(row["approval"])
var eventType string
switch approval {
case "Approved":
eventType = changeApproved
case "Rejected":
eventType = changeRejected
default:
cursor = str(row["sys_id"])
rts, rid := str(row["sys_updated_on"]), str(row["sys_id"])
if !afterCursor(rts, rid, ts, id) {
continue
}
fields, err := json.Marshal(map[string]any{
"number": str(row["number"]),
"approval": approval,
"state": str(row["state"]),
"short_description": str(row["short_description"]),
})
if err != nil {
return nil, cursor, err
approval := display(row["approval"])
if strings.EqualFold(approval, "approved") || strings.EqualFold(approval, "rejected") {
fields, err := json.Marshal(map[string]any{
"number": str(row["number"]),
"approval": approval,
"state": str(row["state"]),
"short_description": str(row["short_description"]),
})
if err != nil {
return nil, cursor, err
}
eventType := changeApproved
if strings.EqualFold(approval, "rejected") {
eventType = changeRejected
}
events = append(events, connector.Event{
Type: eventType,
SourceEventID: rid,
Fields: fields,
LinkKey: str(row["number"]),
OccurredAt: parseTime(rts),
})
}
events = append(events, connector.Event{
Type: eventType,
SourceEventID: str(row["sys_id"]),
Fields: fields,
LinkKey: str(row["number"]),
OccurredAt: parseTime(str(row["sys_updated_on"])),
})
cursor = str(row["sys_id"])
ts, id = rts, rid
cursor = formatTableCursor(ts, id)
}
return events, cursor, nil
}
+29 -2
View File
@@ -70,7 +70,7 @@ func TestPollChangesEmitsApprovalEvents(t *testing.T) {
{"sys_id": "0000000000000011", "number": "CHG0000011", "approval": "Approved", "state": "-3", "sys_updated_on": "2026-11-16 10:01:00"},
{"sys_id": "0000000000000012", "number": "CHG0000012", "approval": "Rejected", "state": "7", "sys_updated_on": "2026-11-16 10:02:00"},
}
events, cursor, err := s.pollChanges(context.Background(), "0")
events, cursor, err := s.pollChanges(context.Background(), "")
if err != nil {
t.Fatalf("poll: %v", err)
}
@@ -83,7 +83,34 @@ func TestPollChangesEmitsApprovalEvents(t *testing.T) {
if events[1].Type != changeRejected || events[1].LinkKey != "CHG0000012" {
t.Fatalf("events[1] = %+v", events[1])
}
if cursor != "0000000000000012" {
if cursor != "2026-11-16 10:02:00|0000000000000012" {
t.Fatalf("cursor = %q", cursor)
}
}
// An approver's decision updates the change in place, so its sys_id does not
// move. The cursor is sys_updated_on, and a re-poll must still emit the event.
func TestPollChangesCatchesInPlaceApproval(t *testing.T) {
s, fake := newTestConnector(t)
fake.tables[changeTable] = []map[string]any{
{"sys_id": "0000000000000011", "number": "CHG0000011", "approval": "Not Yet Requested", "state": "-5", "sys_updated_on": "2026-11-16 10:00:00"},
}
ctx := context.Background()
if _, cursor, err := s.pollChanges(ctx, ""); err != nil {
t.Fatalf("first poll: %v", err)
} else if cursor != "2026-11-16 10:00:00|0000000000000011" {
t.Fatalf("cursor after first poll = %q", cursor)
}
fake.tables[changeTable][0]["approval"] = "Approved"
fake.tables[changeTable][0]["sys_updated_on"] = "2026-11-16 10:05:00"
events, cursor, err := s.pollChanges(ctx, "2026-11-16 10:00:00|0000000000000011")
if err != nil {
t.Fatalf("second poll: %v", err)
}
if len(events) != 1 || events[0].Type != changeApproved || events[0].LinkKey != "CHG0000011" {
t.Fatalf("events = %+v, want one change.approved for CHG0000011", events)
}
if cursor != "2026-11-16 10:05:00|0000000000000011" {
t.Fatalf("cursor = %q", cursor)
}
}
+9 -4
View File
@@ -61,22 +61,27 @@ func (c *client) one(ctx context.Context, table string, query url.Values) (map[s
return rows[0], nil
}
// singleResult is the envelope the Table API wraps one written row in.
type singleResult struct {
Result map[string]any `json:"result"`
}
// create inserts one row and returns it.
func (c *client) create(ctx context.Context, table string, body map[string]any) (map[string]any, error) {
var out map[string]any
var out singleResult
if err := c.do(ctx, http.MethodPost, c.base+"/api/now/table/"+table, body, &out); err != nil {
return nil, err
}
return out, nil
return out.Result, nil
}
// update changes one row, addressed by sys_id, and returns it.
func (c *client) update(ctx context.Context, table, sysID string, body map[string]any) (map[string]any, error) {
var out map[string]any
var out singleResult
if err := c.do(ctx, http.MethodPatch, c.base+"/api/now/table/"+table+"/"+url.PathEscape(sysID), body, &out); err != nil {
return nil, err
}
return out, nil
return out.Result, nil
}
func (c *client) do(ctx context.Context, method, u string, body map[string]any, out any) error {
+10 -20
View File
@@ -53,12 +53,13 @@ type cmdbInput struct {
}
// findCI returns the first CI matching the target, and which table it came
// from. Reference fields are requested as display values.
func (s *servicenow) findCI(ctx context.Context, target string, fields ...string) (map[string]any, string, error) {
// from. The whole row is read with display values, because which of the
// candidate fields exist depends on the instance's CMDB schema; the caller
// picks the fields it wants from what is there.
func (s *servicenow) findCI(ctx context.Context, target string) (map[string]any, string, error) {
for _, table := range ciTables {
q := url.Values{
"sysparm_query": {"name=" + target + "^ORhost_name=" + target},
"sysparm_fields": {"sys_id,name,host_name," + joinComma(fields)},
"sysparm_display_value": {"true"},
"sysparm_limit": {"1"},
}
@@ -84,7 +85,7 @@ func (s *servicenow) lookupOwner(ctx context.Context, raw jsontext.Value) (jsont
if in.Target == "" {
return nil, false, fmt.Errorf("cmdb_owner: target is required")
}
row, table, err := s.findCI(ctx, in.Target, "owner", "delegate", "support_group")
row, table, err := s.findCI(ctx, in.Target)
if err != nil {
return nil, false, err
}
@@ -92,9 +93,9 @@ func (s *servicenow) lookupOwner(ctx context.Context, raw jsontext.Value) (jsont
return nil, false, nil
}
out, err := marshalFields(map[string]any{
"owner": str(row["owner"]),
"delegate": str(row["delegate"]),
"support_group": str(row["support_group"]),
"owner": display(row["owner"]),
"delegate": display(row["delegate"]),
"support_group": display(row["support_group"]),
"source": sourceName(table),
})
if err != nil {
@@ -113,7 +114,7 @@ func (s *servicenow) lookupCIField(ctx context.Context, raw jsontext.Value, fiel
if in.Target == "" {
return nil, false, fmt.Errorf("target is required")
}
row, table, err := s.findCI(ctx, in.Target, fields...)
row, table, err := s.findCI(ctx, in.Target)
if err != nil {
return nil, false, err
}
@@ -121,7 +122,7 @@ func (s *servicenow) lookupCIField(ctx context.Context, raw jsontext.Value, fiel
return nil, false, nil
}
for _, f := range fields {
if v := str(row[f]); v != "" {
if v := display(row[f]); v != "" {
out, err := marshalFields(map[string]any{"value": v, "source": sourceName(table)})
if err != nil {
return nil, false, err
@@ -143,14 +144,3 @@ func sourceName(table string) string {
return "ci"
}
}
func joinComma(ss []string) string {
out := ""
for i, s := range ss {
if i > 0 {
out += ","
}
out += s
}
return out
}
+22 -6
View File
@@ -123,11 +123,13 @@ func (s *servicenow) deliverIncident(ctx context.Context, raw jsontext.Value, ke
return str(created["number"]), nil
}
// pollIncidents reads incidents newer than cursor and turns each into an
// incident.updated event. It returns the events and the new cursor.
// pollIncidents reads incidents updated at or after the cursor and turns each
// into an incident.updated event. The cursor works like pollChanges'.
func (s *servicenow) pollIncidents(ctx context.Context, cursor string) ([]connector.Event, string, error) {
ts, id := splitCursor(cursor)
now := time.Now().UTC().Format("2006-01-02 15:04:05")
q := url.Values{
"sysparm_query": {"sys_id>" + cursor + "^ORDERBYsys_id"},
"sysparm_query": {"sys_updated_on>=" + ts + "^sys_updated_on<=" + now + "^ORDERBYsys_updated_on^ORDERBYsys_id"},
"sysparm_fields": {"sys_id,number,state,short_description,sys_updated_on"},
}
rows, err := s.client.rows(ctx, incidentTable, q)
@@ -136,6 +138,10 @@ func (s *servicenow) pollIncidents(ctx context.Context, cursor string) ([]connec
}
var events []connector.Event
for _, row := range rows {
rts, rid := str(row["sys_updated_on"]), str(row["sys_id"])
if !afterCursor(rts, rid, ts, id) {
continue
}
fields, err := json.Marshal(map[string]any{
"number": str(row["number"]),
"state": str(row["state"]),
@@ -146,12 +152,13 @@ func (s *servicenow) pollIncidents(ctx context.Context, cursor string) ([]connec
}
events = append(events, connector.Event{
Type: incidentUpdated,
SourceEventID: str(row["sys_id"]),
SourceEventID: rid,
Fields: fields,
LinkKey: str(row["number"]),
OccurredAt: parseTime(str(row["sys_updated_on"])),
OccurredAt: parseTime(rts),
})
cursor = str(row["sys_id"])
ts, id = rts, rid
cursor = formatTableCursor(ts, id)
}
return events, cursor, nil
}
@@ -168,6 +175,15 @@ func str(v any) string {
return s
}
// display reads a field read with sysparm_display_value=true: a reference comes
// back as {"display_value": ..., "link": ...}, a plain field as a string.
func display(v any) string {
if m, ok := v.(map[string]any); ok {
return str(m["display_value"])
}
return str(v)
}
func parseTime(s string) time.Time {
t, err := time.Parse("2006-01-02 15:04:05", s)
if err != nil {
+2 -2
View File
@@ -65,14 +65,14 @@ func TestPollIncidentsEmitsUpdates(t *testing.T) {
fake.tables[incidentTable] = []map[string]any{
{"sys_id": "0000000000000001", "number": "INC0000001", "state": "2", "short_description": "demo", "sys_updated_on": "2026-11-16 10:00:00"},
}
events, cursor, err := s.pollIncidents(context.Background(), "0")
events, cursor, err := s.pollIncidents(context.Background(), "")
if err != nil {
t.Fatalf("poll: %v", err)
}
if len(events) != 1 || events[0].Type != incidentUpdated || events[0].LinkKey != "INC0000001" {
t.Fatalf("events = %+v", events)
}
if cursor != "0000000000000001" {
if cursor != "2026-11-16 10:00:00|0000000000000001" {
t.Fatalf("cursor = %q", cursor)
}
}
+22 -2
View File
@@ -9,8 +9,9 @@ import (
)
// listen.go is the event source: a poll over the change and incident tables.
// The cursor holds both tables' last seen sys_id, as "chg:<id>;inc:<id>", so a
// resumed listener picks up where the last batch ended.
// The cursor holds both tables' last seen "sys_updated_on|sys_id", as
// "chg:<ts>|<id>;inc:<ts>|<id>", so a resumed listener picks up where the last
// batch ended and an in-place update (an approval) is not missed.
// Listen polls until ctx ends. Events are emitted in batches, and the cursor
// only advances once a batch is stored, so a failed store replays the batch.
@@ -57,3 +58,22 @@ func parseCursor(cursor string) (chg, inc string) {
func formatCursor(chg, inc string) string {
return "chg:" + chg + ";inc:" + inc
}
// splitCursor splits one table's "<sys_updated_on>|<sys_id>" cursor.
func splitCursor(cursor string) (ts, id string) {
ts, id, _ = strings.Cut(cursor, "|")
return ts, id
}
func formatTableCursor(ts, id string) string {
return ts + "|" + id
}
// afterCursor reports whether a row is newer than the cursor, comparing
// sys_updated_on first and sys_id to break same-second ties.
func afterCursor(rts, rid, ts, id string) bool {
if rts != ts {
return rts > ts
}
return rid > id
}
+1 -1
View File
@@ -40,7 +40,7 @@ func TestListenEmitsBatchedEventsAndAdvancesCursor(t *testing.T) {
if len(got) != 2 {
t.Fatalf("events = %d, want 2", len(got))
}
if cursor != "chg:0000000000000011;inc:0000000000000021" {
if cursor != "chg:2026-11-16 10:01:00|0000000000000011;inc:2026-11-16 10:02:00|0000000000000021" {
t.Fatalf("cursor = %q", cursor)
}
}
+2 -2
View File
@@ -64,7 +64,7 @@ func (f *fakeServiceNow) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
f.tables[table] = append(rows, body)
w.WriteHeader(http.StatusCreated)
_ = json.MarshalWrite(w, body)
_ = json.MarshalWrite(w, map[string]any{"result": body})
case http.MethodPatch:
sysID := parts[1]
for i, row := range rows {
@@ -75,7 +75,7 @@ func (f *fakeServiceNow) ServeHTTP(w http.ResponseWriter, r *http.Request) {
row[k] = v
}
f.tables[table][i] = row
_ = json.MarshalWrite(w, row)
_ = json.MarshalWrite(w, map[string]any{"result": row})
return
}
}