diff --git a/internal/domain/workflow/prreview/diff.go b/internal/domain/workflow/prreview/diff.go index 120707c19..edfee3c71 100644 --- a/internal/domain/workflow/prreview/diff.go +++ b/internal/domain/workflow/prreview/diff.go @@ -28,6 +28,10 @@ const ( FileModified FileStatus = "modified" FileRemoved FileStatus = "deleted" FileRenamed FileStatus = "renamed" + // FileTypeChanged is a file whose object type changed: a symlink that + // became a regular file, a submodule that became one. A permission change + // on the same type is a modification, not a typechange. + FileTypeChanged FileStatus = "typechange" ) // LineKind says where a line sits relative to a diff. @@ -97,6 +101,7 @@ type DiffStats struct { FilesModified int FilesRemoved int FilesRenamed int + FilesTypeChanged int TestFilesChanged int // TestToCodeRatio is changed test files per changed non-test file. A // change that touches no non-test file reports its own test count rather @@ -119,34 +124,141 @@ const rootClusterName = "root" const ( gitFilePrefix = "diff --git " devNull = "/dev/null" + // oldSidePrefix and newSidePrefix are the a/ and b/ git puts in front of + // the two sides of its own diff. They are not part of the path: a finding + // anchored to "b/x" names a file the snapshot does not have. A plain + // unified diff written by another tool carries no prefix, and a path whose + // first directory really is "a" or "b" keeps it, because only its own + // side's prefix is stripped. + oldSidePrefix = "a/" + newSidePrefix = "b/" ) -// gitHeaderPaths reads both paths out of a "diff --git" line. git C-quotes a -// path that needs escaping, and a quoted line is left to the ---/+++ headers: -// those name the file in every section that has content, and this is only the -// fallback for the sections that have none (a pure rename, a mode change). +// gitHeaderPaths reads both paths out of a "diff --git" line, each without its +// side prefix. git writes the paths plainly, or C-quoted when one of them holds +// something it has to escape -- and a quoted line names both sides in quotes, +// because a path that needs escaping is exactly a path this line could not be +// split on a space. func gitHeaderPaths(line string) (oldPath, newPath string) { rest := strings.TrimPrefix(line, gitFilePrefix) - if strings.HasPrefix(rest, `"`) { - return "", "" + if oldField, remaining, quoted := quotedField(rest); quoted { + newField, _, _ := quotedField(remaining) + return strings.TrimPrefix(unquotePath(oldField), oldSidePrefix), + strings.TrimPrefix(unquotePath(newField), newSidePrefix) } split := strings.LastIndex(rest, " b/") if split < 0 { return "", "" } - return strings.TrimPrefix(rest[:split], "a/"), rest[split+len(" b/"):] + return strings.TrimPrefix(rest[:split], oldSidePrefix), strings.TrimRight(rest[split+len(" b/"):], " \r") } -// headerPath reads the path out of a "--- "/"+++ " header: "b/main.go" in -// git's own output, quoted when the path carries something git has to escape, -// and tab-separated from a timestamp in a plain unified diff. -func headerPath(field string) string { - filePath := strings.TrimSpace(field) - if tab := strings.IndexByte(filePath, '\t'); tab >= 0 { - filePath = filePath[:tab] +// headerPath reads the path out of a "--- " or "+++ " header: git's "b/main.go", +// a plain unified diff's path with a timestamp after a tab, a /dev/null side, or +// the C-quoted form git writes for a path it has to escape. side is the prefix +// that side of a git diff carries; it is stripped here and only here, so the +// other side's prefix survives in a path that happens to start with it. +func headerPath(field, side string) string { + if quoted, _, ok := quotedField(field); ok { + return strings.TrimPrefix(unquotePath(quoted), side) } - filePath = strings.Trim(filePath, `"`) - return strings.TrimPrefix(strings.TrimPrefix(filePath, "b/"), "a/") + if tab := strings.IndexByte(field, '\t'); tab >= 0 { + field = field[:tab] + } + return strings.TrimPrefix(unquotePath(strings.TrimRight(field, " \r")), side) +} + +// quotedField reads the C-quoted field at the front of a header line -- the +// quotes included -- and whatever follows it: the second quoted path of a +// "diff --git" line, a timestamp, or nothing at all. +func quotedField(rest string) (field, remaining string, ok bool) { + if !strings.HasPrefix(rest, `"`) { + return "", "", false + } + for index := 1; index < len(rest); index++ { + switch rest[index] { + case '\\': + // The escaped byte, whatever it is. An octal run needs no special + // handling here: its digits can hold neither a backslash nor a + // quote. + index++ + case '"': + return rest[:index+1], strings.TrimLeft(rest[index+1:], " "), true + } + } + return "", "", false +} + +// unquoteEscapes maps the named C escapes git writes onto the bytes they stand +// for. +var unquoteEscapes = map[byte]byte{ + 'a': '\a', 'b': '\b', 'f': '\f', 'n': '\n', 'r': '\r', 't': '\t', 'v': '\v', + '\\': '\\', '"': '"', +} + +// unquotePath decodes the C-style quoting git writes around a path. The escapes +// are C's: the named ones for the bytes that cannot be written, and up to three +// octal digits for every other byte, which is how a non-ASCII path comes out of +// the default core.quotePath. A field that is not quoted is already the path. +func unquotePath(field string) string { + if !strings.HasPrefix(field, `"`) || len(field) < 2 { + return field + } + // The field ends with the quote git closed it with, and an escaped quote + // before that is written \", so the last byte is never the path's. + last := len(field) - 1 + var path strings.Builder + path.Grow(last) + for index := 1; index < last; index++ { + char := field[index] + if char != '\\' { + path.WriteByte(char) + continue + } + index++ + if index >= last { + // A trailing backslash escapes nothing, and the byte that follows + // it is the quote this path was closed with. + break + } + if named, known := unquoteEscapes[field[index]]; known { + path.WriteByte(named) + continue + } + if field[index] < '0' || field[index] > '7' { + // An escape this decoder does not know: the byte after the + // backslash is the byte git wrote. + path.WriteByte(field[index]) + continue + } + octal := 0 + for digits := 0; digits < 3 && index < last && field[index] >= '0' && field[index] <= '7'; digits++ { + octal = octal<<3 | int(field[index]-'0') + index++ + } + // The loop above leaves index on the first byte past the digits, and + // the for statement moves it one further. + index-- + path.WriteByte(byte(octal)) + } + return path.String() +} + +// fileType is the object type a git mode names. A change between two of them -- +// a regular file becoming a symlink, a submodule becoming a file -- is a +// file-type change, which git reports as "typechange"; a change between two +// permission sets of the same type is a modification. +func fileType(mode int) int { return mode & 0o170000 } + +// parseFileType reads the object type out of the mode an "old mode" or +// "new mode" line carries, and is zero when the line holds no mode this +// package can read. +func parseFileType(field string) int { + mode, err := strconv.ParseInt(strings.TrimSpace(field), 8, 32) + if err != nil || mode <= 0 { + return 0 + } + return fileType(int(mode)) } // hunkHeaderRE reads the position a hunk header gives it in both files. Both @@ -154,7 +266,11 @@ func headerPath(field string) string { // "@@" may contain anything, so nothing after it is matched. var hunkHeaderRE = regexp.MustCompile(`^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@`) -// fileInProgress is the parser's state for the file section it is inside. +// fileInProgress is the parser's state for the file section it is inside. The +// section's headers are read into it as facts -- which paths, what kind of +// change, which object type -- and close() turns those facts into one status, +// so the status is a statement about the section and not about the line of it +// that happened to come last. type fileInProgress struct { change FileChange oldPath string @@ -162,44 +278,99 @@ type fileInProgress struct { // a hunk. It is an index rather than a pointer because appending to the // hunks slice moves every element. hunkAt int + // oldSeen and newSeen are the old-file and new-file lines the open hunk + // has already been given. A hunk is over when both have reached the counts + // its header declared, which is how the format ends a hunk with no line + // that says so, and what lets the next file's "--- " header be read as one. + oldSeen int + newSeen int + // headerSeen records that the section has read its "+++ " line, so a later + // "--- " header opens the next file rather than restating this one's old + // path. + headerSeen bool + // added, removed, renamed and the two object types are what the section's + // own metadata lines said the change did. A section whose patch has no + // hunks -- a binary file, an empty file, a pure rename -- states it here + // and nowhere else. + added bool + removed bool + renamed bool + oldType int + newType int } // newFileInProgress starts the file section a "diff --git" line opens. The // header's own paths seed it, so a section with no content still has a path. func newFileInProgress(line string) *fileInProgress { oldPath, newPath := gitHeaderPaths(line) - return &fileInProgress{ - change: FileChange{Path: newPath, Status: FileModified}, - oldPath: oldPath, - hunkAt: -1, - } + return &fileInProgress{change: FileChange{Path: newPath}, oldPath: oldPath, hunkAt: -1} +} + +// newSectionInProgress starts the file section a "--- "/"+++ " header pair +// opens, read from its old-path line: the shape of a plain unified diff, which +// has no "diff --git" line and says where a file begins with that pair alone. +func newSectionInProgress(line string) *fileInProgress { + file := &fileInProgress{hunkAt: -1} + file.note(line) + return file } // note reads one of the file's own header lines: the old and new paths, the -// /dev/null side that makes a file an addition or a deletion, and the rename -// pair. Anything else the section says -- index, mode, similarity -- says +// /dev/null side that makes a file an addition or a deletion, the rename pair, +// and the mode lines -- which say the same thing for a file whose patch has no +// hunks at all. Anything else the section says -- index, similarity -- says // nothing about where the change is, which is all findings are positioned by. func (f *fileInProgress) note(line string) { switch { case strings.HasPrefix(line, "--- "): - f.oldPath = headerPath(strings.TrimPrefix(line, "--- ")) + f.oldPath = headerPath(strings.TrimPrefix(line, "--- "), oldSidePrefix) if f.oldPath == devNull { - f.change.Status = FileAdded + f.added = true f.oldPath = "" } case strings.HasPrefix(line, "+++ "): - target := headerPath(strings.TrimPrefix(line, "+++ ")) + f.headerSeen = true + target := headerPath(strings.TrimPrefix(line, "+++ "), newSidePrefix) if target == devNull { - f.change.Status = FileRemoved + f.removed = true return } f.change.Path = target case strings.HasPrefix(line, "rename from "): - f.oldPath = strings.TrimPrefix(line, "rename from ") - f.change.Status = FileRenamed + f.oldPath = unquotePath(strings.TrimPrefix(line, "rename from ")) + f.renamed = true case strings.HasPrefix(line, "rename to "): - f.change.Path = strings.TrimPrefix(line, "rename to ") - f.change.Status = FileRenamed + f.change.Path = unquotePath(strings.TrimPrefix(line, "rename to ")) + f.renamed = true + case strings.HasPrefix(line, "new file mode "): + f.added = true + case strings.HasPrefix(line, "deleted file mode "): + f.removed = true + case strings.HasPrefix(line, "old mode "): + f.oldType = parseFileType(strings.TrimPrefix(line, "old mode ")) + case strings.HasPrefix(line, "new mode "): + f.newType = parseFileType(strings.TrimPrefix(line, "new mode ")) + } +} + +// status is what the section's own headers said the change did to the file, +// read off every fact the section stated at once. A rename wins over a mode +// change, because git reports a file that was renamed and had its mode changed +// as a rename; an addition or a deletion wins over a type change, because a +// file that was not there has no old type; and a mode change between two +// permission sets of one object type is a modification, not a typechange. +func (f *fileInProgress) status() FileStatus { + switch { + case f.renamed: + return FileRenamed + case f.added: + return FileAdded + case f.removed: + return FileRemoved + case f.oldType != 0 && f.newType != 0 && f.oldType != f.newType: + return FileTypeChanged + default: + return FileModified } } @@ -220,11 +391,45 @@ func (f *fileInProgress) startHunk(line string) { Header: line, }) f.hunkAt = len(f.change.Hunks) - 1 + f.oldSeen = 0 + f.newSeen = 0 } -// body folds one line of hunk body into the change: its count and the raw -// line. Outside a hunk the same bytes are a file header, so the caller decides -// which of the two it is; this is only ever called inside one. +// hunkOpen reports whether the next line is a line of the hunk being read: a +// hunk is open and the counts its header declared still have lines to place. +func (f *fileInProgress) hunkOpen() bool { + if f.hunkAt < 0 { + return false + } + hunk := f.change.Hunks[f.hunkAt] + return f.oldSeen < hunk.OldCount || f.newSeen < hunk.NewCount +} + +// headerPair reports whether the line at index opens a file section: the "--- " +// old-path header with the "+++ " new-path header on the next line. That pair +// is how a plain unified diff -- one written by a tool that is not git -- says +// where a file begins. +func headerPair(lines []string, index int) bool { + if !strings.HasPrefix(lines[index], "--- ") { + return false + } + return index+1 < len(lines) && strings.HasPrefix(lines[index+1], "+++ ") +} + +// opensNextFile reports whether the header pair at index begins the next file +// rather than being body text of this one. A pair is body text -- a removed +// line whose text begins "-- " followed by an added line whose text begins +// "++ " -- while the open hunk is still waiting for the lines its header +// counted; and a section has read its own "+++ " line before a pair can be +// another file's, because the pair of the section being opened is its own +// header. +func (f *fileInProgress) opensNextFile(lines []string, index int) bool { + return f.headerSeen && !f.hunkOpen() && headerPair(lines, index) +} + +// body folds one line of hunk body into the change: its count, its side, and +// the raw line. Outside a hunk the same bytes are a file header, so the caller +// decides which of the two it is; this is only ever called inside one. func (f *fileInProgress) body(line string) { switch { case line == "": @@ -236,12 +441,17 @@ func (f *fileInProgress) body(line string) { // it and holds no position of its own. case strings.HasPrefix(line, "+"): f.change.LinesAdded++ + f.newSeen++ f.hunk().Lines = append(f.hunk().Lines, line) case strings.HasPrefix(line, "-"): f.change.LinesRemoved++ + f.oldSeen++ f.hunk().Lines = append(f.hunk().Lines, line) default: - // A context line advances the position without being the change. + // A context line advances the position in both files without being + // the change. + f.oldSeen++ + f.newSeen++ f.hunk().Lines = append(f.hunk().Lines, line) } } @@ -254,6 +464,7 @@ func (f *fileInProgress) hunk() *Hunk { return &f.change.Hunks[f.hunkAt] } // path survives as PreviousPath only when the change moved the file. func (f *fileInProgress) close() FileChange { change := f.change + change.Status = f.status() if change.Path == "" { change.Path = f.oldPath } @@ -264,16 +475,21 @@ func (f *fileInProgress) close() FileChange { return change } -// ParseDiff parses git's unified diff into per-file changes: the file sections -// "diff --git" opens, their rename and /dev/null headers, and the hunks that -// carry the change. It is the one parser: every position, count and path this -// package reports is read here. +// ParseDiff parses a unified diff into per-file changes: the file sections +// "diff --git" opens, the sections a "--- "/"+++ " header pair opens in a diff +// that has no such line, their rename, mode and /dev/null headers, and the +// hunks that carry the change. It is the one parser: every position, count and +// path this package reports is read here. func ParseDiff(diff string) []FileChange { var ( files []FileChange file *fileInProgress ) - for line := range strings.SplitSeq(diff, "\n") { + // The diff is read as a list rather than as a stream: whether a "--- " + // header opens the next file or is a removed line is decided by the header + // that follows it. + lines := strings.Split(diff, "\n") + for index, line := range lines { switch { case strings.HasPrefix(line, gitFilePrefix): if file != nil { @@ -281,14 +497,29 @@ func ParseDiff(diff string) []FileChange { } file = newFileInProgress(line) case file == nil: - // Text before the first "diff --git" belongs to no file, so there - // is nothing it could position a finding against. + // Text before the first file section belongs to no file, so there + // is nothing it could position a finding against -- except the + // header pair a plain unified diff begins with. + if headerPair(lines, index) { + file = newSectionInProgress(line) + } case strings.HasPrefix(line, "@@"): + // A hunk header is read as one even while a hunk is open: no body + // line of a diff begins with "@@", so this is the next hunk of the + // file, or a recovery from a section whose counts were wrong. file.startHunk(line) - case file.hunkAt < 0: - file.note(line) - default: + case file.opensNextFile(lines, index): + // A plain unified diff has no line that ends a file, so the next + // one begins where the previous one's last hunk ended. + files = append(files, file.close()) + file = newSectionInProgress(line) + case file.hunkAt >= 0: + // Every line of a hunk body carries a ' ', '+' or '-' prefix, so a + // line that looks like a file header is body text while a hunk is + // being read. file.body(line) + default: + file.note(line) } } if file != nil { @@ -390,6 +621,8 @@ func SummarizeFiles(files []FileChange) DiffStats { stats.FilesRemoved++ case FileRenamed: stats.FilesRenamed++ + case FileTypeChanged: + stats.FilesTypeChanged++ default: stats.FilesModified++ } diff --git a/internal/domain/workflow/prreview/diff_test.go b/internal/domain/workflow/prreview/diff_test.go index ad6c5e4ef..ffb500571 100644 --- a/internal/domain/workflow/prreview/diff_test.go +++ b/internal/domain/workflow/prreview/diff_test.go @@ -89,6 +89,56 @@ func TestParseDiffDescribesEachFile(t *testing.T) { diff: "", want: nil, }, + { + name: "a plain unified diff opens its files from the --- header", + diff: "--- a/a.go\n+++ b/a.go\n@@ -1,2 +1,3 @@\n package a\n+var b = 1\n var c = 2\n--- a/b.go\n+++ b/b.go\n@@ -1,1 +1,2 @@\n package b\n+var x = 1\n", + want: []fileSummary{ + {Path: "a.go", Status: "modified", Language: "go", Added: 1, Hunks: 1}, + {Path: "b.go", Status: "modified", Language: "go", Added: 1, Hunks: 1}, + }, + }, + { + name: "a plain unified diff's added file", + diff: "--- /dev/null\n+++ b/created.go\n@@ -0,0 +1,1 @@\n+package created\n", + want: []fileSummary{{Path: "created.go", Status: "added", Language: "go", Added: 1, Hunks: 1}}, + }, + { + name: "a plain unified diff's deleted file", + diff: "--- a/gone.go\n+++ /dev/null\n@@ -1,1 +0,0 @@\n-package gone\n", + want: []fileSummary{{Path: "gone.go", Status: "deleted", Language: "go", Removed: 1, Hunks: 1}}, + }, + { + // A binary file's patch carries no hunks, so the ---/+++ headers + // are absent and the mode line is the only statement of what the + // change did. git's own output, byte for byte. + name: "an added binary file is an addition", + diff: "diff --git a/logo.png b/logo.png\nnew file mode 100644\nindex 0000000..8352675\nBinary files /dev/null and b/logo.png differ\n", + want: []fileSummary{{Path: "logo.png", Status: "added"}}, + }, + { + name: "a deleted binary file is a deletion", + diff: "diff --git a/logo.png b/logo.png\ndeleted file mode 100644\nindex 8352675..0000000\nBinary files a/logo.png and /dev/null differ\n", + want: []fileSummary{{Path: "logo.png", Status: "deleted"}}, + }, + { + name: "an added empty file is an addition", + diff: "diff --git a/empty.txt b/empty.txt\nnew file mode 100644\nindex 0000000..e69de29\n", + want: []fileSummary{{Path: "empty.txt", Status: "added"}}, + }, + { + name: "a deleted empty file is a deletion", + diff: "diff --git a/empty.txt b/empty.txt\ndeleted file mode 100644\nindex e69de29..0000000\n", + want: []fileSummary{{Path: "empty.txt", Status: "deleted"}}, + }, + { + // A permission change is a modification; a change of object type + // -- a symlink becoming a regular file, a submodule becoming a + // file -- is a typechange, the status the change capture persists + // as "typechange". + name: "a file-type change is a typechange", + diff: "diff --git a/link b/link\nold mode 120000\nnew mode 100644\nindex 4cbb553..7898192\n--- a/link\n+++ b/link\n@@ -1,1 +1,1 @@\n-target.txt\n+content\n", + want: []fileSummary{{Path: "link", Status: "typechange", Added: 1, Removed: 1, Hunks: 1}}, + }, } for _, test := range tests { @@ -108,6 +158,109 @@ func TestParseDiffDescribesEachFile(t *testing.T) { } } +func TestParseDiffDecodesGitQuotedPaths(t *testing.T) { + t.Parallel() + + // git C-quotes a path it cannot write plainly: every byte outside ASCII + // under the default core.quotePath, and a quote, a backslash or a tab + // anywhere. FileChange.Path is the path a reviewer hands back to the + // snapshot, so it has to be the path and not git's spelling of it. + tests := []struct { + name string + diff string + want []fileSummary + }{ + { + name: "a non-ASCII path is escaped as the bytes of its UTF-8", + diff: "diff --git \"a/caf\\303\\251.txt\" \"b/caf\\303\\251.txt\"\n" + + "index 572eb43..1e76f07 100644\n" + + "--- \"a/caf\\303\\251.txt\"\n" + + "+++ \"b/caf\\303\\251.txt\"\n" + + "@@ -1,1 +1,2 @@\n" + + " old\n" + + "+b\n", + // 303 251 is C3 A9: "é" in UTF-8. + want: []fileSummary{{Path: "caf\u00e9.txt", Status: "modified", Added: 1, Hunks: 1}}, + }, + { + name: "a quote is escaped in the path and decoded out of it", + diff: "diff --git \"a/qu\\\"ote.txt\" \"b/qu\\\"ote.txt\"\n" + + "index 7898192..422c2b7 100644\n" + + "--- \"a/qu\\\"ote.txt\"\n" + + "+++ \"b/qu\\\"ote.txt\"\n" + + "@@ -1,1 +1,2 @@\n" + + " old\n" + + "+b\n", + want: []fileSummary{{Path: `qu"ote.txt`, Status: "modified", Added: 1, Hunks: 1}}, + }, + { + name: "a tab in a path is a tab once it is decoded", + diff: "diff --git \"a/tab\\tname.txt\" \"b/tab\\tname.txt\"\n" + + "--- \"a/tab\\tname.txt\"\n" + + "+++ \"b/tab\\tname.txt\"\n", + want: []fileSummary{{Path: "tab\tname.txt", Status: "modified"}}, + }, + { + name: "a backslash is escaped twice and decoded once", + diff: "diff --git \"a/back\\\\slash.txt\" \"b/back\\\\slash.txt\"\n" + + "--- \"a/back\\\\slash.txt\"\n" + + "+++ \"b/back\\\\slash.txt\"\n", + want: []fileSummary{{Path: `back\slash.txt`, Status: "modified"}}, + }, + { + name: "a quoted rename names both paths without their quoting", + diff: "diff --git \"a/back\\\\slash.txt\" \"b/renamed\\\\back.txt\"\n" + + "similarity index 100%\n" + + "rename from \"back\\\\slash.txt\"\n" + + "rename to \"renamed\\\\back.txt\"\n", + want: []fileSummary{{Path: `renamed\back.txt`, PreviousPath: `back\slash.txt`, Status: "renamed"}}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + got := summarize(ParseDiff(test.diff)) + if len(got) != len(test.want) { + t.Fatalf("ParseDiff() described %+v, want %+v", got, test.want) + } + for i := range got { + if got[i] != test.want[i] { + t.Errorf("file %d = %+v, want %+v", i, got[i], test.want[i]) + } + } + }) + } +} + +// TestFileStatusIsTheCapturedChangeVocabulary pins this package's status words +// to internal/domain/workflow/task/changes.go, which persists a captured +// change's status as these strings. prreview imports the standard library only, +// so this test -- rather than a shared constant -- is what keeps the two +// vocabularies one. A status read off a diff and a status read off a capture +// have to be the same word, or a caller comparing them silently misses one. +func TestFileStatusIsTheCapturedChangeVocabulary(t *testing.T) { + t.Parallel() + + captured := []struct { + status FileStatus + want string + }{ + {FileAdded, "added"}, + {FileModified, "modified"}, + {FileRemoved, "deleted"}, + {FileRenamed, "renamed"}, + {FileTypeChanged, "typechange"}, + } + + for _, test := range captured { + if string(test.status) != test.want { + t.Errorf("FileStatus %q, want %q as the change capture persists it", test.status, test.want) + } + } +} + func TestParseDiffHunkGeometry(t *testing.T) { t.Parallel() @@ -141,6 +294,14 @@ func TestParseDiffHunkGeometry(t *testing.T) { diff: "diff --git a/a.go b/a.go\n--- a/a.go\n+++ b/a.go\n@@ not a hunk header @@\n+orphan\n", want: nil, }, + { + // A removed line whose text begins "-- " and an added line + // whose text begins "++ ": the shape of a header pair, and body + // text of the hunk that is still waiting for lines. + name: "a header pair inside an unfinished hunk is body text", + diff: "diff --git a/a.md b/a.md\n--- a/a.md\n+++ b/a.md\n@@ -1,1 +1,2 @@\n--- old title\n+++ new title\n", + want: []Hunk{{OldStart: 1, OldCount: 1, NewStart: 1, NewCount: 2, Header: "@@ -1,1 +1,2 @@", Lines: []string{"--- old title", "+++ new title"}}}, + }, } for _, test := range tests { @@ -207,6 +368,16 @@ func TestAddedLinesPositionsTheChange(t *testing.T) { diff: "", want: nil, }, + { + name: "a plain unified diff numbers each file from its own hunks", + diff: "--- a/a.go\n+++ b/a.go\n@@ -1,2 +1,3 @@\n package a\n+var b = 1\n var c = 2\n--- a/b.go\n+++ b/b.go\n@@ -1,1 +1,2 @@\n package b\n+var x = 1\n", + want: []AddedLine{{Path: "a.go", Line: 2, Text: "var b = 1"}, {Path: "b.go", Line: 2, Text: "var x = 1"}}, + }, + { + name: "a plain unified diff's added file numbers from line one", + diff: "--- /dev/null\n+++ b/created.go\n@@ -0,0 +1,2 @@\n+package created\n+var a = 1\n", + want: []AddedLine{{Path: "created.go", Line: 1, Text: "package created"}, {Path: "created.go", Line: 2, Text: "var a = 1"}}, + }, } for _, test := range tests { @@ -297,6 +468,18 @@ func TestSummarizeFilesCountsTheChange(t *testing.T) { diff: "diff --git a/a_test.go b/a_test.go\n--- a/a_test.go\n+++ b/a_test.go\n@@ -1,1 +1,2 @@\n package a\n+var b = 1\n", want: DiffStats{TotalFiles: 1, TotalAdditions: 1, FilesModified: 1, TestFilesChanged: 1, TestToCodeRatio: 1}, }, + { + // The files a patch cannot show hunks for: binary, empty, and a + // pure mode change. They add no lines, and they are still an + // addition, a deletion and a typechange. + name: "files with no hunks are counted by what they did", + diff: "diff --git a/logo.png b/logo.png\nnew file mode 100644\nindex 0000000..8352675\nBinary files /dev/null and b/logo.png differ\n" + + "diff --git a/empty.txt b/empty.txt\nnew file mode 100644\nindex 0000000..e69de29\n" + + "diff --git a/gone.png b/gone.png\ndeleted file mode 100644\nindex 8352675..0000000\nBinary files a/gone.png and /dev/null differ\n" + + "diff --git a/gone.txt b/gone.txt\ndeleted file mode 100644\nindex e69de29..0000000\n" + + "diff --git a/link b/link\nold mode 120000\nnew mode 100644\n", + want: DiffStats{TotalFiles: 5, FilesAdded: 2, FilesRemoved: 2, FilesTypeChanged: 1}, + }, } for _, test := range tests { diff --git a/internal/domain/workflow/prreview/scoring.go b/internal/domain/workflow/prreview/scoring.go index 4c21deee5..528a4f4bc 100644 --- a/internal/domain/workflow/prreview/scoring.go +++ b/internal/domain/workflow/prreview/scoring.go @@ -77,11 +77,18 @@ const ( // Finding is one review finding before scoring: what the reviewer observed, // where, and the two flags that only a verdict can set. type Finding struct { - Dimension string - File string - LineStart int - LineEnd int - Severity Severity + Dimension string + File string + LineStart int + LineEnd int + Severity Severity + // Category is what the finding is about, and the third of the three things + // two findings have to share to be one claim: the PRD merges exact + // duplicates on (file, overlapping lines, category). No producer sets it + // yet, so an empty category is a category -- two findings that both leave + // it empty are duplicates of each other when they also share a file and a + // line. + Category string Title string Body string Suggestion string @@ -110,6 +117,7 @@ type ScoredFinding struct { LineStart int LineEnd int Severity Severity + Category string Title string Body string Suggestion string @@ -132,12 +140,12 @@ type ScoreInputs struct { // Score drops the findings under their severity's confidence floor, scores // what is left, merges the duplicates among the survivors and ranks the rest by -// score. The same findings always produce the same scores and the same order. +// score. The same findings always produce the same scores and the same order, +// whatever order they were handed in. // -// The floor comes before the merge and the merge keeps the higher-scoring -// duplicate, so which findings were handed in decides the result and the order -// they were handed in does not: a discarded finding must not be able to shadow -// a kept one. +// The floor comes before the merge, so a discarded finding cannot shadow a kept +// one, and the merge is a clustering rather than a scan, so a chain of +// overlapping findings collapses the same way however it was listed. func Score(findings []Finding, inputs ScoreInputs) []ScoredFinding { scored := make([]ScoredFinding, 0, len(findings)) for _, finding := range findings { @@ -146,12 +154,13 @@ func Score(findings []Finding, inputs ScoreInputs) []ScoredFinding { continue } score, applied := scoreOf(finding, severity, inputs) - candidate := ScoredFinding{ + scored = append(scored, ScoredFinding{ Dimension: finding.Dimension, File: finding.File, LineStart: finding.LineStart, LineEnd: finding.LineEnd, Severity: severity, + Category: finding.Category, Title: finding.Title, Body: finding.Body, Suggestion: finding.Suggestion, @@ -161,40 +170,146 @@ func Score(findings []Finding, inputs ScoreInputs) []ScoredFinding { Score: score, Multipliers: applied, Blocking: finding.Blocking, + }) + } + + ranked := mergeDuplicates(scored) + slices.SortFunc(ranked, compareScored) + return ranked +} + +// mergeDuplicates collapses the findings that are one claim about one place. +// Two findings are one claim when they name the same file and category and +// their line ranges overlap -- the PRD's exact-duplicate rule, with the wording +// and the severity left out of it. Overlap is transitive, so a chain 1-2, 2-3, +// 3-4 is one claim and not two. +func mergeDuplicates(findings []ScoredFinding) []ScoredFinding { + ordered := slices.Clone(findings) + slices.SortFunc(ordered, compareIdentity) + + merged := make([]ScoredFinding, 0, len(ordered)) + for start := 0; start < len(ordered); { + end := start + 1 + for end < len(ordered) && sameKey(ordered[start], ordered[end]) { + end++ } - if duplicate := indexOfDuplicate(scored, candidate); duplicate >= 0 { - if candidate.Score > scored[duplicate].Score { - scored[duplicate] = candidate - } + merged = append(merged, mergeRun(ordered[start:end])...) + start = end + } + return merged +} + +// sameKey reports whether two findings of a canonically ordered list are in the +// same (file, category) group, which is how far the grouping reaches before the +// line ranges decide. +func sameKey(a, b ScoredFinding) bool { + return a.File == b.File && a.Category == b.Category +} + +// mergeRun unions the overlapping ranges of one (file, category) group into +// clusters. The group arrives ordered by line, so a cluster is a run of members +// whose start is inside the range the run has already covered -- which is what +// makes the overlap transitive: 1-2, 2-3 and 3-4 are one cluster even though +// 1-2 and 3-4 do not touch. A run that begins where the previous one ended is +// two clusters: line 13 and the range 10-12 are two places. +// +// The cluster contributes the highest-scoring member, because that is the one +// that says the most, over the union of the cluster's lines, because the +// cluster is the claim and it covers every line of it. It is blocking when any +// member is: a merge may discard wording, never a verdict. +func mergeRun(group []ScoredFinding) []ScoredFinding { + clusters := make([]findingCluster, 0, len(group)) + for _, member := range group { + if len(clusters) > 0 && member.LineStart <= clusters[len(clusters)-1].last { + clusters[len(clusters)-1].add(member) continue } - scored = append(scored, candidate) + clusters = append(clusters, newFindingCluster(member)) } - slices.SortStableFunc(scored, func(a, b ScoredFinding) int { return cmp.Compare(b.Score, a.Score) }) - return scored -} - -// indexOfDuplicate returns the position of the scoring already kept that -// candidate duplicates, or -1 when it is a finding of its own. Two findings are -// one finding when they are the same claim about one place: the same severity, -// in the same file, over lines that overlap. The wording is not part of it -- -// two reviewers describing one defect are one comment -- and a different -// severity over the same lines is a different claim about them. -func indexOfDuplicate(scored []ScoredFinding, candidate ScoredFinding) int { - for index, kept := range scored { - if kept.File == candidate.File && kept.Severity == candidate.Severity && linesOverlap(kept, candidate) { - return index - } + merged := make([]ScoredFinding, 0, len(clusters)) + for _, cluster := range clusters { + merged = append(merged, cluster.finding()) } - return -1 + return merged } -// linesOverlap reports whether two findings' line ranges share a line. A range -// that merely ends where the next begins does not overlap it: line 13 and the -// range 10-12 are two places. -func linesOverlap(a, b ScoredFinding) bool { - return a.LineStart <= b.LineEnd && b.LineStart <= a.LineEnd +// findingCluster is the run of overlapping findings one output finding stands +// for: the best member's wording and score, the union of the members' lines, +// and the disjunction of their verdicts. The best member is kept whole, so +// choosing between two members reads the fields they carry and never the +// cluster's widened range. +type findingCluster struct { + best ScoredFinding + first int + last int + blocking bool +} + +func newFindingCluster(member ScoredFinding) findingCluster { + return findingCluster{best: member, first: member.LineStart, last: member.LineEnd, blocking: member.Blocking} +} + +// add folds a member whose range overlaps the cluster into it. +func (c *findingCluster) add(member ScoredFinding) { + c.last = max(c.last, member.LineEnd) + c.blocking = c.blocking || member.Blocking + if compareScored(member, c.best) < 0 { + c.best = member + } +} + +// finding is the one finding the cluster contributes. +func (c *findingCluster) finding() ScoredFinding { + merged := c.best + merged.LineStart = c.first + merged.LineEnd = c.last + merged.Blocking = c.blocking + return merged +} + +// compareScored is the order findings are ranked in: the highest score first, +// ties broken by the finding itself. It is a total order over what a scored +// finding carries, so it is also what picks a cluster's member and what makes +// any permutation of the same findings produce the same list in the same order. +func compareScored(a, b ScoredFinding) int { + if order := cmp.Compare(b.Score, a.Score); order != 0 { + return order + } + return compareIdentity(a, b) +} + +// compareIdentity orders two findings by their own content: where they are, +// then what they say. It carries no score, so it is the order the merge walks +// rows in -- it has to be by line for the ranges to be unioned -- and it is the +// tie-break that keeps tied findings out of the order they arrived in. +func compareIdentity(a, b ScoredFinding) int { + if order := cmp.Or( + cmp.Compare(a.File, b.File), + cmp.Compare(a.Category, b.Category), + cmp.Compare(a.LineStart, b.LineStart), + cmp.Compare(a.LineEnd, b.LineEnd), + cmp.Compare(string(a.Severity), string(b.Severity)), + cmp.Compare(a.Dimension, b.Dimension), + cmp.Compare(a.Title, b.Title), + cmp.Compare(a.Body, b.Body), + cmp.Compare(a.Suggestion, b.Suggestion), + cmp.Compare(a.Evidence, b.Evidence), + cmp.Compare(a.Confidence, b.Confidence), + slices.Compare(a.Tags, b.Tags), + slices.Compare(a.Multipliers, b.Multipliers), + ); order != 0 { + return order + } + // The gating finding comes first, so a pair that differs in nothing else + // still has one order. + if a.Blocking == b.Blocking { + return 0 + } + if a.Blocking { + return -1 + } + return 1 } // scoreOf applies the rubric: the severity's weight times the reviewer's diff --git a/internal/domain/workflow/prreview/scoring_test.go b/internal/domain/workflow/prreview/scoring_test.go index ae259e77f..26ecbd7a8 100644 --- a/internal/domain/workflow/prreview/scoring_test.go +++ b/internal/domain/workflow/prreview/scoring_test.go @@ -149,9 +149,13 @@ func TestScoreDropsFindingsUnderTheirConfidenceFloor(t *testing.T) { } } -func TestScoreMergesExactDuplicates(t *testing.T) { +func TestScoreMergesFindingsThatShareAFileLinesAndCategory(t *testing.T) { t.Parallel() + // Two findings are one claim when they name the same file, their line + // ranges overlap and they carry the same category. The wording and the + // severity are not part of that: two reviewers describing one defect at one + // place have made one finding, however differently they graded it. first := scored(SeverityImportant, 0.8) first.Title = "first" @@ -166,20 +170,30 @@ func TestScoreMergesExactDuplicates(t *testing.T) { want: []string{"first"}, }, { - name: "the same place with another severity is another finding", + name: "the same place with another severity is still one finding", second: func() Finding { f := first; f.Title = "second"; f.Severity = SeverityCritical; return f }(), - want: []string{"first", "second"}, - }, - { - name: "the same severity elsewhere is another finding", - second: func() Finding { f := first; f.Title = "second"; f.LineStart = 40; f.LineEnd = 42; return f }(), - want: []string{"first", "second"}, + want: []string{"second"}, }, { name: "another spelling of the same severity is the same finding", second: func() Finding { f := first; f.Title = "second"; f.Severity = Severity("major"); return f }(), want: []string{"first"}, }, + { + name: "the same lines under another category are another finding", + second: func() Finding { f := first; f.Title = "second"; f.Category = "security"; return f }(), + want: []string{"first", "second"}, + }, + { + name: "a finding elsewhere in the same file is another finding", + second: func() Finding { f := first; f.Title = "second"; f.LineStart = 40; f.LineEnd = 42; return f }(), + want: []string{"first", "second"}, + }, + { + name: "the same lines in another file are another finding", + second: func() Finding { f := first; f.Title = "second"; f.File = "pkg/b.go"; return f }(), + want: []string{"first", "second"}, + }, } for _, test := range tests { @@ -199,6 +213,153 @@ func TestScoreMergesExactDuplicates(t *testing.T) { } } +func TestScoreMergesTwoReviewersOfOneLine(t *testing.T) { + t.Parallel() + + // The commonest duplicate there is: two reviewers both cite one line, and + // neither finding names a category, because nothing sets one yet. An empty + // category is a category. + confident := scored(SeverityImportant, 0.9) + confident.Title = "cited by the first reviewer" + confident.LineStart = 7 + confident.LineEnd = 7 + lessConfident := scored(SeverityImportant, 0.5) + lessConfident.Title = "cited by the second reviewer" + lessConfident.LineStart = 7 + lessConfident.LineEnd = 7 + + orders := []struct { + name string + findings []Finding + }{ + {name: "the confident reviewer first", findings: []Finding{confident, lessConfident}}, + {name: "the less confident reviewer first", findings: []Finding{lessConfident, confident}}, + } + + for _, order := range orders { + t.Run(order.name, func(t *testing.T) { + t.Parallel() + + got := Score(order.findings, ScoreInputs{}) + if len(got) != 1 { + t.Fatalf("Score() returned %d findings for one cited line, want 1", len(got)) + } + if got[0].Title != confident.Title { + t.Errorf("Score() kept %q, want %q", got[0].Title, confident.Title) + } + if got[0].LineStart != 7 || got[0].LineEnd != 7 { + t.Errorf("Score() kept lines %d-%d, want 7-7", got[0].LineStart, got[0].LineEnd) + } + }) + } +} + +func TestScoreMergesOverlappingFindingsInAnyOrder(t *testing.T) { + t.Parallel() + + // Three reviewers report one defect over a chain of ranges: 1-2, 2-3 and + // 3-4. Each range overlaps the next, so the three are one claim about one + // place even though the first and the last do not touch -- and the answer + // cannot depend on which reviewer was asked first. Every finding scores the + // same, so nothing but the clustering can decide it. + chain := []Finding{ + chainFinding("reported 1-2", 1, 2), + chainFinding("reported 2-3", 2, 3), + chainFinding("reported 3-4", 3, 4), + } + + for _, order := range permutations(len(chain)) { + findings := make([]Finding, 0, len(chain)) + for _, index := range order { + findings = append(findings, chain[index]) + } + + got := Score(findings, ScoreInputs{}) + if len(got) != 1 { + // Two overlapping findings that tie can only be told apart by the + // order they were handed in, which is the defect this pins. + t.Fatalf("Score() over order %v returned %d findings, want one cluster", order, len(got)) + } + if got[0].Title != chain[0].Title { + t.Errorf("Score() over order %v kept %q, want %q", order, got[0].Title, chain[0].Title) + } + if got[0].LineStart != 1 || got[0].LineEnd != 4 { + t.Errorf("Score() over order %v kept lines %d-%d, want the cluster's 1-4", order, got[0].LineStart, got[0].LineEnd) + } + } +} + +func TestScoreKeepsABlockingDuplicateSweptIntoAMerge(t *testing.T) { + t.Parallel() + + // The merge gate ruled on both duplicates and only one of them is + // blocking. The merge keeps the higher-scoring wording, and the verdict has + // to survive it: an advisory duplicate must not be able to swallow a + // blocking one, whatever order they arrive in. + blocking := scored(SeverityCritical, 0.5) + blocking.Title = "blocking, reported with less confidence" + blocking.Blocking = true + advisory := scored(SeverityImportant, 0.9) + advisory.Title = "advisory, reported with more confidence" + + orders := []struct { + name string + findings []Finding + }{ + {name: "the advisory duplicate first", findings: []Finding{advisory, blocking}}, + {name: "the blocking duplicate first", findings: []Finding{blocking, advisory}}, + } + + for _, order := range orders { + t.Run(order.name, func(t *testing.T) { + t.Parallel() + + got := Score(order.findings, ScoreInputs{}) + if len(got) != 1 { + t.Fatalf("Score() returned %d findings for one place, want 1", len(got)) + } + if got[0].Title != advisory.Title { + t.Errorf("Score() kept %q, want the higher-scoring %q", got[0].Title, advisory.Title) + } + if !got[0].Blocking { + t.Errorf("Score() kept the advisory wording and dropped the blocking verdict: %+v", got[0]) + } + if event := ReviewEventFor(got); event != ReviewEventRequestChanges { + t.Errorf("ReviewEventFor() = %q, want %q", event, ReviewEventRequestChanges) + } + }) + } +} + +// chainFinding builds one finding of the overlapping chain the order test +// scores: the same defect, in the same file and category, at its own range. +func chainFinding(title string, lineStart, lineEnd int) Finding { + finding := scored(SeverityImportant, 0.8) + finding.Title = title + finding.LineStart = lineStart + finding.LineEnd = lineEnd + return finding +} + +// permutations returns every ordering of the indexes 0..n-1, so a test can ask +// the same question of every way the same findings could have been handed over. +func permutations(n int) [][]int { + orders := [][]int{{}} + for len(orders[0]) < n { + next := make([][]int, 0, len(orders)) + for _, order := range orders { + for candidate := range n { + if slices.Contains(order, candidate) { + continue + } + next = append(next, append(slices.Clone(order), candidate)) + } + } + orders = next + } + return orders +} + func TestScoreAppliesTheConfidenceFloorBeforeDeduplication(t *testing.T) { t.Parallel() @@ -257,11 +418,12 @@ func TestScoreMergesFindingsWhoseLinesOverlap(t *testing.T) { after.LineEnd = 13 after.Title = "after the gap" - // The same lines carrying another severity are another claim about them. - other := scored(SeverityCritical, 0.8) + // The same lines under another category are another claim about them. + other := scored(SeverityImportant, 0.8) other.LineStart = 10 other.LineEnd = 13 - other.Title = "another severity" + other.Title = "another category" + other.Category = "security" tests := []struct { name string @@ -284,9 +446,9 @@ func TestScoreMergesFindingsWhoseLinesOverlap(t *testing.T) { want: []string{"after the gap", "before the gap"}, }, { - name: "an overlapping range of another severity is another finding", + name: "an overlapping range of another category is another finding", findings: []Finding{higher, other}, - want: []string{"another severity", "higher"}, + want: []string{"another category", "higher"}, }, } @@ -307,11 +469,12 @@ func TestScoreMergesFindingsWhoseLinesOverlap(t *testing.T) { } } -func TestScoreRanksByScoreAndKeepsInputOrderOnTies(t *testing.T) { +func TestScoreRanksByScoreThenByTheFindingItself(t *testing.T) { t.Parallel() // Every finding here covers a line of its own: the ties are two findings, - // not one reported twice. + // not one reported twice. A tie is broken by the finding's own content, so + // no permutation of the same findings can rank them differently. low := scored(SeveritySuggestion, 0.5) low.Title = "low" low.LineStart = 1 @@ -338,9 +501,16 @@ func TestScoreRanksByScoreAndKeepsInputOrderOnTies(t *testing.T) { } wantStrings(t, titles, []string{"high", "tie-first", "tie-second", "low"}) - // The same findings always produce the same scores and the same order. - if again := Score(findings, ScoreInputs{}); !reflect.DeepEqual(got, again) { - t.Errorf("a second scoring pass differs:\n%+v\n%+v", got, again) + // The same findings always produce the same scores and the same order, + // whatever order they were handed in. + for _, order := range permutations(len(findings)) { + permuted := make([]Finding, 0, len(findings)) + for _, index := range order { + permuted = append(permuted, findings[index]) + } + if again := Score(permuted, ScoreInputs{}); !reflect.DeepEqual(got, again) { + t.Errorf("scoring order %v differs:\n%+v\n%+v", order, got, again) + } } } @@ -348,10 +518,12 @@ func TestScoreCopiesTheFindingItScores(t *testing.T) { t.Parallel() finding := scored(SeverityImportant, 0.8) + finding.Category = "security" got := scoreOne(t, finding, ScoreInputs{}) if got.File != finding.File || got.Dimension != finding.Dimension || got.Title != finding.Title || got.Body != finding.Body || got.Suggestion != finding.Suggestion || got.Evidence != finding.Evidence || + got.Category != finding.Category || got.LineStart != finding.LineStart || got.LineEnd != finding.LineEnd || got.Confidence != finding.Confidence { t.Errorf("ScoredFinding = %+v, want the finding it was scored from", got) }