search: use chunk matches to render multiline streamed results (#807)

CHANGELOG.md

@@ -15,6 +15,8 @@ All notable changes to `src-cli` are documented in this file.
 
 ### Changed
 
+- **IMPORTANT:** Searches using the command `src search -stream` is updated to use a **new and better search result schema**, improving highlighting and accurate result counts for multiline matches. Please see the new JSON schema for results if you use the `src search -stream -json` output: [#807](https://github.com/sourcegraph/src-cli/pull/807)
+
 ### Fixed
 
 ### Removed

cmd/src/format.go

@@ -60,9 +60,6 @@ func parseTemplate(text string) (*template.Template, error) {
 		"addFloat": func(x, y float64) float64 {
 			return x + y
 		},
-		"addInt32": func(x, y int32) int32 {
-			return x + y
-		},
 		"debug": func(v interface{}) string {
 			data, _ := marshalIndent(v)
 			fmt.Println(string(data))
@@ -95,12 +92,11 @@ func parseTemplate(text string) (*template.Template, error) {
 		"renderResult":                      searchTemplateFuncs["renderResult"],
 
 		// Register stream-search specific template functions.
-		"streamSearchSequentialLineNumber": streamSearchTemplateFuncs["streamSearchSequentialLineNumber"],
-		"streamSearchHighlightMatch":       streamSearchTemplateFuncs["streamSearchHighlightMatch"],
-		"streamSearchHighlightCommit":      streamSearchTemplateFuncs["streamSearchHighlightCommit"],
-		"streamSearchRenderCommitLabel":    streamSearchTemplateFuncs["streamSearchRenderCommitLabel"],
-		"matchOrMatches":                   streamSearchTemplateFuncs["matchOrMatches"],
-		"countMatches":                     streamSearchTemplateFuncs["countMatches"],
+		"streamSearchHighlightMatch":    streamSearchTemplateFuncs["streamSearchHighlightMatch"],
+		"streamSearchHighlightCommit":   streamSearchTemplateFuncs["streamSearchHighlightCommit"],
+		"streamSearchRenderCommitLabel": streamSearchTemplateFuncs["streamSearchRenderCommitLabel"],
+		"matchOrMatches":                streamSearchTemplateFuncs["matchOrMatches"],
+		"countMatches":                  streamSearchTemplateFuncs["countMatches"],
 
 		// Alert rendering
 		"searchAlertRender": func(alert searchResultsAlert) string {

cmd/src/search_stream.go

@@ -226,20 +226,16 @@ const streamingTemplate = `
 	{{- color "search-repository"}}{{.Repository}}{{color "nc" -}}
 	{{- " › " -}}
 	{{- color "search-filename"}}{{.Path}}{{color "nc" -}}
-	{{- color "success"}}{{matchOrMatches (countMatches .LineMatches)}}{{color "nc" -}}
+	{{- color "success"}}{{matchOrMatches (countMatches .ChunkMatches)}}{{color "nc" -}}
 	{{- "\n" -}}
 	{{- color "search-border"}}{{"--------------------------------------------------------------------------------\n"}}{{color "nc"}}
-	
-	{{- /* Line matches */ -}}
-	{{- $lineMatches := .LineMatches -}}
-	{{- range $index, $match := $lineMatches -}}
-		{{- if not (streamSearchSequentialLineNumber $lineMatches $index) -}}
-			{{- color "search-border"}}{{"  ------------------------------------------------------------------------------\n"}}{{color "nc"}}
-		{{- end -}}
-		{{- "  "}}{{color "search-line-numbers"}}{{pad (addInt32 $match.LineNumber 1) 6 " "}}{{color "nc" -}}
-		{{- color "search-border"}}{{" |  "}}{{color "nc"}}{{streamSearchHighlightMatch $.Query $match }}
+
+	{{- /* Content matches */ -}}
+	{{- $chunks := .ChunkMatches -}}
+	{{- range $index, $chunk := $chunks -}}
+		{{streamSearchHighlightMatch $chunk }}
+                {{- color "search-border"}}{{"\n  ------------------------------------------------------------------------------\n"}}{{color "nc"}}
 	{{- end -}}
-	{{- "\n" -}}
 {{- end -}}
 
 {{define "symbol"}}
@@ -250,7 +246,7 @@ const streamingTemplate = `
 	{{- color "success"}}{{matchOrMatches (len .Symbols)}}{{color "nc" -}}
 	{{- "\n" -}}
 	{{- color "search-border"}}{{"--------------------------------------------------------------------------------\n"}}{{color "nc"}}
-	
+
 	{{- /* Symbols */ -}}
 	{{- $symbols := .Symbols -}}
 	{{- range $index, $match := $symbols -}}
@@ -278,7 +274,7 @@ const streamingTemplate = `
 	{{- color "search-link"}}{{$.SourcegraphEndpoint}}{{.URL}}{{color "nc" -}}
 	{{- color "search-border"}}{{")\n"}}{{color "nc" -}}
 	{{- color "nc" -}}
-	
+
 	{{- /* Repository > author name "commit subject" (time ago) */ -}}
 	{{- color "search-commit-subject"}}{{(streamSearchRenderCommitLabel .Label)}}{{color "nc" -}}
 	{{- color "success" -}}
@@ -321,25 +317,52 @@ const streamingTemplate = `
 `
 
 var streamSearchTemplateFuncs = map[string]interface{}{
-	"streamSearchHighlightMatch": func(query string, match streaming.EventLineMatch) string {
-		var highlights []highlight
-		if strings.Contains(query, "patterntype:structural") {
-			highlights = streamConvertMatchToHighlights(match, false)
-			return applyHighlightsForFile(match.Line, highlights)
+	"streamSearchHighlightMatch": func(match streaming.ChunkMatch) string {
+		var result []rune
+
+		addLineColumn := func(n int) []rune {
+			result = append(result, []rune("  ")...)
+			result = append(result, []rune(ansiColors["search-line-numbers"])...)
+			result = append(result, []rune(fmt.Sprintf("% 6d", match.ContentStart.Line+1+n))...)
+			result = append(result, []rune(ansiColors["nc"])...)
+			result = append(result, []rune(ansiColors["search-border"])...)
+			result = append(result, []rune(" |  ")...)
+			result = append(result, []rune(ansiColors["nc"])...)
+			return result
 		}
 
-		highlights = streamConvertMatchToHighlights(match, true)
-		return applyHighlights(match.Line, highlights, ansiColors["search-match"], ansiColors["nc"])
-	},
+		// if there are overlapping ranges, use this value to know
+		// whether to terminate highlighting or not.
+		highlightingActive := 0
+
+		lineCount := 0
+		addLineColumn(lineCount)
+		for offset, r := range match.Content {
+			offset++
+			if r == '\n' {
+				lineCount++
+				result = append(result, []rune(ansiColors["nc"])...)
+				result = append(result, r)
+				addLineColumn(lineCount)
+				result = append(result, []rune(ansiColors["search-match"])...)
+				continue
+			}
+			for _, rr := range match.Ranges {
+				if rr.Start.Offset-match.ContentStart.Offset+1 == offset {
+					result = append(result, []rune(ansiColors["search-match"])...)
+					highlightingActive++
+				} else if rr.End.Offset-match.ContentStart.Offset+1 == offset {
+					highlightingActive--
+					if highlightingActive <= 0 {
+						result = append(result, []rune(ansiColors["nc"])...)
+					}
+				}
+			}
+			result = append(result, r)
 
-	"streamSearchSequentialLineNumber": func(lineMatches []streaming.EventLineMatch, index int) bool {
-		prevIndex := index - 1
-		if prevIndex < 0 {
-			return true
 		}
-		prevLineNumber := lineMatches[prevIndex].LineNumber
-		lineNumber := lineMatches[index].LineNumber
-		return prevLineNumber == lineNumber-1
+		result = append(result, []rune(ansiColors["nc"])...)
+		return string(result)
 	},
 
 	"streamSearchHighlightCommit": func(content string, ranges [][3]int32) string {
@@ -372,10 +395,10 @@ var streamSearchTemplateFuncs = map[string]interface{}{
 		return fmt.Sprintf(" (%d matches)", i)
 	},
 
-	"countMatches": func(lineMatches []streaming.EventLineMatch) int {
+	"countMatches": func(chunks []streaming.ChunkMatch) int {
 		count := 0
-		for _, l := range lineMatches {
-			count += len(l.OffsetAndLengths)
+		for _, c := range chunks {
+			count += len(c.Ranges)
 		}
 		return count
 	},
@@ -447,27 +470,6 @@ func stripMarkdownMarkers(content string) string {
 	return strings.TrimSuffix(content, "\n```")
 }
 
-// convertMatchToHighlights converts a FileMatch m to a highlight data type.
-// When isPreview is true, it is assumed that the result to highlight is only on
-// one line, and the offsets are relative to this line. When isPreview is false,
-// the lineNumber from the FileMatch data is used, which is relative to the file
-// content.
-func streamConvertMatchToHighlights(m streaming.EventLineMatch, isPreview bool) (highlights []highlight) {
-	var line int
-	for _, offsetAndLength := range m.OffsetAndLengths {
-		ol := offsetAndLength
-		offset := int(ol[0])
-		length := int(ol[1])
-		if isPreview {
-			line = 1
-		} else {
-			line = int(m.LineNumber)
-		}
-		highlights = append(highlights, highlight{line: line, character: offset, length: length})
-	}
-	return highlights
-}
-
 func logError(msg string) {
 	_, _ = fmt.Fprint(os.Stderr, msg)
 }

cmd/src/search_stream_test.go

@@ -43,11 +43,32 @@ var event = []streaming.EventMatch{
 		Repository: "org/repo",
 		Branches:   nil,
 		Commit:     "",
-		LineMatches: []streaming.EventLineMatch{
+		ChunkMatches: []streaming.ChunkMatch{
 			{
-				Line:             "foo bar foo",
-				LineNumber:       4,
-				OffsetAndLengths: [][2]int32{{0, 3}, {8, 3}},
+				Content:      "foo bar foo",
+				ContentStart: streaming.Location{Line: 4},
+				Ranges: []streaming.Range{
+					{
+						Start: streaming.Location{Offset: 0},
+						End:   streaming.Location{Offset: 3},
+					},
+					{
+						Start: streaming.Location{Offset: 0},
+						End:   streaming.Location{Offset: 3},
+					},
+					{
+						Start: streaming.Location{Offset: 1},
+						End:   streaming.Location{Offset: 2},
+					},
+					{
+						Start: streaming.Location{Offset: 1},
+						End:   streaming.Location{Offset: 3},
+					},
+					{
+						Start: streaming.Location{Offset: 8},
+						End:   streaming.Location{Offset: 11},
+					},
+				},
 			},
 		},
 	},

cmd/src/testdata/TestSearchStream/JSON.golden

@@ -1,4 +1,4 @@
-{"type":"content","path":"path/to/file","repository":"org/repo","lineMatches":[{"line":"foo bar foo","lineNumber":4,"offsetAndLengths":[[0,3],[8,3]]}]}
+{"type":"content","path":"path/to/file","repository":"org/repo","chunkMatches":[{"content":"foo bar foo","contentStart":{"offset":0,"line":4,"column":0},"ranges":[{"start":{"offset":0,"line":0,"column":0},"end":{"offset":3,"line":0,"column":0}},{"start":{"offset":0,"line":0,"column":0},"end":{"offset":3,"line":0,"column":0}},{"start":{"offset":1,"line":0,"column":0},"end":{"offset":2,"line":0,"column":0}},{"start":{"offset":1,"line":0,"column":0},"end":{"offset":3,"line":0,"column":0}},{"start":{"offset":8,"line":0,"column":0},"end":{"offset":11,"line":0,"column":0}}]}]}
 {"type":"repo","repository":"sourcegraph/sourcegraph"}
 {"type":"symbol","path":"path/to/file","repository":"org/repo","symbols":[{"url":"github.com/sourcegraph/sourcegraph/-/blob/cmd/frontend/graphqlbackend/search_results.go#L1591:26-1591:35","name":"doResults","containerName":"","kind":"FUNCTION"},{"url":"github.com/sourcegraph/sourcegraph/-/blob/cmd/frontend/graphqlbackend/search_results.go#L1591:26-1591:35","name":"Results","containerName":"SearchResultsResolver","kind":"FIELD"}]}
 {"type":"commit","icon":"","label":"[sourcegraph/sourcegraph-atom](/github.com/sourcegraph/sourcegraph-atom) › [Stephen Gutekanst](/github.com/sourcegraph/sourcegraph-atom/-/commit/5b098d7fed963d88e23057ed99d73d3c7a33ad89): [all: release v1.0.5](/github.com/sourcegraph/sourcegraph-atom/-/commit/5b098d7fed963d88e23057ed99d73d3c7a33ad89)^","url":"","detail":"","content":"```COMMIT_EDITMSG\nfoo bar\n```","ranges":[[1,3,3]]}

cmd/src/testdata/TestSearchStream/Text.golden

@@ -1,8 +1,8 @@
-org/repo › path/to/file (2 matches)
+org/repo › path/to/file (5 matches)
 --------------------------------------------------------------------------------
-       5 |  foo bar foo
-
-sourcegraph/sourcegraph (http://127.0.0.1:55128/sourcegraph/sourcegraph) (1 match)
+       5 |  foo bar foo
+  ------------------------------------------------------------------------------
+sourcegraph/sourcegraph (http://127.0.0.1:55128/sourcegraph/sourcegraph) (1 match)
 org/repo › path/to/file (2 matches)
 --------------------------------------------------------------------------------
 doResults(FUNCTION) (http://127.0.0.1:55128/github.com/sourcegraph/sourcegraph/-/blob/cmd/frontend/graphqlbackend/search_results.go#L1591:26-1591:35)

internal/streaming/events.go

@@ -17,12 +17,28 @@ type EventContentMatch struct {
 	// Type is always ContentMatchType. Included here for marshalling.
 	Type MatchType `json:"type"`
 
-	Path       string   `json:"path"`
-	Repository string   `json:"repository"`
-	Branches   []string `json:"branches,omitempty"`
-	Commit     string   `json:"commit,omitempty"`
+	Path         string       `json:"path"`
+	Repository   string       `json:"repository"`
+	Branches     []string     `json:"branches,omitempty"`
+	Commit       string       `json:"commit,omitempty"`
+	ChunkMatches []ChunkMatch `json:"chunkMatches"`
+}
+
+type ChunkMatch struct {
+	Content      string   `json:"content"`
+	ContentStart Location `json:"contentStart"`
+	Ranges       []Range  `json:"ranges"`
+}
+
+type Location struct {
+	Offset int `json:"offset"`
+	Line   int `json:"line"`
+	Column int `json:"column"`
+}
 
-	LineMatches []EventLineMatch `json:"lineMatches"`
+type Range struct {
+	Start Location `json:"start"`
+	End   Location `json:"end"`
 }
 
 func (e *EventContentMatch) eventMatch() {}

internal/streaming/search.go

@@ -32,6 +32,13 @@ func Search(query string, opts Opts, client api.Client, decoder Decoder) error {
 		req.URL.RawQuery = q.Encode()
 	}
 
+	{
+		// Consume chunk matches for streaming search.
+		q := req.URL.Query()
+		q.Add("cm", "t")
+		req.URL.RawQuery = q.Encode()
+	}
+
 	// Send request.
 	resp, err := client.Do(req)
 	if err != nil {