Search: improve keyword search prototype (#52233)

We have an experimental search type called `patterntype:keyword`. In
testing it on Cody-style queries, it had worse relevance than our
ripgrep implementation, and was sometimes quite slow.

This PR makes improvements to query analysis:
* Reduce the number of tokens we search by using a more aggressive
stopword list
* Make stemming cheaper and less noisy by using the stem if it's a
prefix of the original
* Limit the max number of tokens we'll search over
* Remove language detection because it was too noisy and makes it hard
to compare to other search strategies

It also improves ranking:
* Enable Zoekt's keyword scoring to rank documents by (approximate) BM25
* Removes unused ranking logic related to "match groups"

Addresses https://github.com/sourcegraph/sourcegraph/issues/50786

Author: jtibshirani

internal/search/client/client.go

@@ -117,14 +117,17 @@ func (s *searchClient) Plan(
 	}
 	tr.LazyPrintf("parsing done")
 
+	features := ToFeatures(featureflag.FromContext(ctx), s.logger)
+	features.KeywordScoring = searchType == query.SearchTypeKeyword
+
 	inputs := &search.Inputs{
 		Plan:                   plan,
 		Query:                  plan.ToQ(),
 		OriginalQuery:          searchQuery,
 		SearchMode:             searchMode,
 		UserSettings:           settings,
 		OnSourcegraphDotCom:    sourcegraphDotComMode,
-		Features:               ToFeatures(featureflag.FromContext(ctx), s.logger),
+		Features:               features,
 		PatternType:            searchType,
 		Protocol:               protocol,
 		SanitizeSearchPatterns: sanitizeSearchPatterns(ctx, s.db, s.logger), // Experimental: check site config to see if search sanitization is enabled

internal/search/keyword/BUILD.bazel

@@ -3,12 +3,11 @@ package keyword
 import (
 	"strings"
 
-	"github.com/go-enry/go-enry/v2"
-	"github.com/kljensen/snowball"
-
 	"github.com/sourcegraph/sourcegraph/internal/search/query"
 )
 
+const maxTransformedPatterns = 10
+
 type keywordQuery struct {
 	query    query.Basic
 	patterns []string
@@ -44,7 +43,6 @@ func nodeToPatternsAndParameters(rootNode query.Node) ([]string, []query.Paramet
 		// Only search file content
 		{Field: query.FieldType, Value: "file"},
 	}
-	seenLangParameter := false
 
 	switch operator.Kind {
 	case query.And:
@@ -58,9 +56,6 @@ func nodeToPatternsAndParameters(rootNode query.Node) ([]string, []query.Paramet
 				if op.Field != query.FieldCount && op.Field != query.FieldCase && op.Field != query.FieldType {
 					parameters = append(parameters, op)
 				}
-				if op.Field == query.FieldLang {
-					seenLangParameter = true
-				}
 			case query.Pattern:
 				patterns = append(patterns, op.Value)
 			}
@@ -69,22 +64,6 @@ func nodeToPatternsAndParameters(rootNode query.Node) ([]string, []query.Paramet
 		patterns = concatNodeToPatterns(operator)
 	}
 
-	// Check if any of the patterns can be substituted as a lang: filter
-	if !seenLangParameter {
-		langPatternIdx := -1
-		for idx, pattern := range patterns {
-			langAlias, ok := enry.GetLanguageByAlias(pattern)
-			if ok {
-				parameters = append(parameters, query.Parameter{Field: query.FieldLang, Value: langAlias})
-				langPatternIdx = idx
-				break
-			}
-		}
-		if langPatternIdx >= 0 {
-			patterns = removeStringAtIndex(patterns, langPatternIdx)
-		}
-	}
-
 	return patterns, parameters
 }
 
@@ -107,18 +86,19 @@ func transformPatterns(patterns []string) []string {
 	}
 
 	for _, pattern := range patterns {
-		patternLowerCase := strings.ToLower(pattern)
-
-		if stopWords.Has(patternLowerCase) {
+		pattern = strings.ToLower(pattern)
+		pattern = removePunctuation(pattern)
+		if len(pattern) < 3 || isCommonTerm(pattern) {
 			continue
 		}
-		add(patternLowerCase)
 
-		stemmed, err := snowball.Stem(patternLowerCase, "english", false)
-		if err != nil {
-			continue
-		}
-		add(stemmed)
+		pattern = stemTerm(pattern)
+		add(pattern)
+	}
+
+	// To maintain decent latency, limit the number of patterns we search.
+	if len(transformedPatterns) > maxTransformedPatterns {
+		transformedPatterns = transformedPatterns[:maxTransformedPatterns]
 	}
 
 	return transformedPatterns

internal/search/keyword/match_groups.go

@@ -11,41 +11,29 @@ import (
 func TestTransformPattern(t *testing.T) {
 	patterns := []string{
 		"compute",
-		"K",
-		"Means",
+		"K",     // very short terms should be removed
+		"Means", // stop words should be removed
 		"Clustering",
-		"convert",
+		"implement", // common code-related terms should be removed
 		"int",
 		"to",
 		"string",
 		"finding",
-		"time",
-		"elapsed",
+		"\"time",    // leading punctuation should be removed
+		"elapsed\"", // trailing punctuation should be removed
 		"using",
 		"a",
 		"timer",
 		"computing",
+		"!?", // punctuation-only token should be removed
 	}
 	wantPatterns := []string{
-		"compute",
 		"comput",
-		"k",
-		"means",
-		"mean",
-		"clustering",
 		"cluster",
-		"convert",
 		"int",
 		"string",
-		"finding",
-		"find",
-		"time",
-		"elapsed",
 		"elaps",
-		"using",
-		"use",
 		"timer",
-		"computing",
 	}
 
 	gotPatterns := transformPatterns(patterns)
@@ -75,8 +63,8 @@ func TestQueryStringToKeywordQuery(t *testing.T) {
 		},
 		{
 			query:        "K MEANS CLUSTERING in python",
-			wantQuery:    autogold.Expect("count:99999999 type:file lang:Python (k OR means OR mean OR clustering OR cluster)"),
-			wantPatterns: autogold.Expect([]string{"k", "means", "mean", "clustering", "cluster"}),
+			wantQuery:    autogold.Expect("count:99999999 type:file (cluster OR python)"),
+			wantPatterns: autogold.Expect([]string{"cluster", "python"}),
 		},
 	}