docsite: allow content to be removed from the search index if it matc…

…hes a pattern in the docsite config (#83)

README.md

@@ -51,6 +51,7 @@ The site data describes the location of its templates, assets, and content. It i
 - `assetsBaseURLPath`: the URL path where the assets are available (such as `/assets/`).
 - `redirects`: an object mapping URL paths (such as `/my/old/page`) to redirect destination URLs (such as `/my/new/page`).
 - `check` (optional): an object containing a single property `ignoreURLPattern`, which is a [RE2 regexp](https://golang.org/pkg/regexp/syntax/) of URLs to ignore when checking for broken URLs with `docsite check`.
+- `search` (optional): an object containing a single proprety `skipIndexURLPattern`, which is a [RE2 regexp](https://golang.org/pkg/regexp/syntax/) pattern that if matching any content file URL will remove that file from the search index.
 
 The possible values for VFS URLs are:
 

cmd/docsite/site.go

@@ -63,6 +63,9 @@ type docsiteConfig struct {
 	Check                 struct {
 		IgnoreURLPattern string
 	}
+	Search struct {
+		SkipIndexURLPattern string
+	}
 }
 
 func partialSiteFromConfig(config docsiteConfig) (*docsite.Site, error) {
@@ -100,6 +103,13 @@ func partialSiteFromConfig(config docsiteConfig) (*docsite.Site, error) {
 	if config.AssetsBaseURLPath != "" {
 		site.AssetsBase = &url.URL{Path: config.AssetsBaseURLPath}
 	}
+	if config.Search.SkipIndexURLPattern != "" {
+		var err error
+		site.SkipIndexURLPattern, err = regexp.Compile(config.Search.SkipIndexURLPattern)
+		if err != nil {
+			return nil, err
+		}
+	}
 
 	for fromPath, toURLStr := range config.Redirects {
 		if err := addSiteRedirect(&site, fromPath, toURLStr); err != nil {

search.go

@@ -28,6 +28,11 @@ func (s *Site) Search(ctx context.Context, contentVersion string, queryStr strin
 		return nil, err
 	}
 	for _, page := range pages {
+		if s.SkipIndexURLPattern != nil && s.SkipIndexURLPattern.MatchString(page.Path) {
+			// this URL matches a pattern that we do not want to index for search, so we will skip it
+			continue
+		}
+
 		ast := markdown.NewParser(nil).Parse(page.Data)
 		data, err := s.renderTextContent(ctx, page, ast, contentVersion)
 		if err != nil {

site.go

@@ -56,6 +56,10 @@ type Site struct {
 
 	// CheckIgnoreURLPattern is a regexp matching URLs to ignore in the Check method.
 	CheckIgnoreURLPattern *regexp.Regexp
+
+	// SkipIndexURLPattern is a regexp matching URLs to ignore when searching. Any files that have a URL that match this
+	// pattern will be ignored from the search index.
+	SkipIndexURLPattern *regexp.Regexp
 }
 
 // newContentPage creates a new ContentPage in the site.