Adjust counting logic for categories/policy templates (#716)

* Adjust counting logic for categories

* Fix tests

* TODOs

* TODO fix

* use HasCategory

* Adjust search logic

* Update CHANGELOG

* Fix: bug

* More test cases

CHANGELOG.md

@@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 * Add -httpprof flag to enable HTTP profiling with pprof. [#709](https://github.com/elastic/package-registry/pull/709)
+* Adjust counting logic for categories/policy templates. [#716](https://github.com/elastic/package-registry/pull/716)
 
 ### Deprecated
 

categories.go

@@ -147,11 +147,15 @@ func (filter categoriesFilter) FilterCategories(ctx context.Context, packageList
 					Count: 0,
 				}
 			}
-
 			categories[c].Count = categories[c].Count + 1
 		}
 
 		if filter.IncludePolicyTemplates {
+			// /categories counts policies and packages separately, but packages are counted too
+			// if they don't match but any of their policies does (for the AWS case this would mean that
+			// the count for "datastore" would be 3: the Package and the RDS and DynamoDB policies).
+			var extraPackageCategories []string
+
 			for _, t := range p.PolicyTemplates {
 				// Skip when policy template level `categories` is empty and there is only one policy template
 				if t.Categories == nil && len(p.PolicyTemplates) == 1 {
@@ -172,6 +176,10 @@ func (filter categoriesFilter) FilterCategories(ctx context.Context, packageList
 						}
 					}
 
+					if !p.HasCategory(c) && !util.StringsContains(extraPackageCategories, c) {
+						extraPackageCategories = append(extraPackageCategories, c)
+						categories[c].Count = categories[c].Count + 1
+					}
 					categories[c].Count = categories[c].Count + 1
 				}
 			}

main_test.go

@@ -64,6 +64,7 @@ func TestEndpoints(t *testing.T) {
 		{"/search?internal=bar", "/search", "search-package-internal-error.json", searchHandler(packagesBasePaths, testCacheTime)},
 		{"/search?experimental=true", "/search", "search-package-experimental.json", searchHandler(packagesBasePaths, testCacheTime)},
 		{"/search?experimental=foo", "/search", "search-package-experimental-error.json", searchHandler(packagesBasePaths, testCacheTime)},
+		{"/search?category=datastore&experimental=true", "/search", "search-category-datastore.json", searchHandler(packagesBasePaths, testCacheTime)},
 		{"/favicon.ico", "", "favicon.ico", faviconHandleFunc},
 	}
 

search.go

@@ -130,7 +130,7 @@ func (filter searchFilter) Filter(ctx context.Context, packages util.Packages) m
 		// Filter by category first as this could heavily reduce the number of packages
 		// It must happen before the version filtering as there only the newest version
 		// is exposed and there could be an older package with more versions.
-		if filter.Category != "" && !p.HasCategory(filter.Category) {
+		if filter.Category != "" && !p.HasCategory(filter.Category) && !p.HasPolicyTemplateWithCategory(filter.Category) {
 			continue
 		}
 
@@ -170,6 +170,12 @@ func (filter searchFilter) Filter(ctx context.Context, packages util.Packages) m
 			if _, ok := packagesList[p.Name]; !ok {
 				packagesList[p.Name] = map[string]util.Package{}
 			}
+
+			if filter.Category != "" && !p.HasCategory(filter.Category) {
+				// It means that package's policy template has the category
+				p = filterPolicyTemplates(p, filter.Category)
+			}
+
 			if _, ok := packagesList[p.Name][p.Version]; !ok {
 				packagesList[p.Name][p.Version] = p
 			}
@@ -179,6 +185,20 @@ func (filter searchFilter) Filter(ctx context.Context, packages util.Packages) m
 	return packagesList
 }
 
+func filterPolicyTemplates(p util.Package, category string) util.Package {
+	var updatedPolicyTemplates []util.PolicyTemplate
+	var updatedBasePolicyTemplates []util.BasePolicyTemplate
+	for i, pt := range p.PolicyTemplates {
+		if util.StringsContains(pt.Categories, category) {
+			updatedPolicyTemplates = append(updatedPolicyTemplates, pt)
+			updatedBasePolicyTemplates = append(updatedBasePolicyTemplates, p.BasePackage.BasePolicyTemplates[i])
+		}
+	}
+	p.PolicyTemplates = updatedPolicyTemplates
+	p.BasePackage.BasePolicyTemplates = updatedBasePolicyTemplates
+	return p
+}
+
 func getPackageOutput(ctx context.Context, packagesList map[string]map[string]util.Package) ([]byte, error) {
 	span, ctx := apm.StartSpan(ctx, "GetPackageOutput", "app")
 	defer span.End()

testdata/generated/categories-include-policy-templates.json

@@ -7,7 +7,7 @@
   {
     "id": "azure",
     "title": "Azure",
-    "count": 1
+    "count": 2
   },
   {
     "id": "cloud",
@@ -17,7 +17,7 @@
   {
     "id": "compute",
     "title": "compute",
-    "count": 1
+    "count": 2
   },
   {
     "id": "containers",
@@ -27,13 +27,18 @@
   {
     "id": "crm",
     "title": "CRM",
-    "count": 1
+    "count": 2
   },
   {
     "id": "custom",
     "title": "Custom",
     "count": 13
   },
+  {
+    "id": "datastore",
+    "title": "Datastore",
+    "count": 2
+  },
   {
     "id": "message_queue",
     "title": "Message Queue",

testdata/generated/package.json

@@ -57,7 +57,10 @@
           "type": "foo"
         }
       ],
-      "multiple": true
+      "multiple": true,
+      "categories": [
+        "datastore"
+      ]
     }
   ],
   "data_streams": [

testdata/generated/package/example/1.0.0/index.json

@@ -57,7 +57,10 @@
           "type": "foo"
         }
       ],
-      "multiple": true
+      "multiple": true,
+      "categories": [
+        "datastore"
+      ]
     }
   ],
   "data_streams": [

testdata/generated/search-category-datastore.json

@@ -0,0 +1,19 @@
+[
+  {
+    "name": "example",
+    "title": "Example Integration",
+    "version": "1.0.0",
+    "release": "ga",
+    "description": "This is the example integration",
+    "type": "integration",
+    "download": "/epr/example/example-1.0.0.zip",
+    "path": "/package/example/1.0.0",
+    "policy_templates": [
+      {
+        "name": "logs",
+        "title": "Logs datasource",
+        "description": "Datasource for your log files."
+      }
+    ]
+  }
+]

testdata/package/example/1.0.0/manifest.yml

@@ -23,5 +23,7 @@ policy_templates:
   - name: logs
     title: Logs datasource
     description: Datasource for your log files.
+    categories:
+      - datastore
     inputs:
       - type: foo

util/package.go

@@ -286,12 +286,15 @@ func NewPackage(basePath string) (*Package, error) {
 }
 
 func (p *Package) HasCategory(category string) bool {
-	for _, c := range p.Categories {
-		if c == category {
+	return StringsContains(p.Categories, category)
+}
+
+func (p *Package) HasPolicyTemplateWithCategory(category string) bool {
+	for _, pt := range p.PolicyTemplates {
+		if StringsContains(pt.Categories, category) {
 			return true
 		}
 	}
-
 	return false
 }
 

util/strings.go

@@ -0,0 +1,15 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package util
+
+// StringsContains function checks if needle is present in the stack.
+func StringsContains(haystack []string, needle string) bool {
+	for _, h := range haystack {
+		if h == needle {
+			return true
+		}
+	}
+	return false
+}