elastic · mtojek · Sep 9, 2021 · Sep 9, 2021 · Sep 9, 2021 · Sep 9, 2021
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -10,6 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Bugfixes
 
+* Fix: don't list old packages with categories incompatible with latest revisions. [#719](https://github.com/elastic/package-registry/pull/719)
+
 ### Added
 
 * Support `elasticsearch.privileges.indices` in data stream manifests. [#713](https://github.com/elastic/package-registry/pull/713)

diff --git a/categories.go b/categories.go
@@ -40,7 +40,7 @@ func categoriesHandler(packagesBasePaths []string, cacheTime time.Duration) func
 			return
 		}
 
-		packageList := filter.Filter(r.Context(), packages)
+		packageList := filter.FilterPackages(r.Context(), packages)
 		categories := filter.FilterCategories(r.Context(), packageList)
 
 		data, err := getCategoriesOutput(r.Context(), categories)
@@ -94,7 +94,7 @@ func newCategoriesFilterFromParams(r *http.Request) (categoriesFilter, error) {
 	return filter, nil
 }
 
-func (filter categoriesFilter) Filter(ctx context.Context, packages util.Packages) map[string]util.Package {
+func (filter categoriesFilter) FilterPackages(ctx context.Context, packages util.Packages) map[string]util.Package {
 	span, ctx := apm.StartSpan(ctx, "FilterPackages", "app")
 	defer span.End()
 

diff --git a/main_test.go b/main_test.go
@@ -57,6 +57,7 @@ func TestEndpoints(t *testing.T) {
 		{"/search?kibana.version=6.5.2", "/search", "search-kibana652.json", searchHandler(packagesBasePaths, testCacheTime)},
 		{"/search?kibana.version=7.2.1", "/search", "search-kibana721.json", searchHandler(packagesBasePaths, testCacheTime)},
 		{"/search?category=web", "/search", "search-category-web.json", searchHandler(packagesBasePaths, testCacheTime)},
+		{"/search?category=web&all=true", "/search", "search-category-web-all.json", searchHandler(packagesBasePaths, testCacheTime)},
 		{"/search?category=custom", "/search", "search-category-custom.json", searchHandler(packagesBasePaths, testCacheTime)},
 		{"/search?package=example", "/search", "search-package-example.json", searchHandler(packagesBasePaths, testCacheTime)},
 		{"/search?package=example&all=true", "/search", "search-package-example-all.json", searchHandler(packagesBasePaths, testCacheTime)},

diff --git a/search.go b/search.go
@@ -35,6 +35,7 @@ func searchHandler(packagesBasePaths []string, cacheTime time.Duration) func(w h
 			return
 		}
 
+		packages = filter.FilterPackages(r.Context(), packages)
 		packagesList := filter.Filter(r.Context(), packages)
 
 		data, err := getPackageOutput(r.Context(), packagesList)
@@ -109,31 +110,65 @@ func newSearchFilterFromParams(r *http.Request) (searchFilter, error) {
 	return filter, nil
 }
 
-func (filter searchFilter) Filter(ctx context.Context, packages util.Packages) map[string]map[string]util.Package {
+func (filter searchFilter) FilterPackages(ctx context.Context, packages util.Packages) util.Packages {
 	span, ctx := apm.StartSpan(ctx, "FilterPackages", "app")
 	defer span.End()
 
-	packagesList := map[string]map[string]util.Package{}
+	if filter.AllVersions {
+		return packages
+	}
 
-	// Checks that only the most recent version of an integration is added to the list
+	packageList := map[string]util.Package{}
+
+	// Get unique list of newest packages
 	for _, p := range packages {
-		// Skip internal packages by default
-		if p.Internal && !filter.Internal {
-			continue
+		if filter.KibanaVersion != nil {
+			if valid := p.HasKibanaVersion(filter.KibanaVersion); !valid {
+				continue
+			}
 		}
 
-		// Skip experimental packages if flag is not specified
-		if p.Release == util.ReleaseExperimental && !filter.Experimental {
+		// Check if the version exists and if it should be added or not.
+		// If the package in the list is newer or equal, do nothing.
+		if pp, ok := packageList[p.Name]; ok && pp.IsNewerOrEqual(p) {
 			continue
 		}
 
+		// Otherwise delete and later add the new one.
+		packageList[p.Name] = p
+	}
+
+	var filtered util.Packages
+	for _, p := range packageList {
+		filtered = append(filtered, p)
+	}
+	return filtered
+}
+
+func (filter searchFilter) Filter(ctx context.Context, packages util.Packages) map[string]map[string]util.Package {
+	span, ctx := apm.StartSpan(ctx, "Filter", "app")
+	defer span.End()
+
+	packagesList := map[string]map[string]util.Package{}
+
+	for _, p := range packages {
 		// Filter by category first as this could heavily reduce the number of packages
 		// It must happen before the version filtering as there only the newest version
 		// is exposed and there could be an older package with more versions.
 		if filter.Category != "" && !p.HasCategory(filter.Category) && !p.HasPolicyTemplateWithCategory(filter.Category) {
 			continue
 		}
 
+		// Skip internal packages
+		if p.Internal && !filter.Internal {
+			continue
+		}
+
+		// Skip experimental packages if flag is not specified
+		if p.Release == util.ReleaseExperimental && !filter.Experimental {
+			continue
+		}
+
 		if filter.KibanaVersion != nil {
 			if valid := p.HasKibanaVersion(filter.KibanaVersion); !valid {
 				continue
@@ -181,7 +216,6 @@ func (filter searchFilter) Filter(ctx context.Context, packages util.Packages) m
 			}
 		}
 	}
-
 	return packagesList
 }
 

diff --git a/testdata/generated/search-category-web-all.json b/testdata/generated/search-category-web-all.json
@@ -0,0 +1,105 @@
+[
+  {
+    "name": "example",
+    "title": "Example",
+    "version": "0.0.2",
+    "release": "beta",
+    "description": "This is the example integration.",
+    "type": "integration",
+    "download": "/epr/example/example-0.0.2.zip",
+    "path": "/package/example/0.0.2"
+  },
+  {
+    "name": "longdocs",
+    "title": "Long Docs",
+    "version": "1.0.4",
+    "release": "ga",
+    "description": "This integration contains pretty long documentation.\nIt is used to show the different visualisations inside a documentation to test how we handle it.\nThe integration does not contain any assets except the documentation page.\n",
+    "type": "integration",
+    "download": "/epr/longdocs/longdocs-1.0.4.zip",
+    "path": "/package/longdocs/1.0.4",
+    "icons": [
+      {
+        "src": "/img/icon.svg",
+        "path": "/package/longdocs/1.0.4/img/icon.svg",
+        "type": "image/svg+xml"
+      }
+    ]
+  },
+  {
+    "name": "multiversion",
+    "title": "Multi Version",
+    "version": "1.0.3",
+    "release": "ga",
+    "description": "Multiple versions of this integration exist.\n",
+    "type": "integration",
+    "download": "/epr/multiversion/multiversion-1.0.3.zip",
+    "path": "/package/multiversion/1.0.3",
+    "icons": [
+      {
+        "src": "/img/icon.svg",
+        "path": "/package/multiversion/1.0.3/img/icon.svg",
+        "type": "image/svg+xml"
+      }
+    ]
+  },
+  {
+    "name": "multiversion",
+    "title": "Multi Version",
+    "version": "1.0.4",
+    "release": "ga",
+    "description": "Multiple versions of this integration exist.\n",
+    "type": "integration",
+    "download": "/epr/multiversion/multiversion-1.0.4.zip",
+    "path": "/package/multiversion/1.0.4",
+    "icons": [
+      {
+        "src": "/img/icon.svg",
+        "path": "/package/multiversion/1.0.4/img/icon.svg",
+        "type": "image/svg+xml"
+      }
+    ]
+  },
+  {
+    "name": "multiversion",
+    "title": "Multi Version Second with the same version! This one should win, because it is first.",
+    "version": "1.1.0",
+    "release": "ga",
+    "description": "Multiple versions of this integration exist.\n",
+    "type": "integration",
+    "download": "/epr/multiversion/multiversion-1.1.0.zip",
+    "path": "/package/multiversion/1.1.0",
+    "icons": [
+      {
+        "src": "/img/icon.svg",
+        "path": "/package/multiversion/1.1.0/img/icon.svg",
+        "type": "image/svg+xml"
+      }
+    ]
+  },
+  {
+    "name": "reference",
+    "title": "Reference package",
+    "version": "1.0.0",
+    "release": "ga",
+    "description": "This package is used for defining all the properties of a package, the possible assets etc. It serves as a reference on all the config options which are possible.\n",
+    "type": "integration",
+    "download": "/epr/reference/reference-1.0.0.zip",
+    "path": "/package/reference/1.0.0",
+    "icons": [
+      {
+        "src": "/img/icon.svg",
+        "path": "/package/reference/1.0.0/img/icon.svg",
+        "size": "32x32",
+        "type": "image/svg+xml"
+      }
+    ],
+    "policy_templates": [
+      {
+        "name": "nginx",
+        "title": "Nginx logs and metrics.",
+        "description": "Collecting logs and metrics from nginx."
+      }
+    ]
+  }
+]
diff --git a/testdata/generated/search-category-web.json b/testdata/generated/search-category-web.json
@@ -1,14 +1,4 @@
 [
-  {
-    "name": "example",
-    "title": "Example",
-    "version": "0.0.2",
-    "release": "beta",
-    "description": "This is the example integration.",
-    "type": "integration",
-    "download": "/epr/example/example-0.0.2.zip",
-    "path": "/package/example/0.0.2"
-  },
   {
     "name": "longdocs",
     "title": "Long Docs",

diff --git a/testdata/package/example/1.0.0/manifest.yml b/testdata/package/example/1.0.0/manifest.yml
@@ -11,7 +11,8 @@ release: ga
 owner.github: "ruflin"
 
 conditions:
-  kibana.version: "~7.x.x"
+  kibana:
+    version: "~7.x.x"
 
 screenshots:
   - src: /img/kibana-envoyproxy.jpg