Merge pull request #421 from nexB/drop_dephell_specifier

Drop dephell specifier

requirements.txt

@@ -1,13 +1,13 @@
 aiohttp==3.7.4
 asgiref==3.2.7
-attrs==19.3.0
+attrs==20.3.0
 backcall==0.1.0
 beautifulsoup4==4.7.1
 cached-property==1.5.1
 cffi==1.14.0
 contextlib2==0.5.5
 decorator==4.4.2
-dephell-specifier==0.2.1
+univers==21.4.8
 dj-database-url==0.4.2
 Django==3.0.13
 django-filter==2.2.0
@@ -22,7 +22,7 @@ jedi==0.17.0
 lxml==4.6.3
 more-itertools==8.0.2
 packageurl-python==0.9.3
-packaging==19.2
+packaging==20.9
 parso==0.7.0
 pexpect==4.8.0
 pickleshare==0.7.5

vulnerabilities/data_source.py

@@ -41,6 +41,8 @@
 
 import pygit2
 from packageurl import PackageURL
+from univers.version_specifier import VersionSpecifier
+from univers.versions import version_class_by_package_type
 
 from vulnerabilities.oval_parser import OvalParser
 from vulnerabilities.severity_systems import ScoringSystem
@@ -514,7 +516,7 @@ def get_data_from_xml_doc(self, xml_doc: ET.ElementTree, pkg_metadata={}) -> Lis
         "type" key.
 
         Example value of pkg_metadata:
-              {"type":"deb","qualifiers":{"distro":"buster"} }
+                {"type":"deb","qualifiers":{"distro":"buster"} }
         """
 
         all_adv = []
@@ -536,7 +538,14 @@ def get_data_from_xml_doc(self, xml_doc: ET.ElementTree, pkg_metadata={}) -> Lis
                 for package_name in test_data["package_list"]:
                     if package_name and len(package_name) >= 50:
                         continue
-                    aff_ver_range = test_data["version_ranges"] or set()
+
+                    affected_version_range = test_data["version_ranges"] or set()
+                    version_class = version_class_by_package_type[pkg_metadata["type"]]
+                    version_scheme = version_class.scheme
+
+                    affected_version_range = VersionSpecifier.from_scheme_version_spec_string(
+                        version_scheme, affected_version_range
+                    )
                     all_versions = self.pkg_manager_api.get(package_name)
 
                     # FIXME: what is this 50 DB limit? that's too small for versions
@@ -546,7 +555,9 @@ def get_data_from_xml_doc(self, xml_doc: ET.ElementTree, pkg_metadata={}) -> Lis
                     all_versions = set(filter(lambda x: len(x) < 50, all_versions))
                     if not all_versions:
                         continue
-                    affected_versions = set(filter(lambda x: x in aff_ver_range, all_versions))
+                    affected_versions = set(
+                        filter(lambda x: version_class(x) in affected_version_range, all_versions)
+                    )
                     safe_versions = all_versions - affected_versions
 
                     for version in affected_versions:

vulnerabilities/helpers.py

@@ -98,3 +98,11 @@ def requests_with_5xx_retry(max_retries=5, backoff_factor=0.5):
     session.mount("https://", adapter)
     session.mount("http://", adapter)
     return session
+
+
+def contains_alpha(string):
+    """
+    Return True if the input 'string' contains any alphabet
+    """
+
+    return any([c.isalpha() for c in string])

vulnerabilities/importers/apache_kafka.py

@@ -24,8 +24,9 @@
 
 import requests
 from bs4 import BeautifulSoup
-from dephell_specifier import RangeSpecifier
 from packageurl import PackageURL
+from univers.versions import MavenVersion
+from univers.version_specifier import VersionSpecifier
 
 from vulnerabilities.data_source import Advisory
 from vulnerabilities.data_source import DataSource
@@ -71,13 +72,23 @@ def to_advisory(self, advisory_page):
             fixed_packages = [
                 PackageURL(type="apache", name="kafka", version=version)
                 for version in self.version_api.get("apache/kafka")
-                if any([version in version_range for version_range in fixed_version_ranges])
+                if any(
+                    [
+                        MavenVersion(version) in version_range
+                        for version_range in fixed_version_ranges
+                    ]
+                )
             ]
 
             affected_packages = [
                 PackageURL(type="apache", name="kafka", version=version)
                 for version in self.version_api.get("apache/kafka")
-                if any([version in version_range for version_range in affected_version_ranges])
+                if any(
+                    [
+                        MavenVersion(version) in version_range
+                        for version_range in affected_version_ranges
+                    ]
+                )
             ]
 
             advisories.append(
@@ -107,14 +118,22 @@ def to_version_ranges(version_range_text):
             lower_bound, upper_bound = range_expression.split("to")
             lower_bound = f">={lower_bound}"
             upper_bound = f"<={upper_bound}"
-            version_ranges.append(RangeSpecifier(f"{lower_bound},{upper_bound}"))
+            version_ranges.append(
+                VersionSpecifier.from_scheme_version_spec_string(
+                    "maven", f"{lower_bound},{upper_bound}"
+                )
+            )
 
         elif "and later" in range_expression:
             # eg range_expression == "2.1.1 and later"
             range_expression = range_expression.replace("and later", "")
-            version_ranges.append(RangeSpecifier(f">={range_expression}"))
+            version_ranges.append(
+                VersionSpecifier.from_scheme_version_spec_string("maven", f">={range_expression}")
+            )
 
         else:
             # eg  range_expression == "3.0.0"
-            version_ranges.append(RangeSpecifier(range_expression))
+            version_ranges.append(
+                VersionSpecifier.from_scheme_version_spec_string("maven", range_expression)
+            )
     return version_ranges

vulnerabilities/importers/apache_tomcat.py

@@ -27,7 +27,8 @@
 
 import requests
 from bs4 import BeautifulSoup
-from dephell_specifier import RangeSpecifier
+from univers.version_specifier import VersionSpecifier
+from univers.versions import MavenVersion
 from packageurl import PackageURL
 
 from vulnerabilities.data_source import Advisory
@@ -101,7 +102,7 @@ def to_advisories(self, apache_tomcat_advisory_html):
                                 type="maven", namespace="apache", name="tomcat", version=version
                             )
                             for version in self.version_api.get("org.apache.tomcat:tomcat")
-                            if version in version_range
+                            if MavenVersion(version) in version_range
                         ]
                     )
 
@@ -126,16 +127,21 @@ def to_advisories(self, apache_tomcat_advisory_html):
 
 def parse_version_ranges(string):
     """
-    This method yields Rangespecifier objects obtained by
+    This method yields VersionSpecifier objects obtained by
     parsing `string`.
-    >> list(parse_version_ranges("Affects: 9.0.0.M1 to 9.0.0.M9"))
-    [RangeSpecifier(<=9.0.0.M9,>=9.0.0.M1)]
-
-    >> list(parse_version_ranges("Affects: 9.0.0.M1"))
-    [RangeSpecifier(>=9.0.0.M1<=9.0.0.M1)]
-
-    >> list(parse_version_ranges("Affects: 9.0.0.M1 to 9.0.0.M9, 1.2.3 to 3.4.5"))
-    [RangeSpecifier(<=9.0.0.M9,>=9.0.0.M1), RangeSpecifier(<=3.4.5,>=1.2.3)]
+    >>> list(parse_version_ranges("Affects: 9.0.0.M1 to 9.0.0.M9")) == [
+    ...     VersionSpecifier.from_scheme_version_spec_string('maven','<=9.0.0.M9,>=9.0.0.M1')
+    ...  ]
+    True
+    >>> list(parse_version_ranges("Affects: 9.0.0.M1")) == [
+    ...     VersionSpecifier.from_scheme_version_spec_string('maven','>=9.0.0.M1,<=9.0.0.M1')
+    ...  ]
+    True
+    >>> list(parse_version_ranges("Affects: 9.0.0.M1 to 9.0.0.M9, 1.2.3 to 3.4.5")) == [
+    ...     VersionSpecifier.from_scheme_version_spec_string('maven','<=9.0.0.M9,>=9.0.0.M1'),
+    ...     VersionSpecifier.from_scheme_version_spec_string('maven','<=3.4.5,>=1.2.3')
+    ...  ]
+    True
     """
     version_rng_txt = string.split("Affects:")[-1].strip()
     version_ranges = version_rng_txt.split(",")
@@ -147,4 +153,6 @@ def parse_version_ranges(string):
         else:
             lower_bound = upper_bound = version_range
 
-        yield RangeSpecifier(">=" + lower_bound + "<=" + upper_bound)
+        yield VersionSpecifier.from_scheme_version_spec_string(
+            "maven", f">={lower_bound},<={upper_bound}"
+        )

vulnerabilities/importers/elixir_security.py

@@ -24,7 +24,8 @@
 from typing import List, Set
 
 import yaml
-from dephell_specifier import RangeSpecifier
+from univers.version_specifier import VersionSpecifier
+from univers.versions import SemverVersion
 from packageurl import PackageURL
 
 from vulnerabilities.data_source import GitDataSource
@@ -86,9 +87,13 @@ def get_versions_for_pkg_from_range_list(self, version_range_list, pkg_name):
         all_version_list = self.pkg_manager_api.get(pkg_name)
         if not version_range_list:
             return [], all_version_list
-        version_ranges = {RangeSpecifier(r) for r in version_range_list}
+        version_ranges = [
+            VersionSpecifier.from_scheme_version_spec_string("semver", r)
+            for r in version_range_list
+        ]
         for version in all_version_list:
-            if any([version in v for v in version_ranges]):
+            version_obj = SemverVersion(version)
+            if any([version_obj in v for v in version_ranges]):
                 safe_pkg_versions.append(version)
 
         vuln_pkg_versions = set(all_version_list) - set(safe_pkg_versions)

vulnerabilities/importers/github.py

@@ -31,8 +31,9 @@
 from typing import Optional
 
 import requests
-from dephell_specifier import RangeSpecifier
 from packageurl import PackageURL
+from univers.version_specifier import VersionSpecifier
+from univers.versions import version_class_by_package_type
 
 from vulnerabilities.data_source import Advisory
 from vulnerabilities.data_source import DataSource
@@ -200,7 +201,7 @@ def process_response(self) -> List[Advisory]:
                         ns, pkg_name = self.process_name(ecosystem, name)
                         aff_range = adv["node"]["vulnerableVersionRange"]
                         aff_vers, unaff_vers = self.categorize_versions(
-                            aff_range, self.version_api.get(name)
+                            self.version_api.package_type, aff_range, self.version_api.get(name)
                         )
                         affected_purls = {
                             PackageURL(name=pkg_name, namespace=ns, version=version, type=pkg_type)
@@ -252,8 +253,14 @@ def process_response(self) -> List[Advisory]:
 
     @staticmethod
     def categorize_versions(
-        version_range: str, all_versions: Set[str]
-    ) -> Tuple[Set[str], Set[str]]:  # nopep8
-        version_range = RangeSpecifier(version_range)
-        affected_versions = {version for version in all_versions if version in version_range}
+        package_type: str, version_range: str, all_versions: Set[str]
+    ) -> Tuple[Set[str], Set[str]]:
+        version_class = version_class_by_package_type[package_type]
+        version_scheme = version_class.scheme
+        version_range = VersionSpecifier.from_scheme_version_spec_string(
+            version_scheme, version_range
+        )
+        affected_versions = {
+            version for version in all_versions if version_class(version) in version_range
+        }
         return (affected_versions, all_versions - affected_versions)

vulnerabilities/importers/istio.py

@@ -23,11 +23,12 @@
 import asyncio
 import re
 from typing import List, Set
-
 import yaml
 
-from dephell_specifier import RangeSpecifier
+from univers.version_specifier import VersionSpecifier
+from univers.versions import SemverVersion
 from packageurl import PackageURL
+
 from vulnerabilities.data_source import Advisory, GitDataSource, Reference
 from vulnerabilities.package_managers import GitHubTagsAPI
 
@@ -64,9 +65,13 @@ def get_pkg_versions_from_ranges(self, version_range_list):
         all_version = self.version_api.get("istio/istio")
         safe_pkg_versions = []
         vuln_pkg_versions = []
-        version_ranges = [RangeSpecifier(r) for r in version_range_list]
+        version_ranges = [
+            VersionSpecifier.from_scheme_version_spec_string("semver", r)
+            for r in version_range_list
+        ]
         for version in all_version:
-            if any([version in v for v in version_ranges]):
+            version_obj = SemverVersion(version)
+            if any([version_obj in v for v in version_ranges]):
                 vuln_pkg_versions.append(version)
 
         safe_pkg_versions = set(all_version) - set(vuln_pkg_versions)

vulnerabilities/importers/nginx.py

@@ -27,7 +27,8 @@
 import requests
 from packageurl import PackageURL
 from bs4 import BeautifulSoup
-from dephell_specifier import RangeSpecifier
+from univers.version_specifier import VersionSpecifier
+from univers.versions import SemverVersion
 
 from vulnerabilities.data_source import Advisory
 from vulnerabilities.data_source import DataSource
@@ -58,10 +59,10 @@ def set_api(self):
 
     def updated_advisories(self):
         advisories = []
-        if create_etag(data_src=self, url=self.url, etag_key="ETag"):
-            self.set_api()
-            data = requests.get(self.url).content
-            advisories.extend(self.to_advisories(data))
+        # if create_etag(data_src=self, url=self.url, etag_key="ETag"):
+        self.set_api()
+        data = requests.get(self.url).content
+        advisories.extend(self.to_advisories(data))
         return self.batch_advisories(advisories)
 
     def to_advisories(self, data):
@@ -126,13 +127,15 @@ def extract_fixed_pkgs(self, vuln_info):
         raw_ranges = version_info.split(",")
         version_ranges = []
         for rng in raw_ranges:
-            # Eg. "1.7.3+" gets converted to RangeSpecifier("^1.7.3")
+            # Eg. "1.7.3+" gets converted to VersionSpecifier.from_scheme_version_spec_string("semver","^1.7.3")
             # The advisory in this case uses `+` in the sense that any version
             # with greater or equal `minor` version satisfies the range.
             # "1.7.4" satisifes "1.7.3+", but "1.8.4" does not. "1.7.3+" has same
             # semantics as that of "^1.7.3"
 
-            version_ranges.append(RangeSpecifier("^" + rng[:-1]))
+            version_ranges.append(
+                VersionSpecifier.from_scheme_version_spec_string("semver", "^" + rng[:-1])
+            )
 
         valid_versions = find_valid_versions(self.version_api.get("nginx/nginx"), version_ranges)
 
@@ -148,16 +151,26 @@ def extract_vuln_pkgs(self, vuln_info):
         version_ranges = []
         windows_only = False
         for version_info in version_infos.split(", "):
+            if version_info == "all":
+                # This is misleading since eventually some version get fixed.
+                continue
+
             if "-" not in version_info:
                 # These are discrete versions
-                version_ranges.append(RangeSpecifier(version_info[0]))
+                version_ranges.append(
+                    VersionSpecifier.from_scheme_version_spec_string("semver", version_info[0])
+                )
                 continue
 
             windows_only = "nginx/Windows" in version_info
             version_info = version_info.replace("nginx/Windows", "")
             lower_bound, upper_bound = version_info.split("-")
 
-            version_ranges.append(RangeSpecifier(f">={lower_bound},<={upper_bound}"))
+            version_ranges.append(
+                VersionSpecifier.from_scheme_version_spec_string(
+                    "semver", f">={lower_bound},<={upper_bound}"
+                )
+            )
 
         valid_versions = find_valid_versions(self.version_api.get("nginx/nginx"), version_ranges)
         qualifiers = {}
@@ -173,7 +186,8 @@ def extract_vuln_pkgs(self, vuln_info):
 def find_valid_versions(versions, version_ranges):
     valid_versions = set()
     for version in versions:
-        if any([version in ver_range for ver_range in version_ranges]):
+        version_obj = SemverVersion(version)
+        if any([version_obj in ver_range for ver_range in version_ranges]):
             valid_versions.add(version)
 
     return valid_versions