From c38f3fc30154fa9a6b8998d94de831cfbecc161f Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Mon, 21 Oct 2024 15:49:44 +1300 Subject: [PATCH 1/9] feat: record ecosystems in a json file Signed-off-by: Gareth Jones --- ecosystems.json | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 ecosystems.json diff --git a/ecosystems.json b/ecosystems.json new file mode 100644 index 0000000..6f4599f --- /dev/null +++ b/ecosystems.json @@ -0,0 +1,35 @@ +{ + "AlmaLinux": "AlmaLinux package ecosystem; the `name` is the name of the source package. The ecosystem string might optionally have a `:` suffix to scope the package to a particular AlmaLinux release. `` is a numeric version.", + "Alpine": "The Alpine package ecosystem; the `name` is the name of the source package. The ecosystem string must have a `:v` suffix to scope the package to a particular Alpine release branch (the `v` prefix is required). E.g. `v3.16`.", + "Android": " The Android ecosystem. Android organizes code using [`repo` tool](https://gerrit.googlesource.com/git-repo/+/HEAD/README.md), which manages multiple git projects under one or more remote git servers, where each project is identified by its name in [repo configuration](https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md#Element-project) (e.g. `platform/frameworks/base`). The `name` field should contain the name of that affected git project/submodule. One exception is when the project contains the Linux kernel source code, in which case `name` field will be `:linux_kernel:`, followed by an optional SoC vendor name e.g. `:linux_kernel:Qualcomm`. The list of recognized SoC vendors is listed in the [Appendix](#android-soc-vendors)", + "Bioconductor": "The biological R package ecosystem. The `name` is an R package name.", + "Bitnami": "Bitnami package ecosystem; the `name` is the name of the affected component.", + "Chainguard": "The Chainguard package ecosystem; the `name` is the name of the package.", + "ConanCenter": "The ConanCenter ecosystem for C and C++; the `name` field is a Conan package name. ", + "CRAN": "The R package ecosystem. The `name` is an R package name.", + "crates.io": "The crates.io ecosystem for Rust; the `name` field is a crate name. ", + "Debian": "The Debian package ecosystem; the `name` is the name of the source package. The ecosystem string might optionally have a `:` suffix to scope the package to a particular Debian release. `` is a numeric version specified in the [Debian distro-info-data](https://debian.pages.debian.net/distro-info-data/debian.csv). For example, the ecosystem string \"Debian:7\" refers to the Debian 7 (wheezy) release. ", + "GHC": "The Haskell compiler ecosystem. The `name` field is the name of a component of the GHC compiler ecosystem (e.g., compiler, GHCI, RTS). ", + "GitHub Actions": "The GitHub Actions ecosystem; the `name` field is the action's repository name with owner e.g. `{owner}/{repo}`.", + "Go": "The Go ecosystem; the `name` field is a Go module path. ", + "Hackage": "The Haskell package ecosystem. The `name` field is a Haskell package name as published on Hackage. ", + "Hex": "The package manager for the Erlang ecosystem; the `name` is a Hex package name. ", + "Linux": "The Linux kernel. The only supported `name` is `Kernel`.", + "Mageia": "The Mageia Linux package ecosystem; the `name` is the name of the source package. The ecosystem string must have a `:` suffix to scope the package to a particular Mageia release. Eg `Mageia:9`.", + "Maven": "The Maven Java package ecosystem. The `name` field is a Maven package name in the format `groupId:artifactId`. The ecosystem string might optionally have a `:` suffix to denote the remote repository URL that best represents the source of truth for this package, without a trailing slash (e.g. `Maven:https://maven.google.com`). If this is omitted, this is assumed to be the Maven Central repository (`https://repo.maven.apache.org/maven2`).", + "npm": "The NPM ecosystem; the `name` field is an NPM package name. ", + "NuGet": "The NuGet package ecosystem. The `name` field is a NuGet package name. ", + "OSS-Fuzz": "For reports from the OSS-Fuzz project that have no more appropriate ecosystem; the `name` field is the name assigned by the OSS-Fuzz project, as recorded in the submitted fuzzing configuration. ", + "openSUSE": "The openSUSE ecosystem; The ecosystem string has a `:` suffix presenting the marketing name of the openSUSE distribution. `` matches the value in the `/etc/os-release` `PRETTY_NAME` field. The `name` field is the name of the source RPM and accompanied by a purl. There is an `ecosystem_specific` specific array `binaries` of the associated RPM binary packages in this specific openSUSE distribution. The ECOSYSTEM version ordering is the RPM versioncompare ordering, and the database uses the `introduced` and `fixed` boundaries.", + "Packagist": "The PHP package manager ecosystem; the `name` is a package name. ", + "Photon OS": "The Photon OS package ecosystem; the `name` is the name of the RPM package. The ecosystem string must have a `:` suffix to scope the package to a particular Photon OS release. Eg `Photon OS:3.0`.", + "Pub": "The package manager for the Dart ecosystem; the `name` field is a Dart package name.", + "PyPI": "the Python PyPI ecosystem; the `name` field is a [normalized](https://www.python.org/dev/peps/pep-0503/#normalized-names) PyPI package name. ", + "Red Hat": "The Red Hat package ecosystem; the `name` field is the name of a binary or source RPM. The ecosystem string has a `:` suffix to scope the RPM to a specific Red Hat product stream. `` is a translation of a Red Hat [Common Platform Enumerations](https://cpe.mitre.org/) (CPE) with the `cpe/:[oa]:(redhat):` prefix removed (for example, `Red Hat:rhel_aus:8.4::appstream` translates to `cpe:/a:redhat:rhel_aus:8.4::appstream`). Red Hat ecosystem identifiers can be used to identify vulnerable RPMs installed on a Red Hat system as explained [here](https://www.redhat.com/en/blog/how-accurately-match-oval-security-data-installed-rpms). ", + "Rocky Linux": "The Rocky Linux package ecosystem; the `name` is the name of the source package. The ecosystem string might optionally have a `:` suffix to scope the package to a particular Rocky Linux release. `` is a numeric version.", + "RubyGems": "The RubyGems ecosystem; the `name` field is a gem name. ", + "SUSE": "The SUSE ecosystem; The ecosystem string has a `:` suffix representing the marketing name of the SUSE product. `` matches the value in the /etc/os-release `PRETTY_NAME` field. The `name` field is the name of the source RPM and accompanied by a purl. There is a `ecosystem_specific` specific array `binaries` of the associated RPM binary packages in this specific SUSE product. The ECOSYSTEM version ordering is the RPM versioncompare ordering, and the database uses the `introduced` and `fixed` boundaries.", + "SwiftURL": "The Swift Package Manager ecosystem. The `name` is a Git URL to the source of the package. Versions are Git tags that comform to [SemVer 2.0](https://docs.swift.org/package-manager/PackageDescription/PackageDescription.html#version).", + "Ubuntu": "The Ubuntu package ecosystem; the `name` field is the name of the source package. The ecosystem string has a `:` suffix to scope the package to a particular Ubuntu release. `` is a numeric (\"YY.MM\") version as specified in [Ubuntu Releases](https://wiki.ubuntu.com/Releases), with a mandatory `:LTS` suffix if the release is marked as LTS. The release version may also be prefixed with `:Pro:` to denote Ubuntu Pro (aka Expanded Security Maintenance (ESM)) updates. For example, the ecosystem string \"Ubuntu:22.04:LTS\" refers to Ubuntu 22.04 LTS (jammy), while \"Ubuntu:Pro:18.04:LTS\" refers to fixes that landed in Ubuntu 18.04 LTS (bionic) under Ubuntu Pro/ESM.", + "Wolfi": "The Wolfi package ecosystem; the `name` is the name of the package." +} From 87f73c6b1977f6c33b6d7697fe13c3fd94c9491e Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Mon, 21 Oct 2024 15:50:28 +1300 Subject: [PATCH 2/9] feat: create script for updating lists of ecosystems Signed-off-by: Gareth Jones --- scripts/update-ecosystems-lists.py | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100755 scripts/update-ecosystems-lists.py diff --git a/scripts/update-ecosystems-lists.py b/scripts/update-ecosystems-lists.py new file mode 100755 index 0000000..88acda3 --- /dev/null +++ b/scripts/update-ecosystems-lists.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python3 + +import json + +ecosystems: dict[str, str] = json.loads(open('ecosystems.json').read()) + +# nothing to do for now ;) From e67e37559f0430350c0afbe33c0b2e85f1e5b734 Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Mon, 21 Oct 2024 16:10:22 +1300 Subject: [PATCH 3/9] feat: support updating json schema file Signed-off-by: Gareth Jones --- scripts/update-ecosystems-lists.py | 26 +++++++++++++++++++++++++- validation/schema.json | 4 ++-- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/scripts/update-ecosystems-lists.py b/scripts/update-ecosystems-lists.py index 88acda3..26d34db 100755 --- a/scripts/update-ecosystems-lists.py +++ b/scripts/update-ecosystems-lists.py @@ -4,4 +4,28 @@ ecosystems: dict[str, str] = json.loads(open('ecosystems.json').read()) -# nothing to do for now ;) + +def update_json_schema(): + """ + Updates references to ecosystems defined in the OSV JSON schema + :return: + """ + schema = json.loads(open('validation/schema.json').read()) + + names = ecosystems.keys() + pattern = "" + + for name in names: + pattern += name.replace(".", "\\.") + pattern += "|" + + # this is a special "ecosystem" name + pattern += "GIT" + + schema['$defs']['ecosystemName']['enum'] = list(names) + schema['$defs']['ecosystemWithSuffix']['pattern'] = f'^({pattern})(:.+)?$' + + open('validation/schema.json', 'w').write(json.dumps(schema, indent=2) + '\n') + + +update_json_schema() diff --git a/validation/schema.json b/validation/schema.json index 63e155d..ab82fc7 100644 --- a/validation/schema.json +++ b/validation/schema.json @@ -321,8 +321,8 @@ "Maven", "npm", "NuGet", - "OSS-Fuzz", "openSUSE", + "OSS-Fuzz", "Packagist", "Photon OS", "Pub", @@ -344,7 +344,7 @@ "type": "string", "title": "Currently supported ecosystems", "description": "These ecosystems are also documented at https://ossf.github.io/osv-schema/#affectedpackage-field", - "pattern": "^(AlmaLinux|Alpine|Android|Bioconductor|Bitnami|Chainguard|ConanCenter|CRAN|crates\\.io|Debian|GHC|GitHub Actions|Go|Hackage|Hex|Linux|Mageia|Maven|npm|NuGet|OSS-Fuzz|openSUSE|Packagist|Photon OS|Pub|PyPI|Red Hat|Rocky Linux|RubyGems|SUSE|SwiftURL|Ubuntu|Wolfi|GIT)(:.+)?$" + "pattern": "^(AlmaLinux|Alpine|Android|Bioconductor|Bitnami|Chainguard|ConanCenter|CRAN|crates\\.io|Debian|GHC|GitHub Actions|Go|Hackage|Hex|Linux|Mageia|Maven|npm|NuGet|openSUSE|OSS-Fuzz|Packagist|Photon OS|Pub|PyPI|Red Hat|Rocky Linux|RubyGems|SUSE|SwiftURL|Ubuntu|Wolfi|GIT)(:.+)?$" }, "prefix": { "type": "string", From 486019f46984a24594c0bb0c618190714777a55b Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Tue, 22 Oct 2024 07:39:36 +1300 Subject: [PATCH 4/9] feat: support updating markdown table in schema file Signed-off-by: Gareth Jones --- docs/schema.md | 23 +++++++++------- scripts/update-ecosystems-lists.py | 44 ++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 10 deletions(-) diff --git a/docs/schema.md b/docs/schema.md index 910e588..6cdb8b0 100644 --- a/docs/schema.md +++ b/docs/schema.md @@ -684,14 +684,15 @@ versions and different potential vulnerabilities: #### Defined ecosystems - The defined ecosystems are: -| Ecosystem | Description | -| --------- |-----------------| -| `AlmaLinux` | AlmaLinux package ecosystem; the `name` is the name of the source package. The ecosystem string might optionally have a `:` suffix to scope the package to a particular AlmaLinux release. `` is a numeric version. + + +| Ecosystem | Description | +|-----------|-------------| +| `AlmaLinux` | AlmaLinux package ecosystem; the `name` is the name of the source package. The ecosystem string might optionally have a `:` suffix to scope the package to a particular AlmaLinux release. `` is a numeric version. | | `Alpine` | The Alpine package ecosystem; the `name` is the name of the source package. The ecosystem string must have a `:v` suffix to scope the package to a particular Alpine release branch (the `v` prefix is required). E.g. `v3.16`. | -| `Android` | The Android ecosystem. Android organizes code using [`repo` tool](https://gerrit.googlesource.com/git-repo/+/HEAD/README.md), which manages multiple git projects under one or more remote git servers, where each project is identified by its name in [repo configuration](https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md#Element-project) (e.g. `platform/frameworks/base`). The `name` field should contain the name of that affected git project/submodule. One exception is when the project contains the Linux kernel source code, in which case `name` field will be `:linux_kernel:`, followed by an optional SoC vendor name e.g. `:linux_kernel:Qualcomm`. The list of recognized SoC vendors is listed in the [Appendix](#android-soc-vendors) | +| `Android` | The Android ecosystem. Android organizes code using [`repo` tool](https://gerrit.googlesource.com/git-repo/+/HEAD/README.md), which manages multiple git projects under one or more remote git servers, where each project is identified by its name in [repo configuration](https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md#Element-project) (e.g. `platform/frameworks/base`). The `name` field should contain the name of that affected git project/submodule. One exception is when the project contains the Linux kernel source code, in which case `name` field will be `:linux_kernel:`, followed by an optional SoC vendor name e.g. `:linux_kernel:Qualcomm`. The list of recognized SoC vendors is listed in the [Appendix](#android-soc-vendors) | | `Bioconductor` | The biological R package ecosystem. The `name` is an R package name. | | `Bitnami` | Bitnami package ecosystem; the `name` is the name of the affected component. | | `Chainguard` | The Chainguard package ecosystem; the `name` is the name of the package. | @@ -706,24 +707,26 @@ The defined ecosystems are: | `Hex` | The package manager for the Erlang ecosystem; the `name` is a Hex package name. | | `Linux` | The Linux kernel. The only supported `name` is `Kernel`. | | `Mageia` | The Mageia Linux package ecosystem; the `name` is the name of the source package. The ecosystem string must have a `:` suffix to scope the package to a particular Mageia release. Eg `Mageia:9`. | -| `Maven` | The Maven Java package ecosystem. The `name` field is a Maven package name in the format `groupId:artifactId`. The ecosystem string might optionally have a `:` suffix to denote the remote repository URL that best represents the source of truth for this package, without a trailing slash (e.g. `Maven:https://maven.google.com`). If this is omitted, this is assumed to be the Maven Central repository (`https://repo.maven.apache.org/maven2`). +| `Maven` | The Maven Java package ecosystem. The `name` field is a Maven package name in the format `groupId:artifactId`. The ecosystem string might optionally have a `:` suffix to denote the remote repository URL that best represents the source of truth for this package, without a trailing slash (e.g. `Maven:https://maven.google.com`). If this is omitted, this is assumed to be the Maven Central repository (`https://repo.maven.apache.org/maven2`). | | `npm` | The NPM ecosystem; the `name` field is an NPM package name. | | `NuGet` | The NuGet package ecosystem. The `name` field is a NuGet package name. | | `OSS-Fuzz` | For reports from the OSS-Fuzz project that have no more appropriate ecosystem; the `name` field is the name assigned by the OSS-Fuzz project, as recorded in the submitted fuzzing configuration. | -| `openSUSE` | The openSUSE ecosystem; The ecosystem string has a `:` suffix presenting the marketing name of the openSUSE distribution. `` matches the value in the `/etc/os-release` `PRETTY_NAME` field. The `name` field is the name of the source RPM and accompanied by a purl. There is an `ecosystem_specific` specific array `binaries` of the associated RPM binary packages in this specific openSUSE distribution. The ECOSYSTEM version ordering is the RPM versioncompare ordering, and the database uses the `introduced` and `fixed` boundaries.| +| `openSUSE` | The openSUSE ecosystem; The ecosystem string has a `:` suffix presenting the marketing name of the openSUSE distribution. `` matches the value in the `/etc/os-release` `PRETTY_NAME` field. The `name` field is the name of the source RPM and accompanied by a purl. There is an `ecosystem_specific` specific array `binaries` of the associated RPM binary packages in this specific openSUSE distribution. The ECOSYSTEM version ordering is the RPM versioncompare ordering, and the database uses the `introduced` and `fixed` boundaries. | | `Packagist` | The PHP package manager ecosystem; the `name` is a package name. | | `Photon OS` | The Photon OS package ecosystem; the `name` is the name of the RPM package. The ecosystem string must have a `:` suffix to scope the package to a particular Photon OS release. Eg `Photon OS:3.0`. | | `Pub` | The package manager for the Dart ecosystem; the `name` field is a Dart package name. | | `PyPI` | the Python PyPI ecosystem; the `name` field is a [normalized](https://www.python.org/dev/peps/pep-0503/#normalized-names) PyPI package name. | | `Red Hat` | The Red Hat package ecosystem; the `name` field is the name of a binary or source RPM. The ecosystem string has a `:` suffix to scope the RPM to a specific Red Hat product stream. `` is a translation of a Red Hat [Common Platform Enumerations](https://cpe.mitre.org/) (CPE) with the `cpe/:[oa]:(redhat):` prefix removed (for example, `Red Hat:rhel_aus:8.4::appstream` translates to `cpe:/a:redhat:rhel_aus:8.4::appstream`). Red Hat ecosystem identifiers can be used to identify vulnerable RPMs installed on a Red Hat system as explained [here](https://www.redhat.com/en/blog/how-accurately-match-oval-security-data-installed-rpms). | -| `Rocky Linux` | The Rocky Linux package ecosystem; the `name` is the name of the source package. The ecosystem string might optionally have a `:` suffix to scope the package to a particular Rocky Linux release. `` is a numeric version. +| `Rocky Linux` | The Rocky Linux package ecosystem; the `name` is the name of the source package. The ecosystem string might optionally have a `:` suffix to scope the package to a particular Rocky Linux release. `` is a numeric version. | | `RubyGems` | The RubyGems ecosystem; the `name` field is a gem name. | -| `SUSE` | The SUSE ecosystem; The ecosystem string has a `:` suffix representing the marketing name of the SUSE product. `` matches the value in the /etc/os-release `PRETTY_NAME` field. The `name` field is the name of the source RPM and accompanied by a purl. There is a `ecosystem_specific` specific array `binaries` of the associated RPM binary packages in this specific SUSE product. The ECOSYSTEM version ordering is the RPM versioncompare ordering, and the database uses the `introduced` and `fixed` boundaries.| +| `SUSE` | The SUSE ecosystem; The ecosystem string has a `:` suffix representing the marketing name of the SUSE product. `` matches the value in the /etc/os-release `PRETTY_NAME` field. The `name` field is the name of the source RPM and accompanied by a purl. There is a `ecosystem_specific` specific array `binaries` of the associated RPM binary packages in this specific SUSE product. The ECOSYSTEM version ordering is the RPM versioncompare ordering, and the database uses the `introduced` and `fixed` boundaries. | | `SwiftURL` | The Swift Package Manager ecosystem. The `name` is a Git URL to the source of the package. Versions are Git tags that comform to [SemVer 2.0](https://docs.swift.org/package-manager/PackageDescription/PackageDescription.html#version). | -| `Ubuntu` | The Ubuntu package ecosystem; the `name` field is the name of the source package. The ecosystem string has a `:` suffix to scope the package to a particular Ubuntu release. `` is a numeric ("YY.MM") version as specified in [Ubuntu Releases](https://wiki.ubuntu.com/Releases), with a mandatory `:LTS` suffix if the release is marked as LTS. The release version may also be prefixed with `:Pro:` to denote Ubuntu Pro (aka Expanded Security Maintenance (ESM)) updates. For example, the ecosystem string "Ubuntu:22.04:LTS" refers to Ubuntu 22.04 LTS (jammy), while "Ubuntu:Pro:18.04:LTS" refers to fixes that landed in Ubuntu 18.04 LTS (bionic) under Ubuntu Pro/ESM. +| `Ubuntu` | The Ubuntu package ecosystem; the `name` field is the name of the source package. The ecosystem string has a `:` suffix to scope the package to a particular Ubuntu release. `` is a numeric ("YY.MM") version as specified in [Ubuntu Releases](https://wiki.ubuntu.com/Releases), with a mandatory `:LTS` suffix if the release is marked as LTS. The release version may also be prefixed with `:Pro:` to denote Ubuntu Pro (aka Expanded Security Maintenance (ESM)) updates. For example, the ecosystem string "Ubuntu:22.04:LTS" refers to Ubuntu 22.04 LTS (jammy), while "Ubuntu:Pro:18.04:LTS" refers to fixes that landed in Ubuntu 18.04 LTS (bionic) under Ubuntu Pro/ESM. | | `Wolfi` | The Wolfi package ecosystem; the `name` is the name of the package. | | Your ecosystem here. | [Send us a PR](https://github.com/ossf/osv-schema/compare). | + + It is permitted for a database name (the DB prefix in the `id` field) and an ecosystem name to be the same, provided they have the same owner who can make decisions about the meaning of the `ecosystem_specific` field (see below). diff --git a/scripts/update-ecosystems-lists.py b/scripts/update-ecosystems-lists.py index 26d34db..3544a11 100755 --- a/scripts/update-ecosystems-lists.py +++ b/scripts/update-ecosystems-lists.py @@ -2,6 +2,9 @@ import json +MARKDOWN_TABLE_MARKER_START = '' +MARKDOWN_TABLE_MARKER_END = '' + ecosystems: dict[str, str] = json.loads(open('ecosystems.json').read()) @@ -28,4 +31,45 @@ def update_json_schema(): open('validation/schema.json', 'w').write(json.dumps(schema, indent=2) + '\n') +def generate_ecosystems_markdown_table() -> str: + """ + Generates a Markdown table of supported ecosystems with descriptions + :return: + """ + table = '| Ecosystem | Description |\n' + table += '|-----------|-------------|\n' + + for name, description in ecosystems.items(): + table += f'| `{name}` | {description} |\n' + + return table + + +def update_schema_md(): + """ + Updates the schema.md file with the list of ecosystems + :return: + """ + md = open('docs/schema.md').read() + + table_start_index = md.index(MARKDOWN_TABLE_MARKER_START) + table_end_index = md.index(MARKDOWN_TABLE_MARKER_END) + + assert table_start_index < table_end_index, "Table start index must be before table end index" + + table = generate_ecosystems_markdown_table() + table += '| Your ecosystem here. | [Send us a PR](https://github.com/ossf/osv-schema/compare). |\n' + + md = '{0}{1}\n\n{2}\n{3}{4}'.format( + md[:table_start_index], + MARKDOWN_TABLE_MARKER_START, + table, + MARKDOWN_TABLE_MARKER_END, + md[table_end_index + len(MARKDOWN_TABLE_MARKER_END):] + ) + + open('docs/schema.md', 'w').write(md) + + update_json_schema() +update_schema_md() From 9c1f9224c58718e103add9db9c3514e7dafcda08 Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Tue, 22 Oct 2024 08:00:24 +1300 Subject: [PATCH 5/9] feat: support updating constants in go package Signed-off-by: Gareth Jones --- bindings/go/osvschema/constants.go | 7 ++++ scripts/update-ecosystems-lists.py | 60 ++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/bindings/go/osvschema/constants.go b/bindings/go/osvschema/constants.go index 7088f12..4dfdea9 100644 --- a/bindings/go/osvschema/constants.go +++ b/bindings/go/osvschema/constants.go @@ -8,17 +8,22 @@ const ( EcosystemAndroid Ecosystem = "Android" EcosystemBioconductor Ecosystem = "Bioconductor" EcosystemBitnami Ecosystem = "Bitnami" + EcosystemChainguard Ecosystem = "Chainguard" EcosystemConanCenter Ecosystem = "ConanCenter" EcosystemCRAN Ecosystem = "CRAN" EcosystemCratesIO Ecosystem = "crates.io" EcosystemDebian Ecosystem = "Debian" + EcosystemGHC Ecosystem = "GHC" EcosystemGitHubActions Ecosystem = "GitHub Actions" EcosystemGo Ecosystem = "Go" + EcosystemHackage Ecosystem = "Hackage" EcosystemHex Ecosystem = "Hex" EcosystemLinux Ecosystem = "Linux" + EcosystemMageia Ecosystem = "Mageia" EcosystemMaven Ecosystem = "Maven" EcosystemNPM Ecosystem = "npm" EcosystemNuGet Ecosystem = "NuGet" + EcosystemOpenSUSE Ecosystem = "openSUSE" EcosystemOSSFuzz Ecosystem = "OSS-Fuzz" EcosystemPackagist Ecosystem = "Packagist" EcosystemPhotonOS Ecosystem = "Photon OS" @@ -27,8 +32,10 @@ const ( EcosystemRedHat Ecosystem = "Red Hat" EcosystemRockyLinux Ecosystem = "Rocky Linux" EcosystemRubyGems Ecosystem = "RubyGems" + EcosystemSUSE Ecosystem = "SUSE" EcosystemSwiftURL Ecosystem = "SwiftURL" EcosystemUbuntu Ecosystem = "Ubuntu" + EcosystemWolfi Ecosystem = "Wolfi" ) type SeverityType string diff --git a/scripts/update-ecosystems-lists.py b/scripts/update-ecosystems-lists.py index 3544a11..420d6c3 100755 --- a/scripts/update-ecosystems-lists.py +++ b/scripts/update-ecosystems-lists.py @@ -71,5 +71,65 @@ def update_schema_md(): open('docs/schema.md', 'w').write(md) +def convert_to_go_constant_name(name: str) -> str: + """ + Converts the "human" name of an ecosystem to a Go constant name, mostly + by removing spaces and dashes and converting to PascalCase. + + Some ecosystems have special cases, like "crates.io" which is converted to "CratesIO". + + :param name: + :return: + """ + if name == 'crates.io': + return 'EcosystemCratesIO' + + if name == 'npm': + return 'EcosystemNPM' + + name = name[0].upper() + name[1:] + name = name.replace('-', '').replace(' ', '') + + return f'Ecosystem{name}' + + +def generate_ecosystems_go_constants() -> str: + """ + Generates a list of Go constants, with a constant per ecosystem + :return: + """ + + constants = list(map(lambda x: (convert_to_go_constant_name(x), x), ecosystems.keys())) + longest = max(map(lambda x: len(x[0]), constants)) + + code = 'const (\n' + for constant, name in constants: + code += f'\t{constant.ljust(longest)} Ecosystem = "{name}"\n' + code += ')\n' + + return code + + +def update_go_constants(): + """ + Updates the Go constants with the list of ecosystems + :return: + """ + go = open('bindings/go/osvschema/constants.go').read() + + ecosystem_constants_start = go.index('type Ecosystem string\n\nconst (\n') + ecosystems_constants_end = go.index(')', ecosystem_constants_start) + + go = '{0}{1}\n\n{2}{3}'.format( + go[:ecosystem_constants_start], + 'type Ecosystem string', + generate_ecosystems_go_constants(), + go[ecosystems_constants_end + 2:] + ) + + open('bindings/go/osvschema/constants.go', 'w').write(go) + + +update_go_constants() update_json_schema() update_schema_md() From 66002106fee0bb6afc279d5e8628dc6df2186856 Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Tue, 22 Oct 2024 08:29:31 +1300 Subject: [PATCH 6/9] feat: ensure that ecosystems are sorted and don't have extra whitespace in their descriptions Signed-off-by: Gareth Jones --- docs/schema.md | 30 +++++++++++++++--------------- ecosystems.json | 30 +++++++++++++++--------------- scripts/update-ecosystems-lists.py | 11 ++++++++++- 3 files changed, 40 insertions(+), 31 deletions(-) diff --git a/docs/schema.md b/docs/schema.md index 6cdb8b0..fca0393 100644 --- a/docs/schema.md +++ b/docs/schema.md @@ -692,33 +692,33 @@ The defined ecosystems are: |-----------|-------------| | `AlmaLinux` | AlmaLinux package ecosystem; the `name` is the name of the source package. The ecosystem string might optionally have a `:` suffix to scope the package to a particular AlmaLinux release. `` is a numeric version. | | `Alpine` | The Alpine package ecosystem; the `name` is the name of the source package. The ecosystem string must have a `:v` suffix to scope the package to a particular Alpine release branch (the `v` prefix is required). E.g. `v3.16`. | -| `Android` | The Android ecosystem. Android organizes code using [`repo` tool](https://gerrit.googlesource.com/git-repo/+/HEAD/README.md), which manages multiple git projects under one or more remote git servers, where each project is identified by its name in [repo configuration](https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md#Element-project) (e.g. `platform/frameworks/base`). The `name` field should contain the name of that affected git project/submodule. One exception is when the project contains the Linux kernel source code, in which case `name` field will be `:linux_kernel:`, followed by an optional SoC vendor name e.g. `:linux_kernel:Qualcomm`. The list of recognized SoC vendors is listed in the [Appendix](#android-soc-vendors) | +| `Android` | The Android ecosystem. Android organizes code using [`repo` tool](https://gerrit.googlesource.com/git-repo/+/HEAD/README.md), which manages multiple git projects under one or more remote git servers, where each project is identified by its name in [repo configuration](https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md#Element-project) (e.g. `platform/frameworks/base`). The `name` field should contain the name of that affected git project/submodule. One exception is when the project contains the Linux kernel source code, in which case `name` field will be `:linux_kernel:`, followed by an optional SoC vendor name e.g. `:linux_kernel:Qualcomm`. The list of recognized SoC vendors is listed in the [Appendix](#android-soc-vendors) | | `Bioconductor` | The biological R package ecosystem. The `name` is an R package name. | | `Bitnami` | Bitnami package ecosystem; the `name` is the name of the affected component. | | `Chainguard` | The Chainguard package ecosystem; the `name` is the name of the package. | -| `ConanCenter` | The ConanCenter ecosystem for C and C++; the `name` field is a Conan package name. | +| `ConanCenter` | The ConanCenter ecosystem for C and C++; the `name` field is a Conan package name. | | `CRAN` | The R package ecosystem. The `name` is an R package name. | -| `crates.io` | The crates.io ecosystem for Rust; the `name` field is a crate name. | -| `Debian` | The Debian package ecosystem; the `name` is the name of the source package. The ecosystem string might optionally have a `:` suffix to scope the package to a particular Debian release. `` is a numeric version specified in the [Debian distro-info-data](https://debian.pages.debian.net/distro-info-data/debian.csv). For example, the ecosystem string "Debian:7" refers to the Debian 7 (wheezy) release. | -| `GHC` | The Haskell compiler ecosystem. The `name` field is the name of a component of the GHC compiler ecosystem (e.g., compiler, GHCI, RTS). | +| `crates.io` | The crates.io ecosystem for Rust; the `name` field is a crate name. | +| `Debian` | The Debian package ecosystem; the `name` is the name of the source package. The ecosystem string might optionally have a `:` suffix to scope the package to a particular Debian release. `` is a numeric version specified in the [Debian distro-info-data](https://debian.pages.debian.net/distro-info-data/debian.csv). For example, the ecosystem string "Debian:7" refers to the Debian 7 (wheezy) release. | +| `GHC` | The Haskell compiler ecosystem. The `name` field is the name of a component of the GHC compiler ecosystem (e.g., compiler, GHCI, RTS). | | `GitHub Actions` | The GitHub Actions ecosystem; the `name` field is the action's repository name with owner e.g. `{owner}/{repo}`. | -| `Go` | The Go ecosystem; the `name` field is a Go module path. | -| `Hackage` | The Haskell package ecosystem. The `name` field is a Haskell package name as published on Hackage. | -| `Hex` | The package manager for the Erlang ecosystem; the `name` is a Hex package name. | +| `Go` | The Go ecosystem; the `name` field is a Go module path. | +| `Hackage` | The Haskell package ecosystem. The `name` field is a Haskell package name as published on Hackage. | +| `Hex` | The package manager for the Erlang ecosystem; the `name` is a Hex package name. | | `Linux` | The Linux kernel. The only supported `name` is `Kernel`. | | `Mageia` | The Mageia Linux package ecosystem; the `name` is the name of the source package. The ecosystem string must have a `:` suffix to scope the package to a particular Mageia release. Eg `Mageia:9`. | | `Maven` | The Maven Java package ecosystem. The `name` field is a Maven package name in the format `groupId:artifactId`. The ecosystem string might optionally have a `:` suffix to denote the remote repository URL that best represents the source of truth for this package, without a trailing slash (e.g. `Maven:https://maven.google.com`). If this is omitted, this is assumed to be the Maven Central repository (`https://repo.maven.apache.org/maven2`). | -| `npm` | The NPM ecosystem; the `name` field is an NPM package name. | -| `NuGet` | The NuGet package ecosystem. The `name` field is a NuGet package name. | -| `OSS-Fuzz` | For reports from the OSS-Fuzz project that have no more appropriate ecosystem; the `name` field is the name assigned by the OSS-Fuzz project, as recorded in the submitted fuzzing configuration. | +| `npm` | The NPM ecosystem; the `name` field is an NPM package name. | +| `NuGet` | The NuGet package ecosystem. The `name` field is a NuGet package name. | | `openSUSE` | The openSUSE ecosystem; The ecosystem string has a `:` suffix presenting the marketing name of the openSUSE distribution. `` matches the value in the `/etc/os-release` `PRETTY_NAME` field. The `name` field is the name of the source RPM and accompanied by a purl. There is an `ecosystem_specific` specific array `binaries` of the associated RPM binary packages in this specific openSUSE distribution. The ECOSYSTEM version ordering is the RPM versioncompare ordering, and the database uses the `introduced` and `fixed` boundaries. | -| `Packagist` | The PHP package manager ecosystem; the `name` is a package name. | +| `OSS-Fuzz` | For reports from the OSS-Fuzz project that have no more appropriate ecosystem; the `name` field is the name assigned by the OSS-Fuzz project, as recorded in the submitted fuzzing configuration. | +| `Packagist` | The PHP package manager ecosystem; the `name` is a package name. | | `Photon OS` | The Photon OS package ecosystem; the `name` is the name of the RPM package. The ecosystem string must have a `:` suffix to scope the package to a particular Photon OS release. Eg `Photon OS:3.0`. | | `Pub` | The package manager for the Dart ecosystem; the `name` field is a Dart package name. | -| `PyPI` | the Python PyPI ecosystem; the `name` field is a [normalized](https://www.python.org/dev/peps/pep-0503/#normalized-names) PyPI package name. | -| `Red Hat` | The Red Hat package ecosystem; the `name` field is the name of a binary or source RPM. The ecosystem string has a `:` suffix to scope the RPM to a specific Red Hat product stream. `` is a translation of a Red Hat [Common Platform Enumerations](https://cpe.mitre.org/) (CPE) with the `cpe/:[oa]:(redhat):` prefix removed (for example, `Red Hat:rhel_aus:8.4::appstream` translates to `cpe:/a:redhat:rhel_aus:8.4::appstream`). Red Hat ecosystem identifiers can be used to identify vulnerable RPMs installed on a Red Hat system as explained [here](https://www.redhat.com/en/blog/how-accurately-match-oval-security-data-installed-rpms). | +| `PyPI` | the Python PyPI ecosystem; the `name` field is a [normalized](https://www.python.org/dev/peps/pep-0503/#normalized-names) PyPI package name. | +| `Red Hat` | The Red Hat package ecosystem; the `name` field is the name of a binary or source RPM. The ecosystem string has a `:` suffix to scope the RPM to a specific Red Hat product stream. `` is a translation of a Red Hat [Common Platform Enumerations](https://cpe.mitre.org/) (CPE) with the `cpe/:[oa]:(redhat):` prefix removed (for example, `Red Hat:rhel_aus:8.4::appstream` translates to `cpe:/a:redhat:rhel_aus:8.4::appstream`). Red Hat ecosystem identifiers can be used to identify vulnerable RPMs installed on a Red Hat system as explained [here](https://www.redhat.com/en/blog/how-accurately-match-oval-security-data-installed-rpms). | | `Rocky Linux` | The Rocky Linux package ecosystem; the `name` is the name of the source package. The ecosystem string might optionally have a `:` suffix to scope the package to a particular Rocky Linux release. `` is a numeric version. | -| `RubyGems` | The RubyGems ecosystem; the `name` field is a gem name. | +| `RubyGems` | The RubyGems ecosystem; the `name` field is a gem name. | | `SUSE` | The SUSE ecosystem; The ecosystem string has a `:` suffix representing the marketing name of the SUSE product. `` matches the value in the /etc/os-release `PRETTY_NAME` field. The `name` field is the name of the source RPM and accompanied by a purl. There is a `ecosystem_specific` specific array `binaries` of the associated RPM binary packages in this specific SUSE product. The ECOSYSTEM version ordering is the RPM versioncompare ordering, and the database uses the `introduced` and `fixed` boundaries. | | `SwiftURL` | The Swift Package Manager ecosystem. The `name` is a Git URL to the source of the package. Versions are Git tags that comform to [SemVer 2.0](https://docs.swift.org/package-manager/PackageDescription/PackageDescription.html#version). | | `Ubuntu` | The Ubuntu package ecosystem; the `name` field is the name of the source package. The ecosystem string has a `:` suffix to scope the package to a particular Ubuntu release. `` is a numeric ("YY.MM") version as specified in [Ubuntu Releases](https://wiki.ubuntu.com/Releases), with a mandatory `:LTS` suffix if the release is marked as LTS. The release version may also be prefixed with `:Pro:` to denote Ubuntu Pro (aka Expanded Security Maintenance (ESM)) updates. For example, the ecosystem string "Ubuntu:22.04:LTS" refers to Ubuntu 22.04 LTS (jammy), while "Ubuntu:Pro:18.04:LTS" refers to fixes that landed in Ubuntu 18.04 LTS (bionic) under Ubuntu Pro/ESM. | diff --git a/ecosystems.json b/ecosystems.json index 6f4599f..907fbee 100644 --- a/ecosystems.json +++ b/ecosystems.json @@ -1,33 +1,33 @@ { "AlmaLinux": "AlmaLinux package ecosystem; the `name` is the name of the source package. The ecosystem string might optionally have a `:` suffix to scope the package to a particular AlmaLinux release. `` is a numeric version.", "Alpine": "The Alpine package ecosystem; the `name` is the name of the source package. The ecosystem string must have a `:v` suffix to scope the package to a particular Alpine release branch (the `v` prefix is required). E.g. `v3.16`.", - "Android": " The Android ecosystem. Android organizes code using [`repo` tool](https://gerrit.googlesource.com/git-repo/+/HEAD/README.md), which manages multiple git projects under one or more remote git servers, where each project is identified by its name in [repo configuration](https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md#Element-project) (e.g. `platform/frameworks/base`). The `name` field should contain the name of that affected git project/submodule. One exception is when the project contains the Linux kernel source code, in which case `name` field will be `:linux_kernel:`, followed by an optional SoC vendor name e.g. `:linux_kernel:Qualcomm`. The list of recognized SoC vendors is listed in the [Appendix](#android-soc-vendors)", + "Android": "The Android ecosystem. Android organizes code using [`repo` tool](https://gerrit.googlesource.com/git-repo/+/HEAD/README.md), which manages multiple git projects under one or more remote git servers, where each project is identified by its name in [repo configuration](https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md#Element-project) (e.g. `platform/frameworks/base`). The `name` field should contain the name of that affected git project/submodule. One exception is when the project contains the Linux kernel source code, in which case `name` field will be `:linux_kernel:`, followed by an optional SoC vendor name e.g. `:linux_kernel:Qualcomm`. The list of recognized SoC vendors is listed in the [Appendix](#android-soc-vendors)", "Bioconductor": "The biological R package ecosystem. The `name` is an R package name.", "Bitnami": "Bitnami package ecosystem; the `name` is the name of the affected component.", "Chainguard": "The Chainguard package ecosystem; the `name` is the name of the package.", - "ConanCenter": "The ConanCenter ecosystem for C and C++; the `name` field is a Conan package name. ", + "ConanCenter": "The ConanCenter ecosystem for C and C++; the `name` field is a Conan package name.", "CRAN": "The R package ecosystem. The `name` is an R package name.", - "crates.io": "The crates.io ecosystem for Rust; the `name` field is a crate name. ", - "Debian": "The Debian package ecosystem; the `name` is the name of the source package. The ecosystem string might optionally have a `:` suffix to scope the package to a particular Debian release. `` is a numeric version specified in the [Debian distro-info-data](https://debian.pages.debian.net/distro-info-data/debian.csv). For example, the ecosystem string \"Debian:7\" refers to the Debian 7 (wheezy) release. ", - "GHC": "The Haskell compiler ecosystem. The `name` field is the name of a component of the GHC compiler ecosystem (e.g., compiler, GHCI, RTS). ", + "crates.io": "The crates.io ecosystem for Rust; the `name` field is a crate name.", + "Debian": "The Debian package ecosystem; the `name` is the name of the source package. The ecosystem string might optionally have a `:` suffix to scope the package to a particular Debian release. `` is a numeric version specified in the [Debian distro-info-data](https://debian.pages.debian.net/distro-info-data/debian.csv). For example, the ecosystem string \"Debian:7\" refers to the Debian 7 (wheezy) release.", + "GHC": "The Haskell compiler ecosystem. The `name` field is the name of a component of the GHC compiler ecosystem (e.g., compiler, GHCI, RTS).", "GitHub Actions": "The GitHub Actions ecosystem; the `name` field is the action's repository name with owner e.g. `{owner}/{repo}`.", - "Go": "The Go ecosystem; the `name` field is a Go module path. ", - "Hackage": "The Haskell package ecosystem. The `name` field is a Haskell package name as published on Hackage. ", - "Hex": "The package manager for the Erlang ecosystem; the `name` is a Hex package name. ", + "Go": "The Go ecosystem; the `name` field is a Go module path.", + "Hackage": "The Haskell package ecosystem. The `name` field is a Haskell package name as published on Hackage.", + "Hex": "The package manager for the Erlang ecosystem; the `name` is a Hex package name.", "Linux": "The Linux kernel. The only supported `name` is `Kernel`.", "Mageia": "The Mageia Linux package ecosystem; the `name` is the name of the source package. The ecosystem string must have a `:` suffix to scope the package to a particular Mageia release. Eg `Mageia:9`.", "Maven": "The Maven Java package ecosystem. The `name` field is a Maven package name in the format `groupId:artifactId`. The ecosystem string might optionally have a `:` suffix to denote the remote repository URL that best represents the source of truth for this package, without a trailing slash (e.g. `Maven:https://maven.google.com`). If this is omitted, this is assumed to be the Maven Central repository (`https://repo.maven.apache.org/maven2`).", - "npm": "The NPM ecosystem; the `name` field is an NPM package name. ", - "NuGet": "The NuGet package ecosystem. The `name` field is a NuGet package name. ", - "OSS-Fuzz": "For reports from the OSS-Fuzz project that have no more appropriate ecosystem; the `name` field is the name assigned by the OSS-Fuzz project, as recorded in the submitted fuzzing configuration. ", + "npm": "The NPM ecosystem; the `name` field is an NPM package name.", + "NuGet": "The NuGet package ecosystem. The `name` field is a NuGet package name.", "openSUSE": "The openSUSE ecosystem; The ecosystem string has a `:` suffix presenting the marketing name of the openSUSE distribution. `` matches the value in the `/etc/os-release` `PRETTY_NAME` field. The `name` field is the name of the source RPM and accompanied by a purl. There is an `ecosystem_specific` specific array `binaries` of the associated RPM binary packages in this specific openSUSE distribution. The ECOSYSTEM version ordering is the RPM versioncompare ordering, and the database uses the `introduced` and `fixed` boundaries.", - "Packagist": "The PHP package manager ecosystem; the `name` is a package name. ", + "OSS-Fuzz": "For reports from the OSS-Fuzz project that have no more appropriate ecosystem; the `name` field is the name assigned by the OSS-Fuzz project, as recorded in the submitted fuzzing configuration.", + "Packagist": "The PHP package manager ecosystem; the `name` is a package name.", "Photon OS": "The Photon OS package ecosystem; the `name` is the name of the RPM package. The ecosystem string must have a `:` suffix to scope the package to a particular Photon OS release. Eg `Photon OS:3.0`.", "Pub": "The package manager for the Dart ecosystem; the `name` field is a Dart package name.", - "PyPI": "the Python PyPI ecosystem; the `name` field is a [normalized](https://www.python.org/dev/peps/pep-0503/#normalized-names) PyPI package name. ", - "Red Hat": "The Red Hat package ecosystem; the `name` field is the name of a binary or source RPM. The ecosystem string has a `:` suffix to scope the RPM to a specific Red Hat product stream. `` is a translation of a Red Hat [Common Platform Enumerations](https://cpe.mitre.org/) (CPE) with the `cpe/:[oa]:(redhat):` prefix removed (for example, `Red Hat:rhel_aus:8.4::appstream` translates to `cpe:/a:redhat:rhel_aus:8.4::appstream`). Red Hat ecosystem identifiers can be used to identify vulnerable RPMs installed on a Red Hat system as explained [here](https://www.redhat.com/en/blog/how-accurately-match-oval-security-data-installed-rpms). ", + "PyPI": "the Python PyPI ecosystem; the `name` field is a [normalized](https://www.python.org/dev/peps/pep-0503/#normalized-names) PyPI package name.", + "Red Hat": "The Red Hat package ecosystem; the `name` field is the name of a binary or source RPM. The ecosystem string has a `:` suffix to scope the RPM to a specific Red Hat product stream. `` is a translation of a Red Hat [Common Platform Enumerations](https://cpe.mitre.org/) (CPE) with the `cpe/:[oa]:(redhat):` prefix removed (for example, `Red Hat:rhel_aus:8.4::appstream` translates to `cpe:/a:redhat:rhel_aus:8.4::appstream`). Red Hat ecosystem identifiers can be used to identify vulnerable RPMs installed on a Red Hat system as explained [here](https://www.redhat.com/en/blog/how-accurately-match-oval-security-data-installed-rpms).", "Rocky Linux": "The Rocky Linux package ecosystem; the `name` is the name of the source package. The ecosystem string might optionally have a `:` suffix to scope the package to a particular Rocky Linux release. `` is a numeric version.", - "RubyGems": "The RubyGems ecosystem; the `name` field is a gem name. ", + "RubyGems": "The RubyGems ecosystem; the `name` field is a gem name.", "SUSE": "The SUSE ecosystem; The ecosystem string has a `:` suffix representing the marketing name of the SUSE product. `` matches the value in the /etc/os-release `PRETTY_NAME` field. The `name` field is the name of the source RPM and accompanied by a purl. There is a `ecosystem_specific` specific array `binaries` of the associated RPM binary packages in this specific SUSE product. The ECOSYSTEM version ordering is the RPM versioncompare ordering, and the database uses the `introduced` and `fixed` boundaries.", "SwiftURL": "The Swift Package Manager ecosystem. The `name` is a Git URL to the source of the package. Versions are Git tags that comform to [SemVer 2.0](https://docs.swift.org/package-manager/PackageDescription/PackageDescription.html#version).", "Ubuntu": "The Ubuntu package ecosystem; the `name` field is the name of the source package. The ecosystem string has a `:` suffix to scope the package to a particular Ubuntu release. `` is a numeric (\"YY.MM\") version as specified in [Ubuntu Releases](https://wiki.ubuntu.com/Releases), with a mandatory `:LTS` suffix if the release is marked as LTS. The release version may also be prefixed with `:Pro:` to denote Ubuntu Pro (aka Expanded Security Maintenance (ESM)) updates. For example, the ecosystem string \"Ubuntu:22.04:LTS\" refers to Ubuntu 22.04 LTS (jammy), while \"Ubuntu:Pro:18.04:LTS\" refers to fixes that landed in Ubuntu 18.04 LTS (bionic) under Ubuntu Pro/ESM.", diff --git a/scripts/update-ecosystems-lists.py b/scripts/update-ecosystems-lists.py index 420d6c3..da63814 100755 --- a/scripts/update-ecosystems-lists.py +++ b/scripts/update-ecosystems-lists.py @@ -5,7 +5,16 @@ MARKDOWN_TABLE_MARKER_START = '' MARKDOWN_TABLE_MARKER_END = '' -ecosystems: dict[str, str] = json.loads(open('ecosystems.json').read()) +# ensure that the ecosystems are sorted alphabetically and don't have extra whitespace +ecosystems: dict[str, str] = { + k: v.strip() for k, v in sorted( + json.loads(open('ecosystems.json').read()).items(), + key=lambda item: item[0].casefold() + ) +} + +# write back to the json file in case there were any changes +open('ecosystems.json', 'w').write(json.dumps(ecosystems, indent=2) + '\n') def update_json_schema(): From d2a227b009e09a47b294da73f0a996dff74844ec Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Tue, 22 Oct 2024 08:34:44 +1300 Subject: [PATCH 7/9] ci: add job to ensure that ecosystem lists are all in sync Signed-off-by: Gareth Jones --- .github/workflows/checks.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index e46d232..8d9954b 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -57,3 +57,22 @@ jobs: check-latest: true - run: go test ./... + + ecosystem_lists: + permissions: + contents: read # to fetch code (actions/checkout) + name: Check ecosystem lists + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 + with: + persist-credentials: false + - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 + with: + python-version: 3.13 + - run: python3 ./scripts/update-ecosystems-lists.py + - run: | + git diff --name-only \ + | xargs -I '{}' bash -c \ + 'echo "::error file={}::This needs to be regenerated by running \`python3 ./scripts/update-ecosystems-lists.py\`" && false' From babe4cac8f448f2480f63528b3ab6c01f1a84439 Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Tue, 5 Nov 2024 08:07:01 +1300 Subject: [PATCH 8/9] docs: add comment explaining how to regenerate constants and table Signed-off-by: Gareth Jones --- docs/schema.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/schema.md b/docs/schema.md index fca0393..705a59e 100644 --- a/docs/schema.md +++ b/docs/schema.md @@ -668,7 +668,7 @@ within its ecosystem. The two fields must both be present, because the The `purl` field is a string following the [Package URL specification](https://github.com/package-url/purl-spec) that identifies the package, without the `@version` component. -This field is optional but recommended. +This field is optional but recommended. Different ecosystems can define the same names; they identify different packages. For example, these denote different libraries with different sets of @@ -686,6 +686,7 @@ versions and different potential vulnerabilities: The defined ecosystems are: + | Ecosystem | Description | From 53637f4ad783c80491e030c8fffe1f759233bd36 Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Tue, 5 Nov 2024 15:04:29 +1300 Subject: [PATCH 9/9] refactor: be consistent in quotes Signed-off-by: Gareth Jones --- scripts/update-ecosystems-lists.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/update-ecosystems-lists.py b/scripts/update-ecosystems-lists.py index da63814..7320cae 100755 --- a/scripts/update-ecosystems-lists.py +++ b/scripts/update-ecosystems-lists.py @@ -25,14 +25,14 @@ def update_json_schema(): schema = json.loads(open('validation/schema.json').read()) names = ecosystems.keys() - pattern = "" + pattern = '' for name in names: - pattern += name.replace(".", "\\.") - pattern += "|" + pattern += name.replace('.', '\\.') + pattern += '|' # this is a special "ecosystem" name - pattern += "GIT" + pattern += 'GIT' schema['$defs']['ecosystemName']['enum'] = list(names) schema['$defs']['ecosystemWithSuffix']['pattern'] = f'^({pattern})(:.+)?$' @@ -64,7 +64,7 @@ def update_schema_md(): table_start_index = md.index(MARKDOWN_TABLE_MARKER_START) table_end_index = md.index(MARKDOWN_TABLE_MARKER_END) - assert table_start_index < table_end_index, "Table start index must be before table end index" + assert table_start_index < table_end_index, 'Table start index must be before table end index' table = generate_ecosystems_markdown_table() table += '| Your ecosystem here. | [Send us a PR](https://github.com/ossf/osv-schema/compare). |\n'