-
Notifications
You must be signed in to change notification settings - Fork 598
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
add cataloger for rust crates from Cargo.lock files #345
Changes from 1 commit
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,18 @@ | ||
package rust | ||
|
||
import "github.com/anchore/syft/syft/pkg" | ||
|
||
type CargoMetadata struct { | ||
Packages []CargoMetadataPackage `toml:"package"` | ||
} | ||
|
||
// Pkgs returns all of the packages referenced within the Cargo.lock metadata. | ||
func (m CargoMetadata) Pkgs() []pkg.Package { | ||
pkgs := make([]pkg.Package, 0) | ||
|
||
for _, p := range m.Packages { | ||
pkgs = append(pkgs, p.Pkg()) | ||
} | ||
|
||
return pkgs | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,21 @@ | ||
package rust | ||
|
||
import "github.com/anchore/syft/syft/pkg" | ||
|
||
type CargoMetadataPackage struct { | ||
Name string `toml:"name"` | ||
Version string `toml:"version"` | ||
Source string `toml:"source"` | ||
Checksum string `toml:"checksum"` | ||
Dependencies []string `toml:"dependencies"` | ||
} | ||
|
||
// Pkg returns the standard `pkg.Package` representation of the package referenced within the Cargo.lock metadata. | ||
func (p CargoMetadataPackage) Pkg() pkg.Package { | ||
return pkg.Package{ | ||
Name: p.Name, | ||
Version: p.Version, | ||
Language: pkg.Rust, | ||
Type: pkg.RustPkg, | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,17 @@ | ||
/* | ||
Package rust provides a concrete Cataloger implementation for Cargo.lock files. | ||
*/ | ||
package rust | ||
|
||
import ( | ||
"github.com/anchore/syft/syft/cataloger/common" | ||
) | ||
|
||
// NewCargoLockCataloger returns a new Rust Cargo lock file cataloger object. | ||
func NewCargoLockCataloger() *common.GenericCataloger { | ||
globParsers := map[string]common.ParserFn{ | ||
"**/Cargo.lock": parseCargoLock, | ||
} | ||
|
||
return common.NewGenericCataloger(nil, globParsers, "rust-cataloger") | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,29 @@ | ||
package rust | ||
|
||
import ( | ||
"fmt" | ||
"io" | ||
|
||
"github.com/anchore/syft/syft/cataloger/common" | ||
"github.com/anchore/syft/syft/pkg" | ||
"github.com/pelletier/go-toml" | ||
) | ||
|
||
// integrity check | ||
var _ common.ParserFn = parseCargoLock | ||
|
||
// parseCargoLock is a parser function for Cargo.lock contents, returning all rust cargo crates discovered. | ||
func parseCargoLock(_ string, reader io.Reader) ([]pkg.Package, error) { | ||
tree, err := toml.LoadReader(reader) | ||
if err != nil { | ||
return nil, fmt.Errorf("unable to load Cargo.lock for parsing: %v", err) | ||
} | ||
|
||
metadata := CargoMetadata{} | ||
err = tree.Unmarshal(&metadata) | ||
if err != nil { | ||
return nil, fmt.Errorf("unable to parse Cargo.lock: %v", err) | ||
} | ||
|
||
return metadata.Pkgs(), nil | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,99 @@ | ||
package rust | ||
|
||
import ( | ||
"os" | ||
"testing" | ||
|
||
"github.com/anchore/syft/syft/pkg" | ||
"github.com/go-test/deep" | ||
) | ||
|
||
func TestParseCargoLock(t *testing.T) { | ||
expected := []pkg.Package{ | ||
{ | ||
Name: "ansi_term", | ||
Version: "0.12.1", | ||
Language: pkg.Rust, | ||
Type: pkg.RustPkg, | ||
Licenses: nil, | ||
}, | ||
{ | ||
Name: "matches", | ||
Version: "0.1.8", | ||
Language: pkg.Rust, | ||
Type: pkg.RustPkg, | ||
Licenses: nil, | ||
}, | ||
{ | ||
Name: "memchr", | ||
Version: "2.3.3", | ||
Language: pkg.Rust, | ||
Type: pkg.RustPkg, | ||
Licenses: nil, | ||
}, | ||
{ | ||
Name: "natord", | ||
Version: "1.0.9", | ||
Language: pkg.Rust, | ||
Type: pkg.RustPkg, | ||
Licenses: nil, | ||
}, | ||
{ | ||
Name: "nom", | ||
Version: "4.2.3", | ||
Language: pkg.Rust, | ||
Type: pkg.RustPkg, | ||
Licenses: nil, | ||
}, | ||
{ | ||
Name: "unicode-bidi", | ||
Version: "0.3.4", | ||
Language: pkg.Rust, | ||
Type: pkg.RustPkg, | ||
Licenses: nil, | ||
}, | ||
{ | ||
Name: "version_check", | ||
Version: "0.1.5", | ||
Language: pkg.Rust, | ||
Type: pkg.RustPkg, | ||
Licenses: nil, | ||
}, | ||
{ | ||
Name: "winapi", | ||
Version: "0.3.9", | ||
Language: pkg.Rust, | ||
Type: pkg.RustPkg, | ||
Licenses: nil, | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. is licenses not a thing with Rust packages? if it is, it would be useful to capture that in these tests There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. No, there isn't any information about licenses available in the Cargo lock file, at least not that I'm aware of There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This is an interesting question! I'm very new to Rust 😃 . From what I can tell, @westonsteimel is right about licenses not being stored in cargo.lock files. It looks like they are stored in cargo.toml files (akin to a There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yes, you can also use the cargo metadata command to output json with everything about all crates and dependencies in the workspace, but that would require executing a command |
||
}, | ||
{ | ||
Name: "winapi-i686-pc-windows-gnu", | ||
Version: "0.4.0", | ||
Language: pkg.Rust, | ||
Type: pkg.RustPkg, | ||
Licenses: nil, | ||
}, | ||
{ | ||
Name: "winapi-x86_64-pc-windows-gnu", | ||
Version: "0.4.0", | ||
Language: pkg.Rust, | ||
Type: pkg.RustPkg, | ||
Licenses: nil, | ||
}, | ||
} | ||
|
||
fixture, err := os.Open("test-fixtures/Cargo.lock") | ||
if err != nil { | ||
t.Fatalf("failed to open fixture: %+v", err) | ||
} | ||
|
||
actual, err := parseCargoLock(fixture.Name(), fixture) | ||
if err != nil { | ||
t.Error(err) | ||
} | ||
|
||
differences := deep.Equal(expected, actual) | ||
if differences != nil { | ||
t.Errorf("returned package list differed from expectation: %+v", differences) | ||
} | ||
} |
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This would be great raw information to capture onto
Package.Metadata
(and hint the type via the newMetadataType
you added ontoPackage.MetadataType
).This also implies that we should migrate the struct to the
syft/pkg
package and rename the struct toCargoPackageMetadata
to mirror the naming scheme for other metadata structs in thesyft/pkg
package.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Okay, I'll try to look at that a bit over the weekend. I was a bit unsure on this as I think I was mostly looking at the go modules one as an example (and the python poetry file parser for the toml bits) and it didn't have a metadata component defined under pkg.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
no problem! Taking a look at the poetry metadata we could have done the same thing (moved it up to
pkg/
and expose it viaPackage.Metadata
) but it seems like we didn't (I admit we're being somewhat inconsistent here).We can also merge this PR as is without capturing the extra cargo metadata onto
Package.Metadata
--we can always do that in a follow up PR if we find we really need it. Let us know which direction you want to go in, either is OK 👍There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
So I had a try at this in another branch westonsteimel@8b620de, but I'm running into issues getting the tests to pass. First the unit tests passed, but the integration tests did not, and I was able to figure out that I needed to increment and generate a new json schema file (1.0.3). I did that and the integration tests now pass, but the
TestJsonDirsPresenter
andTestJsonImgsPresenter
unit tests fail because they expect json schema file version 1.0.2 and now get 1.0.3, and I haven't yet figured out how to resolve that one.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
awesome! I cherry picked that commit over and made a couple minor adjustments:
go test ./syft/presenters/json -update
and then manually ensure the snapshots are what we expect before committing (I did this already, so no need to update)MetadataType
to reflect where the information was parsed from (a cargo package section)