Skip to content

Commit

Permalink
featurefmt: Extract source packages and binary packages
Browse files Browse the repository at this point in the history 
The featurefmt now extracts both binary packages and source packages
from the package manager infos.
  • Loading branch information
@KeyboardNerd
KeyboardNerd committed Feb 19, 2019
1 parent 7dd989c commit 0e0d8b3
Show file tree
Hide file tree
Showing 6 changed files with 608 additions and 380 deletions.
2 changes: 2 additions & 0 deletions ext/featurefmt/apk/apk.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ func (l lister) ListFeatures(files tarutil.FilesMap) ([]database.Feature, error)
line := scanner.Text()
if len(line) < 2 {
if valid(&pkg) {
pkg.Type = database.BinaryPackage
packages.Add(pkg)
pkg = database.Feature{VersionFormat: dpkg.ParserName}
}
Expand All @@ -81,6 +82,7 @@ func (l lister) ListFeatures(files tarutil.FilesMap) ([]database.Feature, error)

// in case of no terminal line
if valid(&pkg) {
pkg.Type = database.BinaryPackage
packages.Add(pkg)
}

Expand Down
22 changes: 11 additions & 11 deletions ext/featurefmt/apk/apk_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,17 +28,17 @@ func TestAPKFeatureDetection(t *testing.T) {
"valid case",
map[string]string{"lib/apk/db/installed": "apk/testdata/valid"},
[]database.Feature{
{"musl", "1.1.14-r10", "", "", dpkg.ParserName},
{"busybox", "1.24.2-r9", "", "", dpkg.ParserName},
{"alpine-baselayout", "3.0.3-r0", "", "", dpkg.ParserName},
{"alpine-keys", "1.1-r0", "", "", dpkg.ParserName},
{"zlib", "1.2.8-r2", "", "", dpkg.ParserName},
{"libcrypto1.0", "1.0.2h-r1", "", "", dpkg.ParserName},
{"libssl1.0", "1.0.2h-r1", "", "", dpkg.ParserName},
{"apk-tools", "2.6.7-r0", "", "", dpkg.ParserName},
{"scanelf", "1.1.6-r0", "", "", dpkg.ParserName},
{"musl-utils", "1.1.14-r10", "", "", dpkg.ParserName},
{"libc-utils", "0.7-r0", "", "", dpkg.ParserName},
{"apk-tools", "2.6.7-r0", "dpkg", "binary"},
{"musl", "1.1.14-r10", "dpkg", "binary"},
{"libssl1.0", "1.0.2h-r1", "dpkg", "binary"},
{"libc-utils", "0.7-r0", "dpkg", "binary"},
{"busybox", "1.24.2-r9", "dpkg", "binary"},
{"scanelf", "1.1.6-r0", "dpkg", "binary"},
{"alpine-keys", "1.1-r0", "dpkg", "binary"},
{"libcrypto1.0", "1.0.2h-r1", "dpkg", "binary"},
{"zlib", "1.2.8-r2", "dpkg", "binary"},
{"musl-utils", "1.1.14-r10", "dpkg", "binary"},
{"alpine-baselayout", "3.0.3-r0", "dpkg", "binary"},
},
},
} {
Expand Down
121 changes: 74 additions & 47 deletions ext/featurefmt/dpkg/dpkg.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,22 +37,12 @@ var (

type lister struct{}

func init() {
featurefmt.RegisterLister("dpkg", "1.0", &lister{})
}

func valid(pkg *database.Feature) bool {
return pkg.Name != "" && pkg.Version != ""
func (l lister) RequiredFilenames() []string {
return []string{"var/lib/dpkg/status"}
}

func addSourcePackage(pkg *database.Feature) {
if pkg.SourceName == "" {
pkg.SourceName = pkg.Name
}

if pkg.SourceVersion == "" {
pkg.SourceVersion = pkg.Version
}
func init() {
featurefmt.RegisterLister("dpkg", "1.0", &lister{})
}

func (l lister) ListFeatures(files tarutil.FilesMap) ([]database.Feature, error) {
Expand All @@ -61,21 +51,45 @@ func (l lister) ListFeatures(files tarutil.FilesMap) ([]database.Feature, error)
return []database.Feature{}, nil
}

packages := mapset.NewSet()
scanner := bufio.NewScanner(strings.NewReader(string(f)))
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" {
continue
}

binary, source := parseDpkgDB(scanner)
if binary != nil {
packages.Add(*binary)
}

if source != nil {
packages.Add(*source)
}
}

return database.ConvertFeatureSetToFeatures(packages), nil
}

// parseDpkgDB consumes the status file scanner exactly one package info, until
// EOF or empty space, and generate the parsed packages from it.
func parseDpkgDB(scanner *bufio.Scanner) (binaryPackage *database.Feature, sourcePackage *database.Feature) {
var (
pkg = database.Feature{VersionFormat: dpkg.ParserName}
pkgs = mapset.NewSet()
err error
name string
version string
sourceName string
sourceVersion string
)

scanner := bufio.NewScanner(strings.NewReader(string(f)))
for scanner.Scan() {
line := scanner.Text()
if strings.HasPrefix(line, "Package: ") {
// Package line
// Defines the name of the package
for {
line := strings.TrimSpace(scanner.Text())
if line == "" {
break
}

pkg.Name = strings.TrimSpace(strings.TrimPrefix(line, "Package: "))
pkg.Version = ""
if strings.HasPrefix(line, "Package: ") {
name = strings.TrimSpace(strings.TrimPrefix(line, "Package: "))
} else if strings.HasPrefix(line, "Source: ") {
// Source line (Optional)
// Gives the name of the source package
Expand All @@ -87,40 +101,53 @@ func (l lister) ListFeatures(files tarutil.FilesMap) ([]database.Feature, error)
md[dpkgSrcCaptureRegexpNames[i]] = strings.TrimSpace(n)
}

pkg.SourceName = md["name"]
sourceName = md["name"]
if md["version"] != "" {
version := md["version"]
if err = versionfmt.Valid(dpkg.ParserName, version); err != nil {
log.WithError(err).WithField("version", string(line[1])).Warning("could not parse package version. skipping")
} else {
pkg.SourceVersion = version
}
sourceVersion = md["version"]
}
} else if strings.HasPrefix(line, "Version: ") {
// Version line
// Defines the version of the package
// This version is less important than a version retrieved from a Source line
// because the Debian vulnerabilities often skips the epoch from the Version field
// which is not present in the Source version, and because +bX revisions don't matter
version := strings.TrimPrefix(line, "Version: ")
if err = versionfmt.Valid(dpkg.ParserName, version); err != nil {
log.WithError(err).WithField("version", string(line[1])).Warning("could not parse package version. skipping")
} else {
pkg.Version = version
}
} else if line == "" {
pkg = database.Feature{VersionFormat: dpkg.ParserName}
version = strings.TrimPrefix(line, "Version: ")
}

if valid(&pkg) {
addSourcePackage(&pkg)
pkgs.Add(pkg)
if !scanner.Scan() {
break
}
}

return database.ConvertFeatureSetToFeatures(pkgs), nil
}
if name != "" && version != "" {
if err := versionfmt.Valid(dpkg.ParserName, version); err != nil {
log.WithError(err).WithFields(log.Fields{"name": name, "version": version}).Warning("skipped unparseable package")
} else {
binaryPackage = &database.Feature{name, version, dpkg.ParserName, database.BinaryPackage}
}
}

func (l lister) RequiredFilenames() []string {
return []string{"var/lib/dpkg/status"}
// Source version and names are computed from binary package names and versions
// in dpkg.
// Source package name:
// https://git.dpkg.org/cgit/dpkg/dpkg.git/tree/lib/dpkg/pkg-format.c#n338
// Source package version:
// https://git.dpkg.org/cgit/dpkg/dpkg.git/tree/lib/dpkg/pkg-format.c#n355
if sourceName == "" {
sourceName = name
}

if sourceVersion == "" {
sourceVersion = version
}

if sourceName != "" && sourceVersion != "" {
if err := versionfmt.Valid(dpkg.ParserName, version); err != nil {
log.WithError(err).WithFields(log.Fields{"name": name, "version": version}).Warning("skipped unparseable package")
} else {
sourcePackage = &database.Feature{sourceName, sourceVersion, dpkg.ParserName, database.SourcePackage}
}
}

return
}
Loading

0 comments on commit 0e0d8b3

Please sign in to comment.