diff --git a/cmd/guacingest/cmd/ingest.go b/cmd/guacingest/cmd/ingest.go index 6b4ac02183..5802d46946 100644 --- a/cmd/guacingest/cmd/ingest.go +++ b/cmd/guacingest/cmd/ingest.go @@ -48,6 +48,7 @@ type options struct { queryVulnOnIngestion bool queryLicenseOnIngestion bool queryEOLOnIngestion bool + queryDepsDevOnIngestion bool } func ingest(cmd *cobra.Command, args []string) { @@ -110,6 +111,7 @@ func ingest(cmd *cobra.Command, args []string) { opts.queryVulnOnIngestion, opts.queryLicenseOnIngestion, opts.queryEOLOnIngestion, + opts.queryDepsDevOnIngestion, ); err != nil { var urlErr *url.Error if errors.As(err, &urlErr) { diff --git a/cmd/guacone/cmd/deps_dev.go b/cmd/guacone/cmd/deps_dev.go index 60b7899fd1..f1da0370ee 100644 --- a/cmd/guacone/cmd/deps_dev.go +++ b/cmd/guacone/cmd/deps_dev.go @@ -97,6 +97,9 @@ var depsDevCmd = &cobra.Command{ opts.queryVulnOnIngestion, opts.queryLicenseOnIngestion, opts.queryEOLOnIngestion, + // since this is a deps.dev collector, by we don't query deps.dev on ingestion + /* queryDepsDevOnIngestion = */ + false, ); err != nil { gotErr = true return fmt.Errorf("unable to ingest document: %w", err) diff --git a/cmd/guacone/cmd/eol.go b/cmd/guacone/cmd/eol.go index 5b782889be..4dbc5631bf 100644 --- a/cmd/guacone/cmd/eol.go +++ b/cmd/guacone/cmd/eol.go @@ -115,7 +115,7 @@ var eolCmd = &cobra.Command{ select { case <-ticker.C: if len(totalDocs) > 0 { - err = ingestor.MergedIngest(ctx, totalDocs, opts.graphqlEndpoint, transport, csubClient, false, false, false) + err = ingestor.MergedIngest(ctx, totalDocs, opts.graphqlEndpoint, transport, csubClient, false, false, false, false) if err != nil { stop = true atomic.StoreInt32(&gotErr, 1) @@ -128,7 +128,7 @@ var eolCmd = &cobra.Command{ totalNum += 1 totalDocs = append(totalDocs, d) if len(totalDocs) >= threshold { - err = ingestor.MergedIngest(ctx, totalDocs, opts.graphqlEndpoint, transport, csubClient, false, false, false) + err = ingestor.MergedIngest(ctx, totalDocs, opts.graphqlEndpoint, transport, csubClient, false, false, false, false) if err != nil { stop = true atomic.StoreInt32(&gotErr, 1) @@ -147,7 +147,7 @@ var eolCmd = &cobra.Command{ totalNum += 1 totalDocs = append(totalDocs, <-docChan) if len(totalDocs) >= threshold { - err = ingestor.MergedIngest(ctx, totalDocs, opts.graphqlEndpoint, transport, csubClient, false, false, false) + err = ingestor.MergedIngest(ctx, totalDocs, opts.graphqlEndpoint, transport, csubClient, false, false, false, false) if err != nil { atomic.StoreInt32(&gotErr, 1) logger.Errorf("unable to ingest documents: %v", err) @@ -156,7 +156,7 @@ var eolCmd = &cobra.Command{ } } if len(totalDocs) > 0 { - err = ingestor.MergedIngest(ctx, totalDocs, opts.graphqlEndpoint, transport, csubClient, false, false, false) + err = ingestor.MergedIngest(ctx, totalDocs, opts.graphqlEndpoint, transport, csubClient, false, false, false, false) if err != nil { atomic.StoreInt32(&gotErr, 1) logger.Errorf("unable to ingest documents: %v", err) diff --git a/cmd/guacone/cmd/files.go b/cmd/guacone/cmd/files.go index 5608261bdf..0d09b20ca2 100644 --- a/cmd/guacone/cmd/files.go +++ b/cmd/guacone/cmd/files.go @@ -54,6 +54,7 @@ type fileOptions struct { queryVulnOnIngestion bool queryLicenseOnIngestion bool queryEOLOnIngestion bool + queryDepsDevOnIngestion bool } var filesCmd = &cobra.Command{ @@ -71,6 +72,7 @@ var filesCmd = &cobra.Command{ viper.GetBool("add-vuln-on-ingest"), viper.GetBool("add-license-on-ingest"), viper.GetBool("add-eol-on-ingest"), + viper.GetBool("add-depsdev-on-ingest"), args) if err != nil { fmt.Printf("unable to validate flags: %v\n", err) @@ -140,6 +142,7 @@ var filesCmd = &cobra.Command{ opts.queryVulnOnIngestion, opts.queryLicenseOnIngestion, opts.queryEOLOnIngestion, + opts.queryDepsDevOnIngestion, ); err != nil { gotErr = true filesWithErrors = append(filesWithErrors, d.SourceInformation.Source) @@ -173,7 +176,7 @@ var filesCmd = &cobra.Command{ } func validateFilesFlags(keyPath, keyID, graphqlEndpoint, headerFile, csubAddr string, csubTls, csubTlsSkipVerify bool, - queryVulnIngestion bool, queryLicenseIngestion bool, queryEOLIngestion bool, args []string) (fileOptions, error) { + queryVulnIngestion bool, queryLicenseIngestion bool, queryEOLIngestion bool, queryDepsDevOnIngestion bool, args []string) (fileOptions, error) { var opts fileOptions opts.graphqlEndpoint = graphqlEndpoint opts.headerFile = headerFile @@ -202,6 +205,7 @@ func validateFilesFlags(keyPath, keyID, graphqlEndpoint, headerFile, csubAddr st opts.queryVulnOnIngestion = queryVulnIngestion opts.queryLicenseOnIngestion = queryLicenseIngestion opts.queryEOLOnIngestion = queryEOLIngestion + opts.queryDepsDevOnIngestion = queryDepsDevOnIngestion return opts, nil } diff --git a/cmd/guacone/cmd/gcs.go b/cmd/guacone/cmd/gcs.go index ae07390a69..2d49c7d992 100644 --- a/cmd/guacone/cmd/gcs.go +++ b/cmd/guacone/cmd/gcs.go @@ -43,6 +43,7 @@ type gcsOptions struct { queryVulnOnIngestion bool queryLicenseOnIngestion bool queryEOLOnIngestion bool + queryDepsDevOnIngestion bool } const gcsCredentialsPathFlag = "gcp-credentials-path" @@ -63,6 +64,7 @@ var gcsCmd = &cobra.Command{ viper.GetBool("add-vuln-on-ingest"), viper.GetBool("add-license-on-ingest"), viper.GetBool("add-eol-on-ingest"), + viper.GetBool("add-depsdev-on-ingest"), args) if err != nil { fmt.Printf("unable to validate flags: %v\n", err) @@ -123,8 +125,8 @@ var gcsCmd = &cobra.Command{ opts.queryVulnOnIngestion, opts.queryLicenseOnIngestion, opts.queryEOLOnIngestion, + opts.queryDepsDevOnIngestion, ) - if err != nil { gotErr = true return fmt.Errorf("unable to ingest document: %w", err) @@ -154,7 +156,7 @@ var gcsCmd = &cobra.Command{ } func validateGCSFlags(gqlEndpoint, headerFile, csubAddr, credentialsPath string, csubTls, csubTlsSkipVerify bool, - queryVulnIngestion bool, queryLicenseIngestion bool, queryEOLIngestion bool, args []string) (gcsOptions, error) { + queryVulnIngestion bool, queryLicenseIngestion bool, queryEOLIngestion bool, queryDepsDevOnIngestion bool, args []string) (gcsOptions, error) { var opts gcsOptions opts.graphqlEndpoint = gqlEndpoint opts.headerFile = headerFile @@ -176,6 +178,7 @@ func validateGCSFlags(gqlEndpoint, headerFile, csubAddr, credentialsPath string, opts.queryVulnOnIngestion = queryVulnIngestion opts.queryLicenseOnIngestion = queryLicenseIngestion opts.queryEOLOnIngestion = queryEOLIngestion + opts.queryDepsDevOnIngestion = queryDepsDevOnIngestion return opts, nil } diff --git a/cmd/guacone/cmd/gcs_test.go b/cmd/guacone/cmd/gcs_test.go index 11cbb288f6..d600e10945 100644 --- a/cmd/guacone/cmd/gcs_test.go +++ b/cmd/guacone/cmd/gcs_test.go @@ -62,7 +62,7 @@ func TestValidateGCSFlags(t *testing.T) { t.Setenv("GOOGLE_APPLICATION_CREDENTIALS", "/path/to/creds.json") } - o, err := validateGCSFlags("", "", "", tc.credentialsPath, false, false, false, false, false, tc.args) + o, err := validateGCSFlags("", "", "", tc.credentialsPath, false, false, false, false, false, false, tc.args) if err != nil { if tc.errorMsg != err.Error() { t.Errorf("expected error message: %s, got: %s", tc.errorMsg, err.Error()) diff --git a/cmd/guacone/cmd/github.go b/cmd/guacone/cmd/github.go index 9379ed4a85..332e4cf616 100644 --- a/cmd/guacone/cmd/github.go +++ b/cmd/guacone/cmd/github.go @@ -70,6 +70,7 @@ type githubOptions struct { queryVulnOnIngestion bool queryLicenseOnIngestion bool queryEOLOnIngestion bool + queryDepsDevOnIngestion bool } var githubCmd = &cobra.Command{ @@ -93,6 +94,7 @@ var githubCmd = &cobra.Command{ viper.GetBool("add-vuln-on-ingest"), viper.GetBool("add-license-on-ingest"), viper.GetBool("add-eol-on-ingest"), + viper.GetBool("add-depsdev-on-ingest"), args) if err != nil { fmt.Printf("unable to validate flags: %v\n", err) @@ -166,8 +168,8 @@ var githubCmd = &cobra.Command{ opts.queryVulnOnIngestion, opts.queryLicenseOnIngestion, opts.queryEOLOnIngestion, + opts.queryDepsDevOnIngestion, ) - if err != nil { errFound = true return fmt.Errorf("unable to ingest document: %w", err) @@ -220,7 +222,7 @@ var githubCmd = &cobra.Command{ } func validateGithubFlags(graphqlEndpoint, headerFile, githubMode, sbomName, workflowFileName, csubAddr string, csubTls, - csubTlsSkipVerify, useCsub, poll bool, queryVulnIngestion bool, queryLicenseIngestion bool, queryEOLIngestion bool, args []string) (githubOptions, error) { + csubTlsSkipVerify, useCsub, poll bool, queryVulnIngestion bool, queryLicenseIngestion bool, queryEOLIngestion bool, queryDepsDevOnIngestion bool, args []string) (githubOptions, error) { var opts githubOptions opts.graphqlEndpoint = graphqlEndpoint opts.headerFile = headerFile @@ -231,6 +233,7 @@ func validateGithubFlags(graphqlEndpoint, headerFile, githubMode, sbomName, work opts.queryVulnOnIngestion = queryVulnIngestion opts.queryLicenseOnIngestion = queryLicenseIngestion opts.queryEOLOnIngestion = queryEOLIngestion + opts.queryDepsDevOnIngestion = queryDepsDevOnIngestion if useCsub { csubOpts, err := csub_client.ValidateCsubClientFlags(csubAddr, csubTls, csubTlsSkipVerify) diff --git a/cmd/guacone/cmd/known.go b/cmd/guacone/cmd/known.go index 2ace3afb41..259cb6a317 100644 --- a/cmd/guacone/cmd/known.go +++ b/cmd/guacone/cmd/known.go @@ -462,8 +462,8 @@ func getOutputBasedOnNode(ctx context.Context, gqlclient graphql.Client, collect certifyLegalStr, legal.Id, "Declared License: " + legal.DeclaredLicense + - ",\nDiscovered License: " + legal.DiscoveredLicense + - ",\nOrigin: " + legal.Origin, + ",\nDiscovered License: " + legal.DiscoveredLicense + + ",\nOrigin: " + legal.Origin, }) } } diff --git a/cmd/guacone/cmd/license.go b/cmd/guacone/cmd/license.go index 6472b3cabc..947702ae6c 100644 --- a/cmd/guacone/cmd/license.go +++ b/cmd/guacone/cmd/license.go @@ -54,6 +54,7 @@ type cdOptions struct { queryVulnOnIngestion bool queryLicenseOnIngestion bool queryEOLOnIngestion bool + queryDepsDevOnIngestion bool // sets artificial latency on the certifier (default to nil) addedLatency *time.Duration // sets the batch size for pagination query for the certifier @@ -78,6 +79,7 @@ var cdCmd = &cobra.Command{ viper.GetBool("add-vuln-on-ingest"), viper.GetBool("add-license-on-ingest"), viper.GetBool("add-eol-on-ingest"), + viper.GetBool("add-depsdev-on-ingest"), viper.GetString("certifier-latency"), viper.GetInt("certifier-batch-size"), viper.GetInt("last-scan"), @@ -134,6 +136,7 @@ var cdCmd = &cobra.Command{ opts.queryVulnOnIngestion, opts.queryLicenseOnIngestion, opts.queryEOLOnIngestion, + opts.queryDepsDevOnIngestion, ) if err != nil { stop = true @@ -155,6 +158,7 @@ var cdCmd = &cobra.Command{ opts.queryVulnOnIngestion, opts.queryLicenseOnIngestion, opts.queryEOLOnIngestion, + opts.queryDepsDevOnIngestion, ) if err != nil { stop = true @@ -183,6 +187,7 @@ var cdCmd = &cobra.Command{ opts.queryVulnOnIngestion, opts.queryLicenseOnIngestion, opts.queryEOLOnIngestion, + opts.queryDepsDevOnIngestion, ) if err != nil { atomic.StoreInt32(&gotErr, 1) @@ -201,6 +206,7 @@ var cdCmd = &cobra.Command{ opts.queryVulnOnIngestion, opts.queryLicenseOnIngestion, opts.queryEOLOnIngestion, + opts.queryDepsDevOnIngestion, ) if err != nil { atomic.StoreInt32(&gotErr, 1) @@ -269,6 +275,7 @@ func validateCDFlags( queryVulnIngestion bool, queryLicenseIngestion bool, queryEOLIngestion bool, + queryDepsDevIngestion bool, certifierLatencyStr string, batchSize int, lastScan int, ) (cdOptions, error) { @@ -306,6 +313,7 @@ func validateCDFlags( opts.queryVulnOnIngestion = queryVulnIngestion opts.queryLicenseOnIngestion = queryLicenseIngestion opts.queryEOLOnIngestion = queryEOLIngestion + opts.queryDepsDevOnIngestion = queryDepsDevIngestion return opts, nil } diff --git a/cmd/guacone/cmd/oci.go b/cmd/guacone/cmd/oci.go index 5f090333ff..30f62c7dd9 100644 --- a/cmd/guacone/cmd/oci.go +++ b/cmd/guacone/cmd/oci.go @@ -44,6 +44,7 @@ type ociOptions struct { queryVulnOnIngestion bool queryLicenseOnIngestion bool queryEOLOnIngestion bool + queryDepsDevOnIngestion bool } var ociCmd = &cobra.Command{ @@ -60,6 +61,7 @@ var ociCmd = &cobra.Command{ viper.GetBool("add-vuln-on-ingest"), viper.GetBool("add-license-on-ingest"), viper.GetBool("add-eol-on-ingest"), + viper.GetBool("add-depsdev-on-ingest"), args) if err != nil { fmt.Printf("unable to validate flags: %v\n", err) @@ -101,8 +103,8 @@ var ociCmd = &cobra.Command{ opts.queryVulnOnIngestion, opts.queryLicenseOnIngestion, opts.queryEOLOnIngestion, + opts.queryDepsDevOnIngestion, ) - if err != nil { gotErr = true return fmt.Errorf("unable to ingest document: %w", err) @@ -132,13 +134,14 @@ var ociCmd = &cobra.Command{ } func validateOCIFlags(gqlEndpoint, headerFile, csubAddr string, csubTls, csubTlsSkipVerify bool, - queryVulnIngestion bool, queryLicenseIngestion bool, queryEOLIngestion bool, args []string) (ociOptions, error) { + queryVulnIngestion bool, queryLicenseIngestion bool, queryEOLIngestion bool, queryDepsDevOnIngestion bool, args []string) (ociOptions, error) { var opts ociOptions opts.graphqlEndpoint = gqlEndpoint opts.headerFile = headerFile opts.queryVulnOnIngestion = queryVulnIngestion opts.queryLicenseOnIngestion = queryLicenseIngestion opts.queryEOLOnIngestion = queryEOLIngestion + opts.queryDepsDevOnIngestion = queryDepsDevOnIngestion csubOpts, err := csub_client.ValidateCsubClientFlags(csubAddr, csubTls, csubTlsSkipVerify) if err != nil { diff --git a/cmd/guacone/cmd/osv.go b/cmd/guacone/cmd/osv.go index d58d41add5..4f5789e782 100644 --- a/cmd/guacone/cmd/osv.go +++ b/cmd/guacone/cmd/osv.go @@ -54,6 +54,7 @@ type osvOptions struct { queryVulnOnIngestion bool queryLicenseOnIngestion bool queryEOLOnIngestion bool + queryDepsDevOnIngestion bool // sets artificial latency on the certifier (default to nil) addedLatency *time.Duration // sets the batch size for pagination query for the certifier @@ -78,6 +79,7 @@ var osvCmd = &cobra.Command{ viper.GetBool("add-vuln-on-ingest"), viper.GetBool("add-license-on-ingest"), viper.GetBool("add-eol-on-ingest"), + viper.GetBool("add-depsdev-on-ingest"), viper.GetString("certifier-latency"), viper.GetInt("certifier-batch-size"), viper.GetInt("last-scan"), @@ -135,6 +137,7 @@ var osvCmd = &cobra.Command{ opts.queryVulnOnIngestion, opts.queryLicenseOnIngestion, opts.queryEOLOnIngestion, + opts.queryDepsDevOnIngestion, ) if err != nil { stop = true @@ -157,6 +160,7 @@ var osvCmd = &cobra.Command{ opts.queryVulnOnIngestion, opts.queryLicenseOnIngestion, opts.queryEOLOnIngestion, + opts.queryDepsDevOnIngestion, ) if err != nil { stop = true @@ -185,6 +189,7 @@ var osvCmd = &cobra.Command{ opts.queryVulnOnIngestion, opts.queryLicenseOnIngestion, opts.queryEOLOnIngestion, + opts.queryDepsDevOnIngestion, ) if err != nil { atomic.StoreInt32(&gotErr, 1) @@ -203,6 +208,7 @@ var osvCmd = &cobra.Command{ opts.queryVulnOnIngestion, opts.queryLicenseOnIngestion, opts.queryEOLOnIngestion, + opts.queryDepsDevOnIngestion, ) if err != nil { atomic.StoreInt32(&gotErr, 1) @@ -272,6 +278,7 @@ func validateOSVFlags( queryVulnIngestion bool, queryLicenseIngestion bool, queryEOLIngestion bool, + queryDepsDevIngestion bool, certifierLatencyStr string, batchSize int, lastScan int, ) (osvOptions, error) { @@ -309,6 +316,7 @@ func validateOSVFlags( opts.queryVulnOnIngestion = queryVulnIngestion opts.queryLicenseOnIngestion = queryLicenseIngestion opts.queryEOLOnIngestion = queryEOLIngestion + opts.queryDepsDevOnIngestion = queryDepsDevIngestion return opts, nil } diff --git a/cmd/guacone/cmd/s3.go b/cmd/guacone/cmd/s3.go index 19e078e2a7..ebbdf4ab3f 100644 --- a/cmd/guacone/cmd/s3.go +++ b/cmd/guacone/cmd/s3.go @@ -52,6 +52,7 @@ type s3Options struct { queryVulnOnIngestion bool queryLicenseOnIngestion bool queryEOLOnIngestion bool + queryDepsDevOnIngestion bool } var s3Cmd = &cobra.Command{ @@ -98,6 +99,7 @@ $ guacone collect s3 --s3-url http://localhost:9000 --s3-bucket guac-test --poll viper.GetBool("add-vuln-on-ingest"), viper.GetBool("add-license-on-ingest"), viper.GetBool("add-eol-on-ingest"), + viper.GetBool("add-depsdev-on-ingest"), ) if err != nil { fmt.Printf("failed to validate flags: %v\n", err) @@ -148,6 +150,7 @@ $ guacone collect s3 --s3-url http://localhost:9000 --s3-bucket guac-test --poll s3Opts.queryVulnOnIngestion, s3Opts.queryLicenseOnIngestion, s3Opts.queryEOLOnIngestion, + s3Opts.queryDepsDevOnIngestion, ) if err != nil { @@ -195,7 +198,7 @@ $ guacone collect s3 --s3-url http://localhost:9000 --s3-bucket guac-test --poll } func validateS3Opts(graphqlEndpoint, headerFile, csubAddr, s3url, s3bucket, s3path, region, s3item, mp, mpEndpoint, queues string, - csubTls, csubTlsSkipVerify, poll bool, queryVulnIngestion bool, queryLicenseIngestion bool, queryEOLIngestion bool) (s3Options, error) { + csubTls, csubTlsSkipVerify, poll bool, queryVulnIngestion bool, queryLicenseIngestion bool, queryEOLIngestion bool, queryDepsDevIngestion bool) (s3Options, error) { var opts s3Options if poll { @@ -219,7 +222,7 @@ func validateS3Opts(graphqlEndpoint, headerFile, csubAddr, s3url, s3bucket, s3pa } opts = s3Options{s3url, s3bucket, s3path, s3item, region, queues, mp, mpEndpoint, poll, graphqlEndpoint, headerFile, - csubClientOptions, queryVulnIngestion, queryLicenseIngestion, queryEOLIngestion} + csubClientOptions, queryVulnIngestion, queryLicenseIngestion, queryEOLIngestion, queryDepsDevIngestion} return opts, nil } diff --git a/cmd/guacone/cmd/scorecard.go b/cmd/guacone/cmd/scorecard.go index 2c939363fc..8c142d54b2 100644 --- a/cmd/guacone/cmd/scorecard.go +++ b/cmd/guacone/cmd/scorecard.go @@ -50,6 +50,7 @@ type scorecardOptions struct { queryVulnOnIngestion bool queryLicenseOnIngestion bool queryEOLOnIngestion bool + queryDepsDevOnIngestion bool // sets artificial latency on the certifier (default to nil) addedLatency *time.Duration // sets the batch size for pagination query for the certifier @@ -71,6 +72,7 @@ var scorecardCmd = &cobra.Command{ viper.GetBool("add-vuln-on-ingest"), viper.GetBool("add-license-on-ingest"), viper.GetBool("add-eol-on-ingest"), + viper.GetBool("add-depsdev-on-ingest"), viper.GetString("certifier-latency"), viper.GetInt("certifier-batch-size"), ) @@ -143,8 +145,8 @@ var scorecardCmd = &cobra.Command{ opts.queryVulnOnIngestion, opts.queryLicenseOnIngestion, opts.queryEOLOnIngestion, + opts.queryDepsDevOnIngestion, ) - if err != nil { return fmt.Errorf("unable to ingest document: %v", err) } @@ -202,6 +204,7 @@ func validateScorecardFlags( queryVulnIngestion bool, queryLicenseIngestion bool, queryEOLOnIngestion bool, + queryDepsDevIngestion bool, certifierLatencyStr string, batchSize int, ) (scorecardOptions, error) { @@ -236,6 +239,7 @@ func validateScorecardFlags( opts.queryVulnOnIngestion = queryVulnIngestion opts.queryLicenseOnIngestion = queryLicenseIngestion opts.queryEOLOnIngestion = queryEOLOnIngestion + opts.queryDepsDevOnIngestion = queryDepsDevIngestion return opts, nil } diff --git a/internal/testing/cmd/ingest/cmd/example.go b/internal/testing/cmd/ingest/cmd/example.go index f56b3d8382..895e58dcf7 100644 --- a/internal/testing/cmd/ingest/cmd/example.go +++ b/internal/testing/cmd/ingest/cmd/example.go @@ -59,7 +59,7 @@ func ingestExample(cmd *cobra.Command, args []string) { var inputs []assembler.IngestPredicates for _, doc := range docs { // This is a test example, so we will ignore calling out to a collectsub service - input, _, err := parser.ParseDocumentTree(ctx, doc, false, false, false) + input, _, err := parser.ParseDocumentTree(ctx, doc, false, false, false, false) if err != nil { logger.Fatalf("unable to parse document: %v", err) } diff --git a/pkg/ingestor/ingestor.go b/pkg/ingestor/ingestor.go index 83d67b6f27..782ff7264d 100644 --- a/pkg/ingestor/ingestor.go +++ b/pkg/ingestor/ingestor.go @@ -45,11 +45,12 @@ func Ingest( scanForVulns bool, scanForLicense bool, scanForEOL bool, + scanForDepsDev bool, ) (*helpers.AssemblerIngestedIDs, error) { logger := d.ChildLogger // Get pipeline of components processorFunc := GetProcessor(ctx) - ingestorFunc := GetIngestor(ctx, scanForVulns, scanForLicense, scanForEOL) + ingestorFunc := GetIngestor(ctx, scanForVulns, scanForLicense, scanForEOL, scanForDepsDev) collectSubEmitFunc := GetCollectSubEmit(ctx, csubClient) assemblerFunc := GetAssembler(ctx, d.ChildLogger, graphqlEndpoint, transport) @@ -89,11 +90,12 @@ func MergedIngest( scanForVulns bool, scanForLicense bool, scanForEOL bool, + scanForDepsDev bool, ) error { logger := logging.FromContext(ctx) // Get pipeline of components processorFunc := GetProcessor(ctx) - ingestorFunc := GetIngestor(ctx, scanForVulns, scanForLicense, scanForEOL) + ingestorFunc := GetIngestor(ctx, scanForVulns, scanForLicense, scanForEOL, scanForDepsDev) collectSubEmitFunc := GetCollectSubEmit(ctx, csubClient) assemblerFunc := GetAssembler(ctx, logger, graphqlEndpoint, transport) @@ -166,9 +168,9 @@ func GetProcessor(ctx context.Context) func(*processor.Document) (processor.Docu } } -func GetIngestor(ctx context.Context, scanForVulns bool, scanForLicense bool, scanForEOL bool) func(processor.DocumentTree) ([]assembler.IngestPredicates, []*parser_common.IdentifierStrings, error) { +func GetIngestor(ctx context.Context, scanForVulns bool, scanForLicense bool, scanForEOL bool, scanForDepsDev bool) func(processor.DocumentTree) ([]assembler.IngestPredicates, []*parser_common.IdentifierStrings, error) { return func(doc processor.DocumentTree) ([]assembler.IngestPredicates, []*parser_common.IdentifierStrings, error) { - return parser.ParseDocumentTree(ctx, doc, scanForVulns, scanForLicense, scanForEOL) + return parser.ParseDocumentTree(ctx, doc, scanForVulns, scanForLicense, scanForEOL, scanForDepsDev) } } diff --git a/pkg/ingestor/parser/common/scanner/scanner.go b/pkg/ingestor/parser/common/scanner/scanner.go index 1665981d93..ab5c5bd7a3 100644 --- a/pkg/ingestor/parser/common/scanner/scanner.go +++ b/pkg/ingestor/parser/common/scanner/scanner.go @@ -22,8 +22,8 @@ import ( "github.com/guacsec/guac/pkg/assembler" cd_certifier "github.com/guacsec/guac/pkg/certifier/clearlydefined" - osv_certifier "github.com/guacsec/guac/pkg/certifier/osv" eol_certifier "github.com/guacsec/guac/pkg/certifier/eol" + osv_certifier "github.com/guacsec/guac/pkg/certifier/osv" "github.com/guacsec/guac/pkg/ingestor/parser/clearlydefined" "github.com/guacsec/guac/pkg/ingestor/parser/common" "github.com/guacsec/guac/pkg/ingestor/parser/eol" @@ -106,6 +106,10 @@ func PurlsLicenseScan(ctx context.Context, purls []string) ([]assembler.CertifyL return certLegalIngest, hasSourceAtIngest, nil } +func PurlsDepsDevScan(ctx context.Context, purls []string) ([]assembler.CertifyScorecardIngest, []assembler.HasSourceAtIngest, error) { + return nil, nil, fmt.Errorf("Unimplemented") +} + // runQueryOnBatchedPurls runs EvaluateClearlyDefinedDefinition from the clearly defined // certifier to evaluate the batched purls for license information func runQueryOnBatchedPurls(ctx context.Context, cdParser common.DocumentParser, batchPurls []string) ([]assembler.CertifyLegalIngest, []assembler.HasSourceAtIngest, error) { diff --git a/pkg/ingestor/parser/parser.go b/pkg/ingestor/parser/parser.go index 415427a993..884185f986 100644 --- a/pkg/ingestor/parser/parser.go +++ b/pkg/ingestor/parser/parser.go @@ -79,7 +79,7 @@ func RegisterDocumentParser(p func() common.DocumentParser, d processor.Document } // ParseDocumentTree takes the DocumentTree and create graph inputs (nodes and edges) per document node. -func ParseDocumentTree(ctx context.Context, docTree processor.DocumentTree, scanForVulns bool, scanForLicense bool, scanForEOL bool) ([]assembler.IngestPredicates, []*common.IdentifierStrings, error) { +func ParseDocumentTree(ctx context.Context, docTree processor.DocumentTree, scanForVulns bool, scanForLicense bool, scanForEOL bool, scanForDepsDev bool) ([]assembler.IngestPredicates, []*common.IdentifierStrings, error) { var wg sync.WaitGroup assemblerInputs := []assembler.IngestPredicates{} @@ -126,6 +126,28 @@ func ParseDocumentTree(ctx context.Context, docTree processor.DocumentTree, scan }() } + if scanForDepsDev { + wg.Add(1) + go func() { + defer wg.Done() + // scan purls via deps.dev on initial ingestion to capture additional deps.dev information + var purls []string + for _, idString := range identifierStrings { + purls = append(purls, idString.PurlStrings...) + } + + certScorecard, hasSrcAt, err := scanner.PurlsDepsDevScan(ctx, purls) + if err != nil { + logger.Errorf("error scanning purls for vulnerabilities %v", err) + } else { + if len(assemblerInputs) > 0 { + assemblerInputs[0].CertifyScorecard = append(assemblerInputs[0].CertifyScorecard, certScorecard...) + assemblerInputs[0].HasSourceAt = append(assemblerInputs[0].HasSourceAt, hasSrcAt...) + } + } + }() + } + if scanForLicense { wg.Add(1) go func() { diff --git a/pkg/ingestor/parser/parser_test.go b/pkg/ingestor/parser/parser_test.go index 5aa639b552..8b8096759d 100644 --- a/pkg/ingestor/parser/parser_test.go +++ b/pkg/ingestor/parser/parser_test.go @@ -293,7 +293,7 @@ func TestParseDocumentTree(t *testing.T) { _ = RegisterDocumentParser(f, test.registerDocType) // Ignoring error because it is mutating a global variable - got, got1, err := ParseDocumentTree(ctx, test.docTree, true, true, true) + got, got1, err := ParseDocumentTree(ctx, test.docTree, true, true, true, false) if (err != nil) != test.wantErr { t.Errorf("ParseDocumentTree() error = %v, wantErr %v", err, test.wantErr)