plumbing to enable deps.dev on ingest (guacsec#2265)

Signed-off-by: Brandon Lum <lumb@google.com>
mrizzi · Nov 9, 2024 · bc24861 · bc24861
1 parent 5f4ea90
commit bc24861
Show file tree

Hide file tree

Showing 18 changed files with 94 additions and 25 deletions.
diff --git a/cmd/guacingest/cmd/ingest.go b/cmd/guacingest/cmd/ingest.go
@@ -48,6 +48,7 @@ type options struct {
 	queryVulnOnIngestion    bool
 	queryLicenseOnIngestion bool
 	queryEOLOnIngestion     bool
+	queryDepsDevOnIngestion bool
 }
 
 func ingest(cmd *cobra.Command, args []string) {
@@ -110,6 +111,7 @@ func ingest(cmd *cobra.Command, args []string) {
 			opts.queryVulnOnIngestion,
 			opts.queryLicenseOnIngestion,
 			opts.queryEOLOnIngestion,
+			opts.queryDepsDevOnIngestion,
 		); err != nil {
 			var urlErr *url.Error
 			if errors.As(err, &urlErr) {

diff --git a/cmd/guacone/cmd/deps_dev.go b/cmd/guacone/cmd/deps_dev.go
@@ -97,6 +97,9 @@ var depsDevCmd = &cobra.Command{
 				opts.queryVulnOnIngestion,
 				opts.queryLicenseOnIngestion,
 				opts.queryEOLOnIngestion,
+				// since this is a deps.dev collector, by we don't query deps.dev on ingestion
+				/* queryDepsDevOnIngestion = */
+				false,
 			); err != nil {
 				gotErr = true
 				return fmt.Errorf("unable to ingest document: %w", err)

diff --git a/cmd/guacone/cmd/eol.go b/cmd/guacone/cmd/eol.go
@@ -115,7 +115,7 @@ var eolCmd = &cobra.Command{
 				select {
 				case <-ticker.C:
 					if len(totalDocs) > 0 {
-						err = ingestor.MergedIngest(ctx, totalDocs, opts.graphqlEndpoint, transport, csubClient, false, false, false)
+						err = ingestor.MergedIngest(ctx, totalDocs, opts.graphqlEndpoint, transport, csubClient, false, false, false, false)
 						if err != nil {
 							stop = true
 							atomic.StoreInt32(&gotErr, 1)
@@ -128,7 +128,7 @@ var eolCmd = &cobra.Command{
 					totalNum += 1
 					totalDocs = append(totalDocs, d)
 					if len(totalDocs) >= threshold {
-						err = ingestor.MergedIngest(ctx, totalDocs, opts.graphqlEndpoint, transport, csubClient, false, false, false)
+						err = ingestor.MergedIngest(ctx, totalDocs, opts.graphqlEndpoint, transport, csubClient, false, false, false, false)
 						if err != nil {
 							stop = true
 							atomic.StoreInt32(&gotErr, 1)
@@ -147,7 +147,7 @@ var eolCmd = &cobra.Command{
 				totalNum += 1
 				totalDocs = append(totalDocs, <-docChan)
 				if len(totalDocs) >= threshold {
-					err = ingestor.MergedIngest(ctx, totalDocs, opts.graphqlEndpoint, transport, csubClient, false, false, false)
+					err = ingestor.MergedIngest(ctx, totalDocs, opts.graphqlEndpoint, transport, csubClient, false, false, false, false)
 					if err != nil {
 						atomic.StoreInt32(&gotErr, 1)
 						logger.Errorf("unable to ingest documents: %v", err)
@@ -156,7 +156,7 @@ var eolCmd = &cobra.Command{
 				}
 			}
 			if len(totalDocs) > 0 {
-				err = ingestor.MergedIngest(ctx, totalDocs, opts.graphqlEndpoint, transport, csubClient, false, false, false)
+				err = ingestor.MergedIngest(ctx, totalDocs, opts.graphqlEndpoint, transport, csubClient, false, false, false, false)
 				if err != nil {
 					atomic.StoreInt32(&gotErr, 1)
 					logger.Errorf("unable to ingest documents: %v", err)

diff --git a/cmd/guacone/cmd/files.go b/cmd/guacone/cmd/files.go
@@ -54,6 +54,7 @@ type fileOptions struct {
 	queryVulnOnIngestion    bool
 	queryLicenseOnIngestion bool
 	queryEOLOnIngestion     bool
+	queryDepsDevOnIngestion bool
 }
 
 var filesCmd = &cobra.Command{
@@ -71,6 +72,7 @@ var filesCmd = &cobra.Command{
 			viper.GetBool("add-vuln-on-ingest"),
 			viper.GetBool("add-license-on-ingest"),
 			viper.GetBool("add-eol-on-ingest"),
+			viper.GetBool("add-depsdev-on-ingest"),
 			args)
 		if err != nil {
 			fmt.Printf("unable to validate flags: %v\n", err)
@@ -140,6 +142,7 @@ var filesCmd = &cobra.Command{
 				opts.queryVulnOnIngestion,
 				opts.queryLicenseOnIngestion,
 				opts.queryEOLOnIngestion,
+				opts.queryDepsDevOnIngestion,
 			); err != nil {
 				gotErr = true
 				filesWithErrors = append(filesWithErrors, d.SourceInformation.Source)
@@ -173,7 +176,7 @@ var filesCmd = &cobra.Command{
 }
 
 func validateFilesFlags(keyPath, keyID, graphqlEndpoint, headerFile, csubAddr string, csubTls, csubTlsSkipVerify bool,
-	queryVulnIngestion bool, queryLicenseIngestion bool, queryEOLIngestion bool, args []string) (fileOptions, error) {
+	queryVulnIngestion bool, queryLicenseIngestion bool, queryEOLIngestion bool, queryDepsDevOnIngestion bool, args []string) (fileOptions, error) {
 	var opts fileOptions
 	opts.graphqlEndpoint = graphqlEndpoint
 	opts.headerFile = headerFile
@@ -202,6 +205,7 @@ func validateFilesFlags(keyPath, keyID, graphqlEndpoint, headerFile, csubAddr st
 	opts.queryVulnOnIngestion = queryVulnIngestion
 	opts.queryLicenseOnIngestion = queryLicenseIngestion
 	opts.queryEOLOnIngestion = queryEOLIngestion
+	opts.queryDepsDevOnIngestion = queryDepsDevOnIngestion
 	return opts, nil
 }
 

diff --git a/cmd/guacone/cmd/gcs.go b/cmd/guacone/cmd/gcs.go
@@ -43,6 +43,7 @@ type gcsOptions struct {
 	queryVulnOnIngestion    bool
 	queryLicenseOnIngestion bool
 	queryEOLOnIngestion     bool
+	queryDepsDevOnIngestion bool
 }
 
 const gcsCredentialsPathFlag = "gcp-credentials-path"
@@ -63,6 +64,7 @@ var gcsCmd = &cobra.Command{
 			viper.GetBool("add-vuln-on-ingest"),
 			viper.GetBool("add-license-on-ingest"),
 			viper.GetBool("add-eol-on-ingest"),
+			viper.GetBool("add-depsdev-on-ingest"),
 			args)
 		if err != nil {
 			fmt.Printf("unable to validate flags: %v\n", err)
@@ -123,8 +125,8 @@ var gcsCmd = &cobra.Command{
 				opts.queryVulnOnIngestion,
 				opts.queryLicenseOnIngestion,
 				opts.queryEOLOnIngestion,
+				opts.queryDepsDevOnIngestion,
 			)
-
 			if err != nil {
 				gotErr = true
 				return fmt.Errorf("unable to ingest document: %w", err)
@@ -154,7 +156,7 @@ var gcsCmd = &cobra.Command{
 }
 
 func validateGCSFlags(gqlEndpoint, headerFile, csubAddr, credentialsPath string, csubTls, csubTlsSkipVerify bool,
-	queryVulnIngestion bool, queryLicenseIngestion bool, queryEOLIngestion bool, args []string) (gcsOptions, error) {
+	queryVulnIngestion bool, queryLicenseIngestion bool, queryEOLIngestion bool, queryDepsDevOnIngestion bool, args []string) (gcsOptions, error) {
 	var opts gcsOptions
 	opts.graphqlEndpoint = gqlEndpoint
 	opts.headerFile = headerFile
@@ -176,6 +178,7 @@ func validateGCSFlags(gqlEndpoint, headerFile, csubAddr, credentialsPath string,
 	opts.queryVulnOnIngestion = queryVulnIngestion
 	opts.queryLicenseOnIngestion = queryLicenseIngestion
 	opts.queryEOLOnIngestion = queryEOLIngestion
+	opts.queryDepsDevOnIngestion = queryDepsDevOnIngestion
 	return opts, nil
 }
 

diff --git a/cmd/guacone/cmd/gcs_test.go b/cmd/guacone/cmd/gcs_test.go
@@ -62,7 +62,7 @@ func TestValidateGCSFlags(t *testing.T) {
 				t.Setenv("GOOGLE_APPLICATION_CREDENTIALS", "/path/to/creds.json")
 			}
 
-			o, err := validateGCSFlags("", "", "", tc.credentialsPath, false, false, false, false, false, tc.args)
+			o, err := validateGCSFlags("", "", "", tc.credentialsPath, false, false, false, false, false, false, tc.args)
 			if err != nil {
 				if tc.errorMsg != err.Error() {
 					t.Errorf("expected error message: %s, got: %s", tc.errorMsg, err.Error())

diff --git a/cmd/guacone/cmd/github.go b/cmd/guacone/cmd/github.go
@@ -70,6 +70,7 @@ type githubOptions struct {
 	queryVulnOnIngestion    bool
 	queryLicenseOnIngestion bool
 	queryEOLOnIngestion     bool
+	queryDepsDevOnIngestion bool
 }
 
 var githubCmd = &cobra.Command{
@@ -93,6 +94,7 @@ var githubCmd = &cobra.Command{
 			viper.GetBool("add-vuln-on-ingest"),
 			viper.GetBool("add-license-on-ingest"),
 			viper.GetBool("add-eol-on-ingest"),
+			viper.GetBool("add-depsdev-on-ingest"),
 			args)
 		if err != nil {
 			fmt.Printf("unable to validate flags: %v\n", err)
@@ -166,8 +168,8 @@ var githubCmd = &cobra.Command{
 				opts.queryVulnOnIngestion,
 				opts.queryLicenseOnIngestion,
 				opts.queryEOLOnIngestion,
+				opts.queryDepsDevOnIngestion,
 			)
-
 			if err != nil {
 				errFound = true
 				return fmt.Errorf("unable to ingest document: %w", err)
@@ -220,7 +222,7 @@ var githubCmd = &cobra.Command{
 }
 
 func validateGithubFlags(graphqlEndpoint, headerFile, githubMode, sbomName, workflowFileName, csubAddr string, csubTls,
-	csubTlsSkipVerify, useCsub, poll bool, queryVulnIngestion bool, queryLicenseIngestion bool, queryEOLIngestion bool, args []string) (githubOptions, error) {
+	csubTlsSkipVerify, useCsub, poll bool, queryVulnIngestion bool, queryLicenseIngestion bool, queryEOLIngestion bool, queryDepsDevOnIngestion bool, args []string) (githubOptions, error) {
 	var opts githubOptions
 	opts.graphqlEndpoint = graphqlEndpoint
 	opts.headerFile = headerFile
@@ -231,6 +233,7 @@ func validateGithubFlags(graphqlEndpoint, headerFile, githubMode, sbomName, work
 	opts.queryVulnOnIngestion = queryVulnIngestion
 	opts.queryLicenseOnIngestion = queryLicenseIngestion
 	opts.queryEOLOnIngestion = queryEOLIngestion
+	opts.queryDepsDevOnIngestion = queryDepsDevOnIngestion
 
 	if useCsub {
 		csubOpts, err := csub_client.ValidateCsubClientFlags(csubAddr, csubTls, csubTlsSkipVerify)

diff --git a/cmd/guacone/cmd/known.go b/cmd/guacone/cmd/known.go
@@ -462,8 +462,8 @@ func getOutputBasedOnNode(ctx context.Context, gqlclient graphql.Client, collect
 				certifyLegalStr,
 				legal.Id,
 				"Declared License: " + legal.DeclaredLicense +
-				",\nDiscovered License: " + legal.DiscoveredLicense +
-				",\nOrigin: " + legal.Origin,
+					",\nDiscovered License: " + legal.DiscoveredLicense +
+					",\nOrigin: " + legal.Origin,
 			})
 		}
 	}

diff --git a/cmd/guacone/cmd/license.go b/cmd/guacone/cmd/license.go
@@ -54,6 +54,7 @@ type cdOptions struct {
 	queryVulnOnIngestion    bool
 	queryLicenseOnIngestion bool
 	queryEOLOnIngestion     bool
+	queryDepsDevOnIngestion bool
 	// sets artificial latency on the certifier (default to nil)
 	addedLatency *time.Duration
 	// sets the batch size for pagination query for the certifier
@@ -78,6 +79,7 @@ var cdCmd = &cobra.Command{
 			viper.GetBool("add-vuln-on-ingest"),
 			viper.GetBool("add-license-on-ingest"),
 			viper.GetBool("add-eol-on-ingest"),
+			viper.GetBool("add-depsdev-on-ingest"),
 			viper.GetString("certifier-latency"),
 			viper.GetInt("certifier-batch-size"),
 			viper.GetInt("last-scan"),
@@ -134,6 +136,7 @@ var cdCmd = &cobra.Command{
 							opts.queryVulnOnIngestion,
 							opts.queryLicenseOnIngestion,
 							opts.queryEOLOnIngestion,
+							opts.queryDepsDevOnIngestion,
 						)
 						if err != nil {
 							stop = true
@@ -155,6 +158,7 @@ var cdCmd = &cobra.Command{
 							opts.queryVulnOnIngestion,
 							opts.queryLicenseOnIngestion,
 							opts.queryEOLOnIngestion,
+							opts.queryDepsDevOnIngestion,
 						)
 						if err != nil {
 							stop = true
@@ -183,6 +187,7 @@ var cdCmd = &cobra.Command{
 						opts.queryVulnOnIngestion,
 						opts.queryLicenseOnIngestion,
 						opts.queryEOLOnIngestion,
+						opts.queryDepsDevOnIngestion,
 					)
 					if err != nil {
 						atomic.StoreInt32(&gotErr, 1)
@@ -201,6 +206,7 @@ var cdCmd = &cobra.Command{
 					opts.queryVulnOnIngestion,
 					opts.queryLicenseOnIngestion,
 					opts.queryEOLOnIngestion,
+					opts.queryDepsDevOnIngestion,
 				)
 				if err != nil {
 					atomic.StoreInt32(&gotErr, 1)
@@ -269,6 +275,7 @@ func validateCDFlags(
 	queryVulnIngestion bool,
 	queryLicenseIngestion bool,
 	queryEOLIngestion bool,
+	queryDepsDevIngestion bool,
 	certifierLatencyStr string,
 	batchSize int, lastScan int,
 ) (cdOptions, error) {
@@ -306,6 +313,7 @@ func validateCDFlags(
 	opts.queryVulnOnIngestion = queryVulnIngestion
 	opts.queryLicenseOnIngestion = queryLicenseIngestion
 	opts.queryEOLOnIngestion = queryEOLIngestion
+	opts.queryDepsDevOnIngestion = queryDepsDevIngestion
 
 	return opts, nil
 }

diff --git a/cmd/guacone/cmd/oci.go b/cmd/guacone/cmd/oci.go
@@ -44,6 +44,7 @@ type ociOptions struct {
 	queryVulnOnIngestion    bool
 	queryLicenseOnIngestion bool
 	queryEOLOnIngestion     bool
+	queryDepsDevOnIngestion bool
 }
 
 var ociCmd = &cobra.Command{
@@ -60,6 +61,7 @@ var ociCmd = &cobra.Command{
 			viper.GetBool("add-vuln-on-ingest"),
 			viper.GetBool("add-license-on-ingest"),
 			viper.GetBool("add-eol-on-ingest"),
+			viper.GetBool("add-depsdev-on-ingest"),
 			args)
 		if err != nil {
 			fmt.Printf("unable to validate flags: %v\n", err)
@@ -101,8 +103,8 @@ var ociCmd = &cobra.Command{
 				opts.queryVulnOnIngestion,
 				opts.queryLicenseOnIngestion,
 				opts.queryEOLOnIngestion,
+				opts.queryDepsDevOnIngestion,
 			)
-
 			if err != nil {
 				gotErr = true
 				return fmt.Errorf("unable to ingest document: %w", err)
@@ -132,13 +134,14 @@ var ociCmd = &cobra.Command{
 }
 
 func validateOCIFlags(gqlEndpoint, headerFile, csubAddr string, csubTls, csubTlsSkipVerify bool,
-	queryVulnIngestion bool, queryLicenseIngestion bool, queryEOLIngestion bool, args []string) (ociOptions, error) {
+	queryVulnIngestion bool, queryLicenseIngestion bool, queryEOLIngestion bool, queryDepsDevOnIngestion bool, args []string) (ociOptions, error) {
 	var opts ociOptions
 	opts.graphqlEndpoint = gqlEndpoint
 	opts.headerFile = headerFile
 	opts.queryVulnOnIngestion = queryVulnIngestion
 	opts.queryLicenseOnIngestion = queryLicenseIngestion
 	opts.queryEOLOnIngestion = queryEOLIngestion
+	opts.queryDepsDevOnIngestion = queryDepsDevOnIngestion
 
 	csubOpts, err := csub_client.ValidateCsubClientFlags(csubAddr, csubTls, csubTlsSkipVerify)
 	if err != nil {

diff --git a/cmd/guacone/cmd/osv.go b/cmd/guacone/cmd/osv.go
@@ -54,6 +54,7 @@ type osvOptions struct {
 	queryVulnOnIngestion    bool
 	queryLicenseOnIngestion bool
 	queryEOLOnIngestion     bool
+	queryDepsDevOnIngestion bool
 	// sets artificial latency on the certifier (default to nil)
 	addedLatency *time.Duration
 	// sets the batch size for pagination query for the certifier
@@ -78,6 +79,7 @@ var osvCmd = &cobra.Command{
 			viper.GetBool("add-vuln-on-ingest"),
 			viper.GetBool("add-license-on-ingest"),
 			viper.GetBool("add-eol-on-ingest"),
+			viper.GetBool("add-depsdev-on-ingest"),
 			viper.GetString("certifier-latency"),
 			viper.GetInt("certifier-batch-size"),
 			viper.GetInt("last-scan"),
@@ -135,6 +137,7 @@ var osvCmd = &cobra.Command{
 							opts.queryVulnOnIngestion,
 							opts.queryLicenseOnIngestion,
 							opts.queryEOLOnIngestion,
+							opts.queryDepsDevOnIngestion,
 						)
 						if err != nil {
 							stop = true
@@ -157,6 +160,7 @@ var osvCmd = &cobra.Command{
 							opts.queryVulnOnIngestion,
 							opts.queryLicenseOnIngestion,
 							opts.queryEOLOnIngestion,
+							opts.queryDepsDevOnIngestion,
 						)
 						if err != nil {
 							stop = true
@@ -185,6 +189,7 @@ var osvCmd = &cobra.Command{
 						opts.queryVulnOnIngestion,
 						opts.queryLicenseOnIngestion,
 						opts.queryEOLOnIngestion,
+						opts.queryDepsDevOnIngestion,
 					)
 					if err != nil {
 						atomic.StoreInt32(&gotErr, 1)
@@ -203,6 +208,7 @@ var osvCmd = &cobra.Command{
 					opts.queryVulnOnIngestion,
 					opts.queryLicenseOnIngestion,
 					opts.queryEOLOnIngestion,
+					opts.queryDepsDevOnIngestion,
 				)
 				if err != nil {
 					atomic.StoreInt32(&gotErr, 1)
@@ -272,6 +278,7 @@ func validateOSVFlags(
 	queryVulnIngestion bool,
 	queryLicenseIngestion bool,
 	queryEOLIngestion bool,
+	queryDepsDevIngestion bool,
 	certifierLatencyStr string,
 	batchSize int, lastScan int,
 ) (osvOptions, error) {
@@ -309,6 +316,7 @@ func validateOSVFlags(
 	opts.queryVulnOnIngestion = queryVulnIngestion
 	opts.queryLicenseOnIngestion = queryLicenseIngestion
 	opts.queryEOLOnIngestion = queryEOLIngestion
+	opts.queryDepsDevOnIngestion = queryDepsDevIngestion
 
 	return opts, nil
 }