diff --git a/cmd/detectExecuteScan.go b/cmd/detectExecuteScan.go index 5655582638..3f54283aaf 100644 --- a/cmd/detectExecuteScan.go +++ b/cmd/detectExecuteScan.go @@ -30,6 +30,8 @@ import ( "github.com/pkg/errors" ) +const NO_VERSION_SUFFIX = "" + type detectUtils interface { piperutils.FileUtils @@ -202,7 +204,7 @@ func runDetect(ctx context.Context, config detectExecuteScanOptions, utils detec blackduckSystem := newBlackduckSystem(config) args := []string{"./detect.sh"} - args, err = addDetectArgs(args, config, utils, blackduckSystem) + args, err = addDetectArgs(args, config, utils, blackduckSystem, NO_VERSION_SUFFIX, NO_VERSION_SUFFIX) if err != nil { return err } @@ -214,7 +216,12 @@ func runDetect(ctx context.Context, config detectExecuteScanOptions, utils detec utils.SetDir(".") utils.SetEnv(envs) - err = utils.RunShell("/bin/bash", script) + if !config.ScanImages { + err = mapDetectError(utils.RunShell("/bin/bash", script), config, utils) + } else { + err = mapDetectError(runDetectImages(ctx, config, utils, blackduckSystem, influx, blackduckSystem), config, utils) + } + reportingErr := postScanChecksAndReporting(ctx, config, influx, utils, blackduckSystem) if reportingErr != nil { if strings.Contains(reportingErr.Error(), "License Policy Violations found") { @@ -247,6 +254,71 @@ func runDetect(ctx context.Context, config detectExecuteScanOptions, utils detec return err } +func mapDetectError(err error, config detectExecuteScanOptions, utils detectUtils) error { + if err != nil { + // Setting error category based on exit code + mapErrorCategory(utils.GetExitCode()) + if log.GetErrorCategory() == log.ErrorCompliance && !config.FailOnSevereVulnerabilities { + err = nil + log.Entry().Infof("policy violation(s) found - step will only create data but not fail due to setting failOnSevereVulnerabilities: false") + } else { + // Error code mapping with more human readable text + err = errors.Wrapf(err, exitCodeMapping(utils.GetExitCode())) + } + } + return err +} + +func runDetectImages(ctx context.Context, config detectExecuteScanOptions, utils detectUtils, sys *blackduckSystem, influx *detectExecuteScanInflux, blackduckSystem *blackduckSystem) error { + // cpePath := filepath.Join(GeneralConfig.EnvRootPath, "commonPipelineEnvironment") + imagesRaw := config.ImageNameTags + if len(imagesRaw) == 0 { + log.Entry().Debugf("No images found to be scanned") + return nil + } + + registryUser := config.ContainerRegistryUser + registryPassword := config.ContainerRegistryPassword + registryURL := config.ContainerRegistryURL + + log.Entry().Infof("Scanning %d images", len(imagesRaw)) + for _, image := range imagesRaw { + // Download image to be scanned + log.Entry().Debugf("Scanning image: %q", image) + tarName := fmt.Sprintf("%s.tar", strings.Split(image, ":")[0]) + + options := containerSaveImageOptions{ + ContainerRegistryURL: registryURL, + ContainerImage: image, + ContainerRegistryPassword: registryPassword, + ContainerRegistryUser: registryUser, + FilePath: tarName, + ImageFormat: "legacy", + } + containerSaveImage(options, &telemetry.CustomData{}) + err := utils.Chmod(tarName, 0777) + if err != nil { + return err + } + + args := []string{"./detect.sh"} + args, err = addDetectArgsImages(args, config, utils, sys, tarName) + if err != nil { + return err + } + script := strings.Join(args, " ") + + err = utils.RunShell("/bin/bash", script) + err = mapDetectError(err, config, utils) + + if err != nil { + return err + } + } + + return nil +} + // Get proper error category func mapErrorCategory(exitCodeKey int) { switch exitCodeKey { @@ -331,8 +403,13 @@ func getDetectScript(config detectExecuteScanOptions, utils detectUtils) error { return nil } -func addDetectArgs(args []string, config detectExecuteScanOptions, utils detectUtils, sys *blackduckSystem) ([]string, error) { +func addDetectArgs(args []string, config detectExecuteScanOptions, utils detectUtils, sys *blackduckSystem, versionSuffix, locationSuffix string) ([]string, error) { detectVersionName := getVersionName(config) + + if versionSuffix != NO_VERSION_SUFFIX { + detectVersionName = fmt.Sprintf("%s-%s", detectVersionName, versionSuffix) + } + // Split on spaces, the scanPropeties, so that each property is available as a single string // instead of all properties being part of a single string config.ScanProperties = piperutils.SplitAndTrim(config.ScanProperties, " ") @@ -467,6 +544,44 @@ func addDetectArgs(args []string, config detectExecuteScanOptions, utils detectU return args, nil } +func addDetectArgsImages(args []string, config detectExecuteScanOptions, utils detectUtils, sys *blackduckSystem, imageTar string) ([]string, error) { + // suffix := strings.Split(imageTar, ".")[0] + // In order to preserve source scan result + config.Unmap = false + args, err := addDetectArgs(args, config, utils, sys, NO_VERSION_SUFFIX, fmt.Sprintf("image-%s", strings.Split(imageTar, ".")[0])) + if err != nil { + return []string{}, err + } + + args = append(args, fmt.Sprintf("--detect.docker.tar=./%s", imageTar)) + args = append(args, "--detect.target.type=IMAGE") + // https://community.synopsys.com/s/article/Docker-image-scanning-CLI-examples-and-some-Q-As + args = append(args, "--detect.tools.excluded=DETECTOR") + args = append(args, "--detect.docker.passthrough.shared.dir.path.local=/opt/blackduck/blackduck-imageinspector/shared") + args = append(args, "--detect.docker.passthrough.shared.dir.path.imageinspector=/opt/blackduck/blackduck-imageinspector/shared") + //args = append(args, "--detect.docker.passthrough.shared.dir.path.local=/home/scanner") + //args = append(args, "--detect.docker.passthrough.shared.dir.path.imageinspector=/home/scanner") + args = append(args, fmt.Sprintf("--detect.docker.passthrough.imageinspector.service.distro.default=%s", config.ContainerDistro)) + args = append(args, "--detect.docker.passthrough.imageinspector.service.start=false") + args = append(args, "--detect.docker.passthrough.output.include.squashedimage=false") + //args = append(args, "--detect.docker.passthrough.cleanup.inspector.container=false") + //args = append(args, "--logging.level.com.synopsys=DEBUG") + //args = append(args, "--detect.diagnostic") + + switch config.ContainerDistro { + case "ubuntu": + args = append(args, "--detect.docker.passthrough.imageinspector.service.url=http://localhost:8082") + case "centos": + args = append(args, "--detect.docker.passthrough.imageinspector.service.url=http://localhost:8081") + case "alpine": + args = append(args, "--detect.docker.passthrough.imageinspector.service.url=http://localhost:8080") + default: + return nil, fmt.Errorf("unknown container distro %q", config.ContainerDistro) + } + + return args, nil +} + func getVersionName(config detectExecuteScanOptions) string { detectVersionName := config.CustomScanVersion if len(detectVersionName) > 0 { diff --git a/cmd/detectExecuteScan_generated.go b/cmd/detectExecuteScan_generated.go index 128b5ca9d1..55e9d94f0c 100644 --- a/cmd/detectExecuteScan_generated.go +++ b/cmd/detectExecuteScan_generated.go @@ -64,6 +64,12 @@ type detectExecuteScanOptions struct { NpmArguments []string `json:"npmArguments,omitempty"` PrivateModules string `json:"privateModules,omitempty"` PrivateModulesGitToken string `json:"privateModulesGitToken,omitempty"` + ScanImages bool `json:"scanImages,omitempty"` + ContainerDistro string `json:"containerDistro,omitempty" validate:"possible-values=ubuntu centos alpine"` + ContainerRegistryURL string `json:"containerRegistryUrl,omitempty"` + ContainerRegistryPassword string `json:"containerRegistryPassword,omitempty"` + ContainerRegistryUser string `json:"containerRegistryUser,omitempty"` + ImageNameTags []string `json:"imageNameTags,omitempty"` } type detectExecuteScanInflux struct { @@ -198,6 +204,8 @@ Please configure your BlackDuck server Url using the serverUrl parameter and the log.RegisterSecret(stepConfig.Token) log.RegisterSecret(stepConfig.GithubToken) log.RegisterSecret(stepConfig.PrivateModulesGitToken) + log.RegisterSecret(stepConfig.ContainerRegistryPassword) + log.RegisterSecret(stepConfig.ContainerRegistryUser) if len(GeneralConfig.HookConfig.SentryConfig.Dsn) > 0 { sentryHook := log.NewSentryHook(GeneralConfig.HookConfig.SentryConfig.Dsn, GeneralConfig.CorrelationID) @@ -310,6 +318,12 @@ func addDetectExecuteScanFlags(cmd *cobra.Command, stepConfig *detectExecuteScan cmd.Flags().StringSliceVar(&stepConfig.NpmArguments, "npmArguments", []string{}, "List of additional arguments that Detect will add at then end of the npm ls command line when Detect executes the NPM CLI Detector on an NPM project.") cmd.Flags().StringVar(&stepConfig.PrivateModules, "privateModules", os.Getenv("PIPER_privateModules"), "Tells go which modules shall be considered to be private (by setting [GOPRIVATE](https://pkg.go.dev/cmd/go#hdr-Configuration_for_downloading_non_public_code)).") cmd.Flags().StringVar(&stepConfig.PrivateModulesGitToken, "privateModulesGitToken", os.Getenv("PIPER_privateModulesGitToken"), "GitHub personal access token as per https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line.") + cmd.Flags().BoolVar(&stepConfig.ScanImages, "scanImages", false, "If images found in the cpe, they will also be scanned") + cmd.Flags().StringVar(&stepConfig.ContainerDistro, "containerDistro", os.Getenv("PIPER_containerDistro"), "Distro of the container that is scanned") + cmd.Flags().StringVar(&stepConfig.ContainerRegistryURL, "containerRegistryUrl", os.Getenv("PIPER_containerRegistryUrl"), "http(s) url of the Container registry where the image should be pushed to - will be used instead of parameter `containerImage`") + cmd.Flags().StringVar(&stepConfig.ContainerRegistryPassword, "containerRegistryPassword", os.Getenv("PIPER_containerRegistryPassword"), "For `buildTool: docker`: Password for container registry access - typically provided by the CI/CD environment.") + cmd.Flags().StringVar(&stepConfig.ContainerRegistryUser, "containerRegistryUser", os.Getenv("PIPER_containerRegistryUser"), "For `buildTool: docker`: Username for container registry access - typically provided by the CI/CD environment.") + cmd.Flags().StringSliceVar(&stepConfig.ImageNameTags, "imageNameTags", []string{}, "For `buildTool: docker`: image name and tags of the image to be scanned.") cmd.MarkFlagRequired("token") cmd.MarkFlagRequired("projectName") @@ -773,11 +787,100 @@ func detectExecuteScanMetadata() config.StepData { Aliases: []config.Alias{}, Default: os.Getenv("PIPER_privateModulesGitToken"), }, + { + Name: "scanImages", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "bool", + Mandatory: false, + Aliases: []config.Alias{}, + Default: false, + }, + { + Name: "containerDistro", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: os.Getenv("PIPER_containerDistro"), + }, + { + Name: "containerRegistryUrl", + ResourceRef: []config.ResourceReference{ + { + Name: "commonPipelineEnvironment", + Param: "container/registryUrl", + }, + }, + Scope: []string{"GENERAL", "PARAMETERS", "STAGES", "STEPS"}, + Type: "string", + Mandatory: false, + Aliases: []config.Alias{{Name: "dockerRegistryUrl"}}, + Default: os.Getenv("PIPER_containerRegistryUrl"), + }, + { + Name: "containerRegistryPassword", + ResourceRef: []config.ResourceReference{ + { + Name: "commonPipelineEnvironment", + Param: "container/repositoryPassword", + }, + + { + Name: "commonPipelineEnvironment", + Param: "custom/repositoryPassword", + }, + }, + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: os.Getenv("PIPER_containerRegistryPassword"), + }, + { + Name: "containerRegistryUser", + ResourceRef: []config.ResourceReference{ + { + Name: "commonPipelineEnvironment", + Param: "container/repositoryUsername", + }, + + { + Name: "commonPipelineEnvironment", + Param: "custom/repositoryUsername", + }, + }, + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: os.Getenv("PIPER_containerRegistryUser"), + }, + { + Name: "imageNameTags", + ResourceRef: []config.ResourceReference{ + { + Name: "commonPipelineEnvironment", + Param: "container/imageNameTags", + }, + }, + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "[]string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: []string{}, + }, }, }, Containers: []config.Container{ {Name: "openjdk", Image: "openjdk:11", WorkingDir: "/root", Options: []config.Option{{Name: "-u", Value: "0"}}}, }, + Sidecars: []config.Container{ + {Name: "inspector-ubuntu", Image: "blackducksoftware/blackduck-imageinspector-ubuntu:5.1.0", Conditions: []config.Condition{{ConditionRef: "strings-equal", Params: []config.Param{{Name: "containerDistro", Value: "ubuntu"}}}}}, + {Name: "inspector-alpine", Image: "blackducksoftware/blackduck-imageinspector-alpine:5.1.0", Conditions: []config.Condition{{ConditionRef: "strings-equal", Params: []config.Param{{Name: "containerDistro", Value: "alpine"}}}}}, + {Name: "inspector-centos", Image: "blackducksoftware/blackduck-imageinspector-centos:5.1.0", Conditions: []config.Condition{{ConditionRef: "strings-equal", Params: []config.Param{{Name: "containerDistro", Value: "centos"}}}}}, + }, Outputs: config.StepOutputs{ Resources: []config.StepResources{ { diff --git a/resources/metadata/detectExecuteScan.yaml b/resources/metadata/detectExecuteScan.yaml index d03dd4b8a7..772b336f26 100644 --- a/resources/metadata/detectExecuteScan.yaml +++ b/resources/metadata/detectExecuteScan.yaml @@ -511,6 +511,74 @@ spec: - type: vaultSecret name: golangPrivateModulesGitTokenVaultSecret default: golang + - name: scanImages + description: If images found in the cpe, they will also be scanned + type: "bool" + scope: + - PARAMETERS + - STAGES + - STEPS + default: false + - name: containerDistro + description: Distro of the container that is scanned + type: "string" + scope: + - PARAMETERS + - STAGES + - STEPS + possibleValues: + - ubuntu + - centos + - alpine + - name: containerRegistryUrl + aliases: + - name: dockerRegistryUrl + type: string + description: http(s) url of the Container registry where the image should be pushed to - will be used instead of parameter `containerImage` + scope: + - GENERAL + - PARAMETERS + - STAGES + - STEPS + resourceRef: + - name: commonPipelineEnvironment + param: container/registryUrl + - name: containerRegistryPassword + description: "For `buildTool: docker`: Password for container registry access - typically provided by the CI/CD environment." + type: string + scope: + - PARAMETERS + - STAGES + - STEPS + secret: true + resourceRef: + - name: commonPipelineEnvironment + param: container/repositoryPassword + - name: commonPipelineEnvironment + param: custom/repositoryPassword + - name: containerRegistryUser + description: "For `buildTool: docker`: Username for container registry access - typically provided by the CI/CD environment." + type: string + scope: + - PARAMETERS + - STAGES + - STEPS + secret: true + resourceRef: + - name: commonPipelineEnvironment + param: container/repositoryUsername + - name: commonPipelineEnvironment + param: custom/repositoryUsername + - name: imageNameTags + description: "For `buildTool: docker`: image name and tags of the image to be scanned." + type: "[]string" + scope: + - PARAMETERS + - STAGES + - STEPS + resourceRef: + - name: commonPipelineEnvironment + param: container/imageNameTags outputs: resources: - name: influx @@ -562,3 +630,40 @@ spec: options: - name: -u value: "0" + volumeMounts: + - mountPath: /opt/blackduck/blackduck-imageinspector/shared + name: volume + sidecars: + - image: blackducksoftware/blackduck-imageinspector-ubuntu:5.1.0 + name: inspector-ubuntu + command: [''] + volumeMounts: + - mountPath: /opt/blackduck/blackduck-imageinspector/shared + name: volume + conditions: + - conditionRef: strings-equal + params: + - name: containerDistro + value: ubuntu + - image: blackducksoftware/blackduck-imageinspector-alpine:5.1.0 + name: inspector-alpine + command: [''] + volumeMounts: + - mountPath: /opt/blackduck/blackduck-imageinspector/shared + name: volume + conditions: + - conditionRef: strings-equal + params: + - name: containerDistro + value: alpine + - image: blackducksoftware/blackduck-imageinspector-centos:5.1.0 + name: inspector-centos + command: [''] + volumeMounts: + - mountPath: /opt/blackduck/blackduck-imageinspector/shared + name: volume + conditions: + - conditionRef: strings-equal + params: + - name: containerDistro + value: centos