Merge branch 'main' into retention-policy

CHANGELOG.md

@@ -1,6 +1,40 @@
 # CHANGELOG
 
-## [Unreleased](https://github.com/bridgecrewio/checkov/compare/3.2.281...HEAD)
+## [Unreleased](https://github.com/bridgecrewio/checkov/compare/3.2.291...HEAD)
+
+## [3.2.291](https://github.com/bridgecrewio/checkov/compare/3.2.287...3.2.291) - 2024-11-12
+
+### Feature
+
+- **general:** remove specific botocore version - [#6796](https://github.com/bridgecrewio/checkov/pull/6796)
+
+### Bug Fix
+
+- **arm:** fix ARM graph block types - [#6824](https://github.com/bridgecrewio/checkov/pull/6824)
+- **dockerfile:** Handle heredoc - [#6828](https://github.com/bridgecrewio/checkov/pull/6828)
+- **sast:** filter unsupported policies - [#6833](https://github.com/bridgecrewio/checkov/pull/6833)
+
+## [3.2.287](https://github.com/bridgecrewio/checkov/compare/3.2.286...3.2.287) - 2024-11-11
+
+### Bug Fix
+
+- **graph:** fix internal checks loading when adding custom policies in cli - [#6819](https://github.com/bridgecrewio/checkov/pull/6819)
+
+## [3.2.286](https://github.com/bridgecrewio/checkov/compare/3.2.282...3.2.286) - 2024-11-10
+
+### Feature
+
+- **secrets:** Add npm detector - [#6821](https://github.com/bridgecrewio/checkov/pull/6821)
+
+### Bug Fix
+
+- **secrets:** fix empty diff scan - [#6822](https://github.com/bridgecrewio/checkov/pull/6822)
+
+## [3.2.282](https://github.com/bridgecrewio/checkov/compare/3.2.281...3.2.282) - 2024-11-07
+
+### Bug Fix
+
+- **arm:** finish variable rendering and use definitions context - [#6814](https://github.com/bridgecrewio/checkov/pull/6814)
 
 ## [3.2.281](https://github.com/bridgecrewio/checkov/compare/3.2.280...3.2.281) - 2024-11-06
 

Pipfile

@@ -43,7 +43,7 @@ types-colorama = "<0.5.0,>=0.4.3"
 # REMINDER: Update "install_requires" deps on setup.py when changing
 #
 bc-python-hcl2 = "==0.4.2"
-bc-detect-secrets = "==1.5.17"
+bc-detect-secrets = "==1.5.22"
 bc-jsonpath-ng = "==1.6.1"
 pycep-parser = "==0.5.1"
 tabulate = ">=0.9.0,<0.10.0"
@@ -52,7 +52,7 @@ termcolor=">=1.1.0,<2.4.0"
 junit-xml = ">=1.9,<2.0"
 dpath = "==2.1.3"
 pyyaml = ">=6.0.0,<7.0.0"
-boto3 = "==1.34.25"
+boto3 = "==1.35.49"
 gitpython = ">=3.1.30,<4.0.0"
 jmespath = ">=1.0.0,<2.0.0"
 tqdm = ">=4.65.0,<5.0.0"
@@ -85,8 +85,7 @@ spdx-tools = ">=0.8.0,<0.9.0"
 license-expression = ">=30.1.0,<31.0.0"
 rustworkx = ">=0.13.0,<0.14.0"
 pydantic = ">=2.0.0,<3.0.0"
-botocore = "==1.34.25"
-urllib3 = "*"
+
 
 [requires]
 python_version = "3.8"

README.md

@@ -389,7 +389,7 @@ The console output is in colour by default, to switch to a monochrome output, se
 
 #### VS Code Extension
 
-If you want to use Checkov within VS Code, give a try to the vscode extension available at [VS Code](https://marketplace.visualstudio.com/items?itemName=Bridgecrew.checkov)
+If you want to use Checkov within VS Code, give the [Prisma Cloud extension](https://marketplace.visualstudio.com/items?itemName=PrismaCloud.prisma-cloud) a try.
 
 ### Configuration using a config file
 

checkov/arm/graph_builder/graph_to_definitions.py

@@ -4,6 +4,7 @@
 from pathlib import Path
 from typing import Any, TYPE_CHECKING
 
+from checkov.arm.graph_builder.graph_components.block_types import BlockType
 from checkov.arm.utils import ArmElements
 
 if TYPE_CHECKING:
@@ -16,12 +17,11 @@ def convert_graph_vertices_to_definitions(vertices: list[ArmBlock], root_folder:
     breadcrumbs: dict[str, dict[str, Any]] = {}
     for vertex in vertices:
         block_path = vertex.path
-        arm_element = vertex.block_type
-        if arm_element == ArmElements.RESOURCES:
-            arm_definitions.setdefault(block_path, {}).setdefault(arm_element, []).append(vertex.config)
+        if vertex.block_type == BlockType.RESOURCE:
+            arm_definitions.setdefault(block_path, {}).setdefault(ArmElements.RESOURCES, []).append(vertex.config)
         else:
             element_name = vertex.name.split('/')[-1]
-            arm_definitions.setdefault(block_path, {}).setdefault(arm_element, {})[element_name] = vertex.config
+            arm_definitions.setdefault(block_path, {}).setdefault(vertex.block_type, {})[element_name] = vertex.config
 
         if vertex.breadcrumbs:
             relative_block_path = f"/{os.path.relpath(block_path, root_folder)}"

checkov/arm/graph_builder/local_graph.py

@@ -8,6 +8,7 @@
 from checkov.arm.utils import ArmElements, extract_resource_name_from_resource_id_func, \
     extract_resource_name_from_reference_func
 from checkov.arm.graph_builder.variable_rendering.renderer import ArmVariableRenderer
+from checkov.arm.graph_builder.graph_components.block_types import BlockType
 from checkov.common.graph.graph_builder import CustomAttributes, Edge
 from checkov.common.graph.graph_builder.local_graph import LocalGraph
 from checkov.common.graph.graph_builder.utils import filter_sub_keys, adjust_value
@@ -85,7 +86,7 @@ def _create_variables_vertices(self, file_path: str, variables: dict[str, dict[s
                     name=f"{file_path}/{name}",
                     config=config,
                     path=file_path,
-                    block_type=ArmElements.VARIABLES,
+                    block_type=BlockType.VARIABLE,
                     attributes=attributes,
                     id=f"{ArmElements.VARIABLES}.{name}",
                 )
@@ -109,7 +110,7 @@ def _create_parameter_vertices(self, file_path: str, parameters: dict[str, dict[
                     name=f"{file_path}/{name}",
                     config=config,
                     path=file_path,
-                    block_type=ArmElements.PARAMETERS,
+                    block_type=BlockType.PARAMETER,
                     attributes=attributes,
                     id=f"{ArmElements.PARAMETERS}.{name}",
                 )
@@ -135,7 +136,7 @@ def _create_resource_vertices(self, file_path: str, resources: list[dict[str, An
                     name=resource_name,
                     config=config,
                     path=file_path,
-                    block_type=ArmElements.RESOURCES,
+                    block_type=BlockType.RESOURCE,
                     attributes=attributes,
                     id=f"{resource_type}.{resource_name}"
                 )
@@ -195,8 +196,8 @@ def _create_implicit_edge(self, origin_vertex_index: int, resource_name: str, re
 
     def _update_resource_vertices_names(self) -> None:
         for i, vertex in enumerate(self.vertices):
-            if (vertex.block_type != ArmElements.RESOURCES or 'name' not in vertex.config or
-                    vertex.name == vertex.config['name']) or not isinstance(vertex.config['name'], str):
+            if ((vertex.block_type != BlockType.RESOURCE or 'name' not in vertex.config or vertex.name == vertex.config['name'])
+                    or not isinstance(vertex.config['name'], str)):
                 continue
 
             if PARAMETER_FUNC in vertex.name or VARIABLE_FUNC in vertex.name:
@@ -221,7 +222,7 @@ def update_vertex_config(vertex: Block, changed_attributes: list[str] | dict[str
 
         for attr in changed_attributes:
             new_value = vertex.attributes.get(attr, None)
-            if vertex.block_type == ArmElements.RESOURCES:
+            if vertex.block_type == BlockType.RESOURCE:
                 ArmLocalGraph.update_config_attribute(
                     config=vertex.config, key_to_update=attr, new_value=new_value
                 )

checkov/common/checks_infra/registry.py

@@ -33,11 +33,12 @@ def __init__(self, checks_dir: str, parser: BaseGraphCheckParser | None = None)
         super().__init__(parser)
         self.checks: list[BaseGraphCheck] = []
         self.checks_dir = checks_dir
+        self.internal_checks_dir_loaded = False
         self.logger = logging.getLogger(__name__)
         add_resource_code_filter_to_logger(self.logger)
 
     def load_checks(self) -> None:
-        if self.checks:
+        if self.checks and self.internal_checks_dir_loaded:
             # checks were previously loaded
             return
 
@@ -78,6 +79,8 @@ def _load_checks_from_dir(self, directory: str, external_check: bool) -> None:
                                 # Note the external check; used in the should_run_check logic
                                 RunnerFilter.notify_external_check(check.id)
                             self.checks.append(check)
+        if not external_check:
+            self.internal_checks_dir_loaded = True
 
     def load_external_checks(self, dir: str) -> None:
         self._load_checks_from_dir(dir, True)

checkov/common/checks_infra/solvers/attribute_solvers/base_attribute_solver.py

@@ -15,7 +15,7 @@
 
 from concurrent.futures import ThreadPoolExecutor
 
-from checkov.common.graph.graph_builder import CustomAttributes
+from checkov.common.graph.graph_builder import CustomAttributes, reserved_attributes_to_scan, wrap_reserved_attributes
 from checkov.common.graph.graph_builder.graph_components.block_types import BlockType
 from checkov.common.util.var_utils import is_terraform_variable_dependent
 from checkov.terraform.graph_builder.graph_components.block_types import BlockType as TerraformBlockType
@@ -38,7 +38,7 @@ def __init__(
     ) -> None:
         super().__init__(SolverType.ATTRIBUTE)
         self.resource_types = resource_types
-        self.attribute = attribute
+        self.attribute = attribute if attribute not in reserved_attributes_to_scan else wrap_reserved_attributes(attribute)
         self.value = value
         self.is_jsonpath_check = is_jsonpath_check
 

checkov/common/graph/graph_builder/graph_components/attribute_names.py

@@ -33,7 +33,12 @@ def props(cls: Any) -> List[str]:
     return [i for i in cls.__dict__.keys() if i[:1] != "_"]
 
 
+def wrap_reserved_attributes(attribute: str, prefix: str = '_') -> str:
+    return f"{prefix}{attribute}"
+
+
 reserved_attribute_names = props(CustomAttributes)
+reserved_attributes_to_scan = [CustomAttributes.RESOURCE_TYPE]
 
 
 class EncryptionValues(str, Enum):

checkov/dockerfile/parser.py

@@ -3,6 +3,7 @@
 from collections import OrderedDict
 from pathlib import Path
 from typing import TYPE_CHECKING
+import io
 
 from dockerfile_parse import DockerfileParser
 from dockerfile_parse.constants import COMMENT_INSTRUCTION
@@ -16,7 +17,9 @@
 
 def parse(filename: str | Path) -> tuple[dict[str, list[_Instruction]], list[str]]:
     with open(filename) as dockerfile:
-        dfp = DockerfileParser(fileobj=dockerfile)
+        content = dockerfile.read()
+        converted_content = convert_multiline_commands(content)
+        dfp = DockerfileParser(fileobj=io.StringIO(converted_content))
         return dfp_group_by_instructions(dfp)
 
 
@@ -39,3 +42,25 @@ def collect_skipped_checks(parse_result: dict[str, list[_Instruction]]) -> list[
         skipped_checks = collect_suppressions_for_context(code_lines=comment_lines)
 
     return skipped_checks
+
+
+def convert_multiline_commands(dockerfile_content: str) -> str:
+    lines = dockerfile_content.splitlines()
+    converted_lines = []
+    in_multiline = False
+    multiline_command: list[str] = []
+
+    for line in lines:
+        if line.strip().startswith('RUN <<EOF'):
+            in_multiline = True
+            continue
+        elif in_multiline and line.strip() == 'EOF':
+            in_multiline = False
+            converted_lines.append(f"RUN {' && '.join(multiline_command)}")
+            multiline_command = []
+        elif in_multiline:
+            multiline_command.append(line.strip())
+        else:
+            converted_lines.append(line)
+
+    return '\n'.join(converted_lines)

checkov/sast/engines/prisma_engine.py

@@ -301,8 +301,22 @@ def create_prisma_report(self, data: Dict[str, Any]) -> PrismaReport:
             data["imports"] = {}
         if not data.get("reachability_report"):
             data["reachability_report"] = {}
+
+        self.remove_none_conf_incidents_policies(data)
+
         return PrismaReport(**data)
 
+    @staticmethod
+    def remove_none_conf_incidents_policies(data: Dict[str, Any]) -> None:
+        remove_list = []
+        for lang, match in data.get('rule_match', dict()).items():
+            for check in match.keys():
+                if check not in bc_integration.customer_run_config_response['policyMetadata']:
+                    remove_list.append((lang, check))
+
+        for lang, check in remove_list:
+            del data['rule_match'][lang][check]
+
     def run_go_library_list_policies(self, document: Dict[str, Any]) -> SastPolicies:
         try:
             library = ctypes.cdll.LoadLibrary(self.lib_path)

checkov/secrets/runner.py

@@ -126,6 +126,7 @@ def run(
             {'name': 'IbmCosHmacDetector'},
             {'name': 'JwtTokenDetector'},
             {'name': 'MailchimpDetector'},
+            {'name': 'NpmDetector'},
             {'name': 'PrivateKeyDetector'},
             {'name': 'SlackDetector'},
             {'name': 'SoftlayerDetector'},

checkov/secrets/scan_git_history.py

@@ -207,6 +207,8 @@ def _run_scan_one_commit(commit: Commit) -> Tuple[List[RawStore], int]:
         scanned_file_count = 0
         commit_hash = commit.metadata.commit_hash
         for file_name, file_diff in commit.files.items():
+            if len(file_diff) == 0:
+                continue
             file_results = [*scan.scan_diff(file_diff)]
             if file_results:
                 logging.debug(

checkov/terraform/graph_builder/graph_components/module.py

@@ -10,6 +10,7 @@
 from checkov.terraform import TFDefinitionKey
 from checkov.terraform.graph_builder.graph_components.block_types import BlockType
 from checkov.terraform.graph_builder.graph_components.blocks import TerraformBlock
+from checkov.common.graph.graph_builder import CustomAttributes, wrap_reserved_attributes, reserved_attributes_to_scan
 from checkov.terraform.parser_functions import handle_dynamic_values
 from hcl2 import START_LINE, END_LINE
 
@@ -182,6 +183,16 @@ def _add_module(self, blocks: List[Dict[str, Dict[str, Any]]], path: TFDefinitio
                     )
                     self._add_to_blocks(module_block)
 
+    def _alter_reserved_attributes(self, attributes: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Reserved attributes (like `resource_type`) needs to be altered in order to be considered in scanning
+        """
+        updated_attributes = pickle_deepcopy(attributes)
+        for reserved_attribute in reserved_attributes_to_scan:
+            if reserved_attribute in updated_attributes:
+                updated_attributes[wrap_reserved_attributes(reserved_attribute)] = updated_attributes[reserved_attribute]
+        return updated_attributes
+
     def _add_resource(self, blocks: List[Dict[str, Dict[str, Any]]], path: TFDefinitionKeyType) -> None:
         for resource_dict in blocks:
             for resource_type, resources in resource_dict.items():
@@ -200,7 +211,8 @@ def _add_resource(self, blocks: List[Dict[str, Dict[str, Any]]], path: TFDefinit
                     provisioner = attributes.get("provisioner")
                     if provisioner:
                         self._handle_provisioner(provisioner, attributes)
-                    attributes["resource_type"] = [resource_type]
+                    attributes = self._alter_reserved_attributes(attributes)
+                    attributes[CustomAttributes.RESOURCE_TYPE] = [resource_type]
                     block_name = f"{resource_type}.{name}"
                     resource_block = TerraformBlock(
                         block_type=BlockType.RESOURCE,

checkov/version.py

@@ -1 +1 @@
-version = '3.2.282'
+version = '3.2.292'

kubernetes/requirements.txt

@@ -1 +1 @@
-checkov==3.2.282
+checkov==3.2.292

setup.py

@@ -66,7 +66,7 @@ def run(self) -> None:
     },
     install_requires=[
         "bc-python-hcl2==0.4.2",
-        "bc-detect-secrets==1.5.17",
+        "bc-detect-secrets==1.5.22",
         "bc-jsonpath-ng==1.6.1",
         "pycep-parser==0.5.1",
         "tabulate>=0.9.0,<0.10.0",
@@ -75,7 +75,7 @@ def run(self) -> None:
         "junit-xml>=1.9,<2.0",
         "dpath==2.1.3",
         "pyyaml<7.0.0,>=6.0.0",
-        "boto3==1.34.25",
+        "boto3==1.35.49",
         "gitpython>=3.1.30,<4.0.0",
         "jmespath>=1.0.0,<2.0.0",
         "tqdm<5.0.0,>=4.65.0",
@@ -107,9 +107,7 @@ def run(self) -> None:
         "spdx-tools>=0.8.0,<0.9.0",
         "license-expression<31.0.0,>=30.1.0",
         "rustworkx>=0.13.0,<0.14.0",
-        "pydantic<3.0.0,>=2.0.0",
-        "botocore==1.34.25",
-        "urllib3",
+        "pydantic<3.0.0,>=2.0.0"
     ],
     dependency_links=[],  # keep it empty, needed for pipenv-setup
     license="Apache License 2.0",