From b7edd8ba70dca3d4c125f6c310e74590a34d1b7e Mon Sep 17 00:00:00 2001 From: lirshindalman Date: Tue, 24 Dec 2024 13:15:11 +0200 Subject: [PATCH 01/10] . --- checkov/common/goget/github/get_git.py | 10 ++++++---- checkov/common/proxy/__init__.py | 0 .../module_loading => common/proxy}/proxy_client.py | 9 +++++++++ .../module_loading/loaders/registry_loader.py | 2 +- 4 files changed, 16 insertions(+), 5 deletions(-) create mode 100644 checkov/common/proxy/__init__.py rename checkov/{terraform/module_loading => common/proxy}/proxy_client.py (75%) diff --git a/checkov/common/goget/github/get_git.py b/checkov/common/goget/github/get_git.py index 0368490c653..27923693d0c 100644 --- a/checkov/common/goget/github/get_git.py +++ b/checkov/common/goget/github/get_git.py @@ -5,6 +5,7 @@ import shutil from checkov.common.goget.base_getter import BaseGetter +from checkov.common.proxy.proxy_client import get_proxy_envs from checkov.common.resource_code_logger_filter import add_resource_code_filter_to_logger from checkov.common.util.contextmanagers import temp_environ @@ -82,16 +83,17 @@ def do_get(self) -> str: def _clone(self, git_url: str, clone_dir: str) -> None: self.logger.debug(f"cloning {self.url if '@' not in self.url else self.url.split('@')[1]} to {clone_dir}") + proxy_env = get_proxy_envs() with temp_environ(GIT_TERMINAL_PROMPT="0"): # disables user prompts originating from GIT if self.branch: - Repo.clone_from(git_url, clone_dir, branch=self.branch, depth=1) # depth=1 for shallow clone + Repo.clone_from(git_url, clone_dir, branch=self.branch, depth=1, env=proxy_env) # depth=1 for shallow clone elif self.commit_id: # no commit id support for branch - repo = Repo.clone_from(git_url, clone_dir, no_checkout=True) # need to be a full git clone + repo = Repo.clone_from(git_url, clone_dir, no_checkout=True, env=proxy_env) # need to be a full git clone repo.git.checkout(self.commit_id) elif self.tag: - Repo.clone_from(git_url, clone_dir, depth=1, b=self.tag) + Repo.clone_from(git_url, clone_dir, depth=1, b=self.tag, env=proxy_env) else: - Repo.clone_from(git_url, clone_dir, depth=1) + Repo.clone_from(git_url, clone_dir, depth=1, env=proxy_env) # Split source url into Git url and subdirectory path e.g. test.com/repo//repo/subpath becomes 'test.com/repo', '/repo/subpath') # Also see reference implementation @ go-getter https://github.com/hashicorp/go-getter/blob/main/source.go diff --git a/checkov/common/proxy/__init__.py b/checkov/common/proxy/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/checkov/terraform/module_loading/proxy_client.py b/checkov/common/proxy/proxy_client.py similarity index 75% rename from checkov/terraform/module_loading/proxy_client.py rename to checkov/common/proxy/proxy_client.py index 915047d345b..e3eba3f4c27 100644 --- a/checkov/terraform/module_loading/proxy_client.py +++ b/checkov/common/proxy/proxy_client.py @@ -31,3 +31,12 @@ def send_request(self, request: requests.Request) -> requests.Response: def call_http_request_with_proxy(request: requests.Request) -> Any: proxy_client = ProxyClient() return proxy_client.send_request(request=request) + + +def get_proxy_envs(): + proxy_env = os.environ.copy() + if os.getenv('PROXY_URL'): + proxy_env["GIT_SSL_CAINFO"] = os.getenv('PROXY_CA_PATH', None) # Path to the CA cert + proxy_env["http_proxy"] = os.getenv('PROXY_URL') # Proxy URL + proxy_env["https_proxy"] = os.getenv('PROXY_URL') # HTTPS Proxy URL (if needed) + return proxy_env diff --git a/checkov/terraform/module_loading/loaders/registry_loader.py b/checkov/terraform/module_loading/loaders/registry_loader.py index 61e1ee93f68..b9267479d40 100644 --- a/checkov/terraform/module_loading/loaders/registry_loader.py +++ b/checkov/terraform/module_loading/loaders/registry_loader.py @@ -19,7 +19,7 @@ order_versions_in_descending_order, get_version_constraints ) -from checkov.terraform.module_loading.proxy_client import call_http_request_with_proxy +from checkov.common.proxy.proxy_client import call_http_request_with_proxy if TYPE_CHECKING: from checkov.terraform.module_loading.module_params import ModuleParams From fdcaf1a1ba2a630004e2980b7fc5b38420a56458 Mon Sep 17 00:00:00 2001 From: lirshindalman Date: Tue, 24 Dec 2024 13:35:17 +0200 Subject: [PATCH 02/10] . --- checkov/common/proxy/proxy_client.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/checkov/common/proxy/proxy_client.py b/checkov/common/proxy/proxy_client.py index e3eba3f4c27..b72725e7e4e 100644 --- a/checkov/common/proxy/proxy_client.py +++ b/checkov/common/proxy/proxy_client.py @@ -1,5 +1,5 @@ import os -from typing import Any +from typing import Any, Mapping import requests @@ -33,10 +33,11 @@ def call_http_request_with_proxy(request: requests.Request) -> Any: return proxy_client.send_request(request=request) -def get_proxy_envs(): - proxy_env = os.environ.copy() +def get_proxy_envs() -> Mapping[str, str] | None: if os.getenv('PROXY_URL'): - proxy_env["GIT_SSL_CAINFO"] = os.getenv('PROXY_CA_PATH', None) # Path to the CA cert + proxy_env = os.environ.copy() + proxy_env["GIT_SSL_CAINFO"] = os.getenv('PROXY_CA_PATH') # Path to the CA cert proxy_env["http_proxy"] = os.getenv('PROXY_URL') # Proxy URL proxy_env["https_proxy"] = os.getenv('PROXY_URL') # HTTPS Proxy URL (if needed) - return proxy_env + return proxy_env + return None From db248daf60d6ffeefecf36c4ada9bc96eeb6a46d Mon Sep 17 00:00:00 2001 From: lirshindalman Date: Tue, 24 Dec 2024 14:16:41 +0200 Subject: [PATCH 03/10] . --- checkov/common/proxy/proxy_client.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/checkov/common/proxy/proxy_client.py b/checkov/common/proxy/proxy_client.py index b72725e7e4e..1f5c7b1fc1d 100644 --- a/checkov/common/proxy/proxy_client.py +++ b/checkov/common/proxy/proxy_client.py @@ -3,17 +3,19 @@ import requests +from checkov.common.util.env_vars_config import env_vars_config + class ProxyClient: def __init__(self) -> None: - self.proxy_ca_path = os.getenv('PROXY_CA_PATH', None) + self.proxy_ca_path = env_vars_config.PROXY_CA_PATH if self.proxy_ca_path is None: raise Exception("[ProxyClient] CA certificate path is missing") def get_session(self) -> requests.Session: - if not os.getenv('PROXY_URL', None): + if not env_vars_config.PROXY_URL: raise Exception('Please provide "PROXY_URL" env var') - proxy_url = os.getenv('PROXY_URL') + proxy_url = env_vars_config.PROXY_URL session = requests.Session() proxies = { "http": proxy_url, @@ -34,10 +36,9 @@ def call_http_request_with_proxy(request: requests.Request) -> Any: def get_proxy_envs() -> Mapping[str, str] | None: + proxy_env = os.environ.copy() if os.getenv('PROXY_URL'): - proxy_env = os.environ.copy() - proxy_env["GIT_SSL_CAINFO"] = os.getenv('PROXY_CA_PATH') # Path to the CA cert - proxy_env["http_proxy"] = os.getenv('PROXY_URL') # Proxy URL - proxy_env["https_proxy"] = os.getenv('PROXY_URL') # HTTPS Proxy URL (if needed) - return proxy_env - return None + proxy_env["GIT_SSL_CAINFO"] = env_vars_config.PROXY_CA_PATH # Path to the CA cert + proxy_env["http_proxy"] = env_vars_config.PROXY_URL # Proxy URL + proxy_env["https_proxy"] = env_vars_config.PROXY_URL # HTTPS Proxy URL (if needed) + return proxy_env From 7f617590546666ebdf3f294d08bfded6d5b2e638 Mon Sep 17 00:00:00 2001 From: lirshindalman Date: Tue, 24 Dec 2024 14:18:35 +0200 Subject: [PATCH 04/10] . --- checkov/common/proxy/proxy_client.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/checkov/common/proxy/proxy_client.py b/checkov/common/proxy/proxy_client.py index 1f5c7b1fc1d..4afdaf0db86 100644 --- a/checkov/common/proxy/proxy_client.py +++ b/checkov/common/proxy/proxy_client.py @@ -36,9 +36,10 @@ def call_http_request_with_proxy(request: requests.Request) -> Any: def get_proxy_envs() -> Mapping[str, str] | None: - proxy_env = os.environ.copy() if os.getenv('PROXY_URL'): + proxy_env = os.environ.copy() proxy_env["GIT_SSL_CAINFO"] = env_vars_config.PROXY_CA_PATH # Path to the CA cert proxy_env["http_proxy"] = env_vars_config.PROXY_URL # Proxy URL proxy_env["https_proxy"] = env_vars_config.PROXY_URL # HTTPS Proxy URL (if needed) - return proxy_env + return proxy_env + return None From e0ac6590ad40061563b189d1f271c0ce8798c5e1 Mon Sep 17 00:00:00 2001 From: lirshindalman Date: Tue, 24 Dec 2024 14:19:28 +0200 Subject: [PATCH 05/10] . --- checkov/common/proxy/proxy_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/checkov/common/proxy/proxy_client.py b/checkov/common/proxy/proxy_client.py index 4afdaf0db86..de7d0a31c60 100644 --- a/checkov/common/proxy/proxy_client.py +++ b/checkov/common/proxy/proxy_client.py @@ -1,5 +1,5 @@ import os -from typing import Any, Mapping +from typing import Any import requests @@ -35,7 +35,7 @@ def call_http_request_with_proxy(request: requests.Request) -> Any: return proxy_client.send_request(request=request) -def get_proxy_envs() -> Mapping[str, str] | None: +def get_proxy_envs() -> dict[str, str] | None: if os.getenv('PROXY_URL'): proxy_env = os.environ.copy() proxy_env["GIT_SSL_CAINFO"] = env_vars_config.PROXY_CA_PATH # Path to the CA cert From fbbfe967cd8d74798235242c01c9b9d0948d1667 Mon Sep 17 00:00:00 2001 From: lirshindalman Date: Tue, 24 Dec 2024 14:21:57 +0200 Subject: [PATCH 06/10] . --- checkov/common/proxy/proxy_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/checkov/common/proxy/proxy_client.py b/checkov/common/proxy/proxy_client.py index de7d0a31c60..433ad37dc9d 100644 --- a/checkov/common/proxy/proxy_client.py +++ b/checkov/common/proxy/proxy_client.py @@ -38,7 +38,7 @@ def call_http_request_with_proxy(request: requests.Request) -> Any: def get_proxy_envs() -> dict[str, str] | None: if os.getenv('PROXY_URL'): proxy_env = os.environ.copy() - proxy_env["GIT_SSL_CAINFO"] = env_vars_config.PROXY_CA_PATH # Path to the CA cert + proxy_env["GIT_SSL_CAINFO"] = env_vars_config.PROXY_CA_PATH # Path to the CA cert proxy_env["http_proxy"] = env_vars_config.PROXY_URL # Proxy URL proxy_env["https_proxy"] = env_vars_config.PROXY_URL # HTTPS Proxy URL (if needed) return proxy_env From f4ff4f0eb7c52c8d844a79d3c46df8eecb613c4b Mon Sep 17 00:00:00 2001 From: lirshindalman Date: Tue, 24 Dec 2024 14:28:10 +0200 Subject: [PATCH 07/10] . --- checkov/common/proxy/proxy_client.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/checkov/common/proxy/proxy_client.py b/checkov/common/proxy/proxy_client.py index 433ad37dc9d..96d7746acc7 100644 --- a/checkov/common/proxy/proxy_client.py +++ b/checkov/common/proxy/proxy_client.py @@ -1,5 +1,7 @@ +from __future__ import annotations + import os -from typing import Any +from typing import Any, Optional import requests @@ -21,7 +23,7 @@ def get_session(self) -> requests.Session: "http": proxy_url, "https": proxy_url, } - session.proxies.update(proxies) # type: ignore + session.proxies.update(proxies) return session def send_request(self, request: requests.Request) -> requests.Response: @@ -35,7 +37,7 @@ def call_http_request_with_proxy(request: requests.Request) -> Any: return proxy_client.send_request(request=request) -def get_proxy_envs() -> dict[str, str] | None: +def get_proxy_envs() -> Optional[dict[str, Optional[str]]]: if os.getenv('PROXY_URL'): proxy_env = os.environ.copy() proxy_env["GIT_SSL_CAINFO"] = env_vars_config.PROXY_CA_PATH # Path to the CA cert From ce42d545af21bbfdf5c8fc38ea4ca9b51dfc1f93 Mon Sep 17 00:00:00 2001 From: lirshindalman Date: Wed, 1 Jan 2025 11:44:52 +0200 Subject: [PATCH 08/10] remove git clone support --- checkov/common/goget/github/get_git.py | 10 ++++---- checkov/common/proxy/proxy_client.py | 23 ++++++++----------- checkov/common/util/env_vars_config.py | 2 ++ .../module_loading/loaders/registry_loader.py | 2 +- 4 files changed, 17 insertions(+), 20 deletions(-) diff --git a/checkov/common/goget/github/get_git.py b/checkov/common/goget/github/get_git.py index 27923693d0c..0368490c653 100644 --- a/checkov/common/goget/github/get_git.py +++ b/checkov/common/goget/github/get_git.py @@ -5,7 +5,6 @@ import shutil from checkov.common.goget.base_getter import BaseGetter -from checkov.common.proxy.proxy_client import get_proxy_envs from checkov.common.resource_code_logger_filter import add_resource_code_filter_to_logger from checkov.common.util.contextmanagers import temp_environ @@ -83,17 +82,16 @@ def do_get(self) -> str: def _clone(self, git_url: str, clone_dir: str) -> None: self.logger.debug(f"cloning {self.url if '@' not in self.url else self.url.split('@')[1]} to {clone_dir}") - proxy_env = get_proxy_envs() with temp_environ(GIT_TERMINAL_PROMPT="0"): # disables user prompts originating from GIT if self.branch: - Repo.clone_from(git_url, clone_dir, branch=self.branch, depth=1, env=proxy_env) # depth=1 for shallow clone + Repo.clone_from(git_url, clone_dir, branch=self.branch, depth=1) # depth=1 for shallow clone elif self.commit_id: # no commit id support for branch - repo = Repo.clone_from(git_url, clone_dir, no_checkout=True, env=proxy_env) # need to be a full git clone + repo = Repo.clone_from(git_url, clone_dir, no_checkout=True) # need to be a full git clone repo.git.checkout(self.commit_id) elif self.tag: - Repo.clone_from(git_url, clone_dir, depth=1, b=self.tag, env=proxy_env) + Repo.clone_from(git_url, clone_dir, depth=1, b=self.tag) else: - Repo.clone_from(git_url, clone_dir, depth=1, env=proxy_env) + Repo.clone_from(git_url, clone_dir, depth=1) # Split source url into Git url and subdirectory path e.g. test.com/repo//repo/subpath becomes 'test.com/repo', '/repo/subpath') # Also see reference implementation @ go-getter https://github.com/hashicorp/go-getter/blob/main/source.go diff --git a/checkov/common/proxy/proxy_client.py b/checkov/common/proxy/proxy_client.py index 96d7746acc7..fda49bce861 100644 --- a/checkov/common/proxy/proxy_client.py +++ b/checkov/common/proxy/proxy_client.py @@ -1,7 +1,7 @@ from __future__ import annotations -import os -from typing import Any, Optional +import logging +from typing import Any import requests @@ -10,13 +10,14 @@ class ProxyClient: def __init__(self) -> None: + self.identity = env_vars_config.PROXY_HEADER_VALUE self.proxy_ca_path = env_vars_config.PROXY_CA_PATH if self.proxy_ca_path is None: - raise Exception("[ProxyClient] CA certificate path is missing") + logging.warning("[ProxyClient] CA certificate path is missing") def get_session(self) -> requests.Session: if not env_vars_config.PROXY_URL: - raise Exception('Please provide "PROXY_URL" env var') + logging.warning('Please provide "PROXY_URL" env var') proxy_url = env_vars_config.PROXY_URL session = requests.Session() proxies = { @@ -26,8 +27,13 @@ def get_session(self) -> requests.Session: session.proxies.update(proxies) return session + def update_request_header(self, request: requests.Request) -> None: + if env_vars_config.PROXY_HEADER_VALUE: + request.headers[env_vars_config.PROXY_HEADER_VALUE] = self.identity + def send_request(self, request: requests.Request) -> requests.Response: session = self.get_session() + self.update_request_header(request=request) prepared_request = session.prepare_request(request) return session.send(prepared_request, verify=self.proxy_ca_path) @@ -36,12 +42,3 @@ def call_http_request_with_proxy(request: requests.Request) -> Any: proxy_client = ProxyClient() return proxy_client.send_request(request=request) - -def get_proxy_envs() -> Optional[dict[str, Optional[str]]]: - if os.getenv('PROXY_URL'): - proxy_env = os.environ.copy() - proxy_env["GIT_SSL_CAINFO"] = env_vars_config.PROXY_CA_PATH # Path to the CA cert - proxy_env["http_proxy"] = env_vars_config.PROXY_URL # Proxy URL - proxy_env["https_proxy"] = env_vars_config.PROXY_URL # HTTPS Proxy URL (if needed) - return proxy_env - return None diff --git a/checkov/common/util/env_vars_config.py b/checkov/common/util/env_vars_config.py index 01028e92da3..9f10b8f0795 100644 --- a/checkov/common/util/env_vars_config.py +++ b/checkov/common/util/env_vars_config.py @@ -81,6 +81,8 @@ def __init__(self) -> None: self.JAVA_FULL_DT = os.getenv('JAVA_FULL_DT', False) self.PROXY_CA_PATH = os.getenv('PROXY_CA_PATH', None) self.PROXY_URL = os.getenv('PROXY_URL', None) + self.PROXY_HEADER_VALUE = os.getenv('PROXY_HEADER_VALUE', None) + self.PROXY_HEADER_KEY = os.getenv('PROXY_HEADER_VALUE', None) env_vars_config = EnvVarsConfig() diff --git a/checkov/terraform/module_loading/loaders/registry_loader.py b/checkov/terraform/module_loading/loaders/registry_loader.py index b9267479d40..9763c0fb753 100644 --- a/checkov/terraform/module_loading/loaders/registry_loader.py +++ b/checkov/terraform/module_loading/loaders/registry_loader.py @@ -90,7 +90,7 @@ def _load_module(self, module_params: ModuleParams) -> ModuleContent: headers={"Authorization": f"Bearer {module_params.token}"} if module_params.token else None ) if os.getenv('PROXY_URL'): - logging.info('Send request with proxy') + logging.info('Sending request with proxy') response = call_http_request_with_proxy(request) else: session = requests.Session() From 551cdeb7720d60dc1dfec43fe460d4b8ac6ec1cb Mon Sep 17 00:00:00 2001 From: lirshindalman Date: Wed, 1 Jan 2025 11:50:14 +0200 Subject: [PATCH 09/10] remove git clone support --- checkov/common/proxy/proxy_client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/checkov/common/proxy/proxy_client.py b/checkov/common/proxy/proxy_client.py index fda49bce861..cdd6bc75197 100644 --- a/checkov/common/proxy/proxy_client.py +++ b/checkov/common/proxy/proxy_client.py @@ -41,4 +41,3 @@ def send_request(self, request: requests.Request) -> requests.Response: def call_http_request_with_proxy(request: requests.Request) -> Any: proxy_client = ProxyClient() return proxy_client.send_request(request=request) - From a108b314bf428c50ca072a6b22f56f70b8f9b439 Mon Sep 17 00:00:00 2001 From: lirshindalman Date: Wed, 1 Jan 2025 12:04:23 +0200 Subject: [PATCH 10/10] remove git clone support --- checkov/common/proxy/proxy_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/checkov/common/proxy/proxy_client.py b/checkov/common/proxy/proxy_client.py index cdd6bc75197..b63e4c6830f 100644 --- a/checkov/common/proxy/proxy_client.py +++ b/checkov/common/proxy/proxy_client.py @@ -24,7 +24,7 @@ def get_session(self) -> requests.Session: "http": proxy_url, "https": proxy_url, } - session.proxies.update(proxies) + session.proxies.update(proxies) # type: ignore return session def update_request_header(self, request: requests.Request) -> None: