diff --git a/docs/how_to_use.md b/docs/how_to_use.md index 7614cb730..62e155ade 100644 --- a/docs/how_to_use.md +++ b/docs/how_to_use.md @@ -15,7 +15,10 @@ Example of globalConfig.json and package folder can be found at Steps ----- -- Install `splunk-add-on-ucc-framework` if it is not installed. +- Use Python virtual environment: + - `python3 -m venv .venv` + - `source .venv/bin/activate` +- Install `splunk-add-on-ucc-framework`. - Run the `ucc-gen` command. - The final addon package will be generated, in the `output` folder. @@ -28,6 +31,8 @@ ucc-gen supports the following params: - ta-version - [optional] override current version of TA, default version is version specified in `globalConfig.json`. Splunkbase compatible version of SEMVER will be used by default. +- python-binary-name - [optional] Python binary name to use when + installing Python libraries. ``` pip install splunk-packaging-toolkit @@ -51,6 +56,9 @@ What ucc-gen does following table: - `lib/requirements.txt` - install Python3 compatible packages into `output//lib` + - Removes `setuptools*`, `bin*`, `pip*`, `distribute*`, `wheel*` if + they exist from `output//lib` + - Removes execute bit from every file under `output//lib` - Replace tokens in views. - Copy addon's `package/*` to `output//*` directory. - If an addon requires some additional configurations in packaging diff --git a/splunk_add_on_ucc_framework/__init__.py b/splunk_add_on_ucc_framework/__init__.py index 2074ac8d5..53e8ba724 100644 --- a/splunk_add_on_ucc_framework/__init__.py +++ b/splunk_add_on_ucc_framework/__init__.py @@ -22,9 +22,7 @@ import logging import os import shutil -import stat import sys -from pathlib import Path from defusedxml import ElementTree as defused_et from dunamai import Style, Version @@ -41,6 +39,9 @@ GlobalConfigValidator, GlobalConfigValidatorException, ) +from splunk_add_on_ucc_framework.install_python_libraries import ( + install_python_libraries, +) from splunk_add_on_ucc_framework.meta_conf import MetaConf from splunk_add_on_ucc_framework.start_alert_build import alert_build from splunk_add_on_ucc_framework.uccrestbuilder import build @@ -297,86 +298,6 @@ def replace_token(ta_name, outputdir): f.write(s) -def install_libs(path, ucc_lib_target): - """ - Install 3rd Party libraries in addon. - - Args: - parent_path (str): Path of parent directory. - ucc_lib_target (str): Target path to install libraries. - """ - - def _install_libs(requirements, ucc_target, installer="python3"): - """ - Install 3rd Party libraries using pip2/pip3 - - Args: - requirements (str): Path to requirements file. - ucc_target (str): Target path to install libraries. - installer (str): Pip version(pip2/pip3). - """ - if not os.path.exists(requirements): - logger.warning(f"Unable to find requirements file. {requirements}") - else: - if not os.path.exists(ucc_target): - os.makedirs(ucc_target) - install_cmd = ( - installer - + ' -m pip install -r "' - + requirements - + '" --no-compile --prefer-binary --ignore-installed --use-deprecated=legacy-resolver --target "' - + ucc_target - + '"' - ) - os.system(installer + " -m pip install pip --upgrade") - os.system(install_cmd) - - logger.info(f" Checking for requirements in {path}") - if os.path.exists(os.path.join(path, "lib", "requirements.txt")): - logger.info(" Uses common requirements") - _install_libs( - requirements=os.path.join(path, "lib", "requirements.txt"), - ucc_target=ucc_lib_target, - ) - elif os.path.exists( - os.path.join(os.path.abspath(os.path.join(path, os.pardir)), "requirements.txt") - ): - logger.info(" Uses common requirements") - _install_libs( - requirements=os.path.join( - os.path.abspath(os.path.join(path, os.pardir)), "requirements.txt" - ), - ucc_target=ucc_lib_target, - ) - else: - logger.info(" Not using common requirements") - - # Prevent certain packages from being included pip could be dangerous others are just wasted space - noshipdirs = ["setuptools", "bin", "pip", "distribute", "wheel"] - p = Path(ucc_lib_target) - for nsd in noshipdirs: - try: - # Glob can return FileNotFoundError exception if no match - for o in p.glob(nsd + "*"): - if o.is_dir(): - logging.info(f" removing directory {o} from output must not ship") - shutil.rmtree(o) - except FileNotFoundError: - pass - - # Remove execute bit from any object in lib - NO_USER_EXEC = ~stat.S_IEXEC - NO_GROUP_EXEC = ~stat.S_IXGRP - NO_OTHER_EXEC = ~stat.S_IXOTH - NO_EXEC = NO_USER_EXEC & NO_GROUP_EXEC & NO_OTHER_EXEC - - for o in p.rglob("*"): - if not o.is_dir() and os.access(o, os.X_OK): - logging.info(f" fixing {o} execute bit") - current_permissions = stat.S_IMODE(os.lstat(o).st_mode) - os.chmod(o, current_permissions & NO_EXEC) - - def generate_rest(ta_name, scheme, import_declare_name, outputdir): """ Build REST for Add-on. @@ -683,8 +604,9 @@ def _removeinput(path): pass -def _generate(source, config, ta_version, outputdir=None): +def _generate(source, config, ta_version, outputdir=None, python_binary_name="python3"): logger.info(f"ucc-gen version {__version__} is used") + logger.info(f"Python binary name to use: {python_binary_name}") if outputdir is None: outputdir = os.path.join(os.getcwd(), "output") if not ta_version: @@ -782,7 +704,7 @@ def _generate(source, config, ta_version, outputdir=None): ) ucc_lib_target = os.path.join(outputdir, ta_name, "lib") logger.info(f"Install add-on requirements into {ucc_lib_target} from {source}") - install_libs(source, ucc_lib_target) + install_python_libraries(logger, source, ucc_lib_target, python_binary_name) replace_token(ta_name, outputdir) @@ -806,7 +728,7 @@ def _generate(source, config, ta_version, outputdir=None): ucc_lib_target = os.path.join(outputdir, ta_name, "lib") logger.info(f"Install add-on requirements into {ucc_lib_target} from {source}") - install_libs(source, ucc_lib_target=ucc_lib_target) + install_python_libraries(logger, source, ucc_lib_target, python_binary_name) ignore_list = get_ignore_list( ta_name, os.path.abspath(os.path.join(source, PARENT_DIR, ".uccignore")) @@ -861,8 +783,14 @@ def _generate(source, config, ta_version, outputdir=None): additional_packaging(ta_name) -def generate(source="package", config=None, ta_version=None, outputdir=None): - _generate(source, config, ta_version, outputdir) +def generate( + source="package", + config=None, + ta_version=None, + outputdir=None, + python_binary_name="python3", +): + _generate(source, config, ta_version, outputdir, python_binary_name) def main(): @@ -888,5 +816,16 @@ def main(): "package such as app.manifest, app.conf, and globalConfig.json", default=None, ) + parser.add_argument( + "--python-binary-name", + type=str, + help="Python binary name to use to install requirements", + default="python3", + ) args = parser.parse_args() - _generate(args.source, args.config, args.ta_version) + _generate( + args.source, + args.config, + args.ta_version, + python_binary_name=args.python_binary_name, + ) diff --git a/splunk_add_on_ucc_framework/install_python_libraries.py b/splunk_add_on_ucc_framework/install_python_libraries.py new file mode 100644 index 000000000..dbc22483e --- /dev/null +++ b/splunk_add_on_ucc_framework/install_python_libraries.py @@ -0,0 +1,109 @@ +# +# Copyright 2021 Splunk Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import logging +import os +import shutil +import stat +from pathlib import Path +from typing import Sequence + + +def install_python_libraries( + logger: logging.Logger, path: str, ucc_lib_target: str, python_binary_name: str +): + logger.info(f" Checking for requirements in {path}") + if os.path.exists(os.path.join(path, "lib", "requirements.txt")): + logger.info(" Uses common requirements") + install_libraries( + logger, + os.path.join(path, "lib", "requirements.txt"), + ucc_lib_target, + python_binary_name, + ) + elif os.path.exists( + os.path.join(os.path.abspath(os.path.join(path, os.pardir)), "requirements.txt") + ): + logger.info(" Uses common requirements") + install_libraries( + logger, + os.path.join( + os.path.abspath(os.path.join(path, os.pardir)), "requirements.txt" + ), + ucc_lib_target, + python_binary_name, + ) + else: + logger.info(" Not using common requirements") + + packages_to_remove = ["setuptools", "bin", "pip", "distribute", "wheel"] + remove_package_from_installed_path( + logger, + ucc_lib_target, + packages_to_remove, + ) + + remove_execute_bit(logger, ucc_lib_target) + + +def install_libraries( + logger: logging.Logger, + requirements_file_path: str, + installation_path: str, + installer: str, +): + if not os.path.exists(requirements_file_path): + logger.warning(f"Unable to find requirements file: {requirements_file_path}") + else: + if not os.path.exists(installation_path): + os.makedirs(installation_path) + install_cmd = ( + installer + + ' -m pip install -r "' + + requirements_file_path + + '" --no-compile --prefer-binary --ignore-installed --use-deprecated=legacy-resolver --target "' + + installation_path + + '"' + ) + os.system(installer + " -m pip install pip --upgrade") + os.system(install_cmd) + + +def remove_package_from_installed_path( + logger: logging.Logger, installation_path: str, package_names: Sequence[str] +): + p = Path(installation_path) + try: + for package_name in package_names: + for o in p.glob(f"{package_name}*"): + if o.is_dir(): + logger.info(f" removing directory {o} from {installation_path}") + shutil.rmtree(o) + except FileNotFoundError: + pass + + +def remove_execute_bit(logger: logging.Logger, installation_path: str): + p = Path(installation_path) + no_user_exec = ~stat.S_IEXEC + no_group_exec = ~stat.S_IXGRP + no_other_exec = ~stat.S_IXOTH + no_exec = no_user_exec & no_group_exec & no_other_exec + + for o in p.rglob("*"): + if not o.is_dir() and os.access(o, os.X_OK): + logger.info(f" fixing {o} execute bit") + current_permissions = stat.S_IMODE(os.lstat(o).st_mode) + os.chmod(o, current_permissions & no_exec) diff --git a/tests/unit/test_install_python_libraries.py b/tests/unit/test_install_python_libraries.py new file mode 100644 index 000000000..e87dd0d0a --- /dev/null +++ b/tests/unit/test_install_python_libraries.py @@ -0,0 +1,97 @@ +# +# Copyright 2021 Splunk Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import os +import stat +from unittest import mock + +from splunk_add_on_ucc_framework.install_python_libraries import ( + install_libraries, + remove_execute_bit, + remove_package_from_installed_path, +) + + +@mock.patch("os.system", autospec=True) +@mock.patch("os.path.exists", autospec=True) +def test_install_libraries(mock_os_path_exists, mock_os_system): + mock_os_path_exists.return_value = True + + install_libraries( + mock.MagicMock(), + "package/lib/requirements.txt", + "/path/to/output/addon_name/lib", + "python3", + ) + + expected_install_command = ( + 'python3 -m pip install -r "package/lib/requirements.txt"' + " --no-compile --prefer-binary --ignore-installed " + '--use-deprecated=legacy-resolver --target "' + '/path/to/output/addon_name/lib"' + ) + expected_pip_update_command = "python3 -m pip install pip --upgrade" + mock_os_system.assert_has_calls( + [ + mock.call(expected_pip_update_command), + mock.call(expected_install_command), + ] + ) + + +def test_remove_package_from_installed_path(tmp_path): + tmp_lib_path = tmp_path / "lib" + tmp_lib_path.mkdir() + tmp_lib_path_foo = tmp_lib_path / "foo" + tmp_lib_path_foo.mkdir() + tmp_lib_path_foo_dist_info = tmp_lib_path / "foo.dist_info" + tmp_lib_path_foo_dist_info.mkdir() + tmp_lib_path_bar = tmp_lib_path / "bar" + tmp_lib_path_bar.mkdir() + tmp_lib_path_baz = tmp_lib_path / "baz" + tmp_lib_path_baz.mkdir() + + remove_package_from_installed_path( + mock.MagicMock(), + str(tmp_lib_path), + ["foo", "bar"], + ) + + assert not tmp_lib_path_foo.exists() + assert not tmp_lib_path_foo_dist_info.exists() + assert not tmp_lib_path_bar.exists() + assert tmp_lib_path_baz.exists() + + +def test_remove_execute_bit(tmp_path): + tmp_lib_path = tmp_path / "lib" + tmp_lib_path.mkdir() + tmp_lib_path_foo = tmp_lib_path / "foo" + tmp_lib_path_foo.mkdir() + tmp_lib_path_foo_file = tmp_lib_path_foo / "file.so" + tmp_lib_path_foo_file.write_text("binary") + tmp_lib_path_foo_file.chmod(stat.S_IEXEC) + tmp_lib_path_bar = tmp_lib_path / "bar" + tmp_lib_path_bar.mkdir() + tmp_lib_path_bar_file = tmp_lib_path_bar / "file.txt" + tmp_lib_path_bar_file.write_text("normal") + + remove_execute_bit( + mock.MagicMock(), + str(tmp_lib_path), + ) + + assert os.access(tmp_lib_path_foo_file, os.X_OK) is False + assert os.access(tmp_lib_path_bar_file, os.X_OK) is False