feat: add support for downloading os-dependent libraries (#963)

this feature allows to download libraries for any platform and unpack
them in a place indicated by the user (
https://splunk.atlassian.net/browse/ADDON-58852 )

docs/advanced/os-dependent_libraries.md

@@ -0,0 +1,125 @@
+This feature allows us to download and unpack libraries with appropriate binaries for the indicated operating system during the build process.
+To do this, you need to expand the **meta** section in global config with the **os-dependentLibraries** field. This field takes the following attributes:
+
+
+| Property                                               | Type    | Description                                                                                                                                                                                                                                                              | default value |
+|--------------------------------------------------------|---------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|
+| name<span class="required-asterisk">*</span>           | string  | Name of the library we want to download.                                                                                                                                                                                                                                 | -             |
+| version<span class="required-asterisk">*</span>        | string  | Specific version of given library.                                                                                                                                                                                                                                       | -             |
+| dependencies                                           | boolean | Optional parameter which determines whether the "--no-deps" flag will be used. When the value is set to true, the flag "--no-deps" is missing and specified library will be downloaded along with all dependencies. In this case dependency versions are handled by pip. | false         |
+| platform<span class="required-asterisk">*</span>       | string  | The platform for which we want to download the specified library. The value depends on the available wheels for a given library e.g. for this wheel **cryptography-41.0.5-cp37-abi3-manylinux_2_28_x86_64.whl** platform is **manylinux_2_28_x86_64**.                   | -             |
+| python_version<span class="required-asterisk">*</span> | string  | Python version compatible with the library.                                                                                                                                                                                                                              | -             |
+| target<span class="required-asterisk">*</span>         | string  | Path where the selected library will be unpacked.                                                                                                                                                                                                                        | -             |
+| os<span class="required-asterisk">*</span>             | string  | The name of the operating system for which the library is intended. Using this parameter, an appropriate insert into sys.path will be created. It takes 3 values **windows**, **linux** and **darwin**.                                                                  | -             |
+
+### Usage
+
+```
+    ...
+    "meta": {
+        "name": "<TA name>",
+        "restRoot": "<restRoot>",
+        "version": "<TA version>",
+        "displayName": "<TA display name>",
+        "schemaVersion": "<schema version>",
+        "os-dependentLibraries": [
+            {
+                "name": "cryptography",
+                "version": "41.0.5",
+                "platform": "manylinux2014_x86_64",
+                "python_version": "37",
+                "os": "linux",
+                "target": "3rdparty/linux"
+            },
+            {
+                "name": "cryptography",
+                "version": "41.0.5",
+                "platform": "win_amd64",
+                "python_version": "37",
+                "os": "windows",
+                "target": "3rdparty/windows"
+            },
+            {
+                "name": "cryptography",
+                "version": "41.0.5",
+                "dependencies": true,
+                "platform": "manylinux2014_x86_64",
+                "python_version": "37",
+                "os": "linux",
+                "target": "3rdparty/linux_with_deps"
+            },
+            {
+                "name": "cffi",
+                "version": "1.15.1",
+                "platform": "win_amd64",
+                "python_version": "37",
+                "os": "windows",
+                "target": "3rdparty/windows"
+            }
+        ]
+    }
+```
+
+### Result
+
+Running the build for the above configuration will result in the creation of the following structure:
+
+
+```
+output
+    └──<TA>
+        ├── bin
+        ...
+        └── lib
+            └── 3rdparty
+                ├── linux
+                │   ├── cryptography
+                │   └── cryptography-41.0.5.dist-info
+                ├── linux_with_deps
+                │   ├── _cffi_backend.cpython-37m-x86_64-linux-gnu.so
+                │   ├── cffi
+                │   ├── cffi-1.15.1.dist-info
+                │   ├── cryptography
+                │   ├── cryptography-41.0.5.dist-info
+                │   ├── pycparser
+                │   └── pycparser-2.21.dist-info
+                └── windows
+                    ├── _cffi_backend.cp37-win_amd64.pyd
+                    ├── cffi
+                    ├── cffi-1.15.1.dist-info
+                    ├── cryptography
+                    └── cryptography-41.0.5.dist-info
+
+```  
+
+During the build process, a python script "import_declare_test.py" will be created in **output/ta_name/bin** to manipulate system paths. 
+In each input using the specified libraries, this script must be imported. 
+Currently, three operating systems are supported: **Windows**, **Linux** and **Darwin**. 
+If, for development purposes, there is a need to create other custom manipulations on sys.path, 
+create your own script called "import_declare_test.py" and place it in the **package/bin** folder. 
+This way, when building the TA, the default script will be replaced with the one created by the developer.  
+The default script for the above configuration will look like this:
+
+```python
+import os
+import sys
+import re
+from os.path import dirname
+
+ta_name = 'demo_addon_for_splunk'
+pattern = re.compile(r'[\\/]etc[\\/]apps[\\/][^\\/]+[\\/]bin[\\/]?$')
+new_paths = [path for path in sys.path if not pattern.search(path) or ta_name in path]
+new_paths.insert(0, os.path.join(dirname(dirname(__file__)), "lib"))
+new_paths.insert(0, os.path.sep.join([os.path.dirname(__file__), ta_name]))
+sys.path = new_paths
+
+bindir = os.path.dirname(os.path.realpath(os.path.dirname(__file__)))
+libdir = os.path.join(bindir, "lib")
+platform = sys.platform
+if platform.startswith("linux"):
+	sys.path.insert(0, os.path.join(libdir, "3rdparty/linux_with_deps"))
+	sys.path.insert(0, os.path.join(libdir, "3rdparty/linux"))
+if platform.startswith("win"):
+	sys.path.insert(0, os.path.join(libdir, "3rdparty/windows"))
+
+```

mkdocs.yml

@@ -62,6 +62,7 @@ nav:
     - Custom REST Handler: "advanced/custom_rest_handler.md"
     - Groups Feature: "advanced/groups_feature.md"
     - Save Validator: "advanced/save_validator.md"
+    - OS-dependent libraries: "advanced/os-dependent_libraries.md"
   - Troubleshooting: "troubleshooting.md"
   - Contributing: "contributing.md"
   - Changelog: "CHANGELOG.md"

splunk_add_on_ucc_framework/commands/build.py

@@ -539,7 +539,11 @@ def generate(
         ucc_lib_target = os.path.join(output_directory, ta_name, "lib")
         try:
             install_python_libraries(
-                source, ucc_lib_target, python_binary_name, includes_ui=True
+                source,
+                ucc_lib_target,
+                python_binary_name,
+                includes_ui=True,
+                os_libraries=global_config.os_libraries,
             )
         except SplunktaucclibNotFound as e:
             logger.error(str(e))

splunk_add_on_ucc_framework/commands/rest_builder/builder.py

@@ -15,13 +15,14 @@
 #
 import os
 import os.path as op
-from typing import Dict, List
+from typing import Dict, List, Set
 
 from splunk_add_on_ucc_framework.commands.rest_builder import (
     global_config_builder_schema,
 )
 from splunk_add_on_ucc_framework.rest_map_conf import RestmapConf
 from splunk_add_on_ucc_framework.web_conf import WebConf
+from splunk_add_on_ucc_framework.global_config import OSDependentLibraryConfig
 
 __all__ = ["RestBuilder"]
 
@@ -40,9 +41,51 @@
 sys.path = new_paths
 """
 
+_import_declare_os_lib_content = """
+bindir = os.path.dirname(os.path.realpath(os.path.dirname(__file__)))
+libdir = os.path.join(bindir, "lib")
+platform = sys.platform
+"""
+
+
+def _generate_import_declare_test(
+    schema: global_config_builder_schema.GlobalConfigBuilderSchema,
+) -> str:
+    base_content = _import_declare_content.format(ta_name=schema.product)
+    libraries = schema.global_config.os_libraries
+    if not libraries:
+        return base_content
+
+    paths = get_paths_to_add(libraries)
+    os_lib_part = _import_declare_os_lib_content
+    for lib_os, targets in paths.items():
+        if lib_os == "windows":
+            os_lib_part += 'if platform.startswith("win"):\n'
+            for target in targets:
+                os_lib_part += get_insert_to_syspath_str(target)
+        elif lib_os == "darwin":
+            os_lib_part += 'if platform.startswith("darwin"):\n'
+            for target in targets:
+                os_lib_part += get_insert_to_syspath_str(target)
+        else:
+            os_lib_part += 'if platform.startswith("linux"):\n'
+            for target in targets:
+                os_lib_part += get_insert_to_syspath_str(target)
+
+    return base_content + os_lib_part
+
+
+def get_paths_to_add(libraries: List[OSDependentLibraryConfig]) -> Dict[str, Set[str]]:
+    result: Dict[str, Set[str]] = {}
+    for library in libraries:
+        lib_os = library.os
+        target = os.path.normpath(library.target)
+        result.setdefault(lib_os, set()).add(target)
+    return result
+
 
-def _generate_import_declare_test(addon_name: str) -> str:
-    return _import_declare_content.format(ta_name=addon_name)
+def get_insert_to_syspath_str(target: str) -> str:
+    return f'\tsys.path.insert(0, os.path.join(libdir, "{target}"))\n'
 
 
 class _RestBuilderOutput:
@@ -146,6 +189,6 @@ def build(self) -> None:
         self.output.put(
             self.output.bin,
             "import_declare_test.py",
-            _generate_import_declare_test(self._schema.product),
+            _generate_import_declare_test(self._schema),
         )
         self.output.save()

splunk_add_on_ucc_framework/global_config.py

@@ -16,6 +16,7 @@
 import functools
 import json
 from typing import Optional, Any, List, Dict
+from dataclasses import dataclass, field, fields
 
 import yaml
 
@@ -25,6 +26,33 @@
 yaml_load = functools.partial(yaml.load, Loader=Loader)
 
 
+@dataclass(frozen=True)
+class OSDependentLibraryConfig:
+    name: str
+    version: str
+    platform: str
+    python_version: str
+    target: str
+    os: str
+    deps_flag: str
+    dependencies: bool = field(default=False)
+
+    @classmethod
+    def from_dict(cls, **kwargs: Any) -> "OSDependentLibraryConfig":
+        result = {
+            dc_field.name: kwargs[dc_field.name]
+            for dc_field in fields(cls)
+            if dc_field.name in kwargs
+            and (
+                isinstance(kwargs[dc_field.name], dc_field.type)
+                or kwargs[dc_field.name] is None
+            )
+        }
+        deps_flag = "" if result.get("dependencies") else "--no-deps"
+        result.update({"deps_flag": deps_flag})
+        return cls(**result)
+
+
 class GlobalConfig:
     def __init__(self, global_config_path: str, is_global_config_yaml: bool) -> None:
         with open(global_config_path) as f_config:
@@ -107,6 +135,15 @@ def original_path(self) -> str:
     def schema_version(self) -> Optional[str]:
         return self.meta.get("schemaVersion")
 
+    @property
+    def os_libraries(self) -> Optional[List[OSDependentLibraryConfig]]:
+        if self._content["meta"].get("os-dependentLibraries"):
+            return [
+                OSDependentLibraryConfig.from_dict(**lib)
+                for lib in self._content["meta"].get("os-dependentLibraries")
+            ]
+        return None
+
     def update_schema_version(self, new_schema_version: str) -> None:
         self.meta["schemaVersion"] = new_schema_version
 

splunk_add_on_ucc_framework/install_python_libraries.py

@@ -18,8 +18,10 @@
 import shutil
 import stat
 import subprocess
+import sys
 from pathlib import Path
-from typing import Sequence
+from typing import Sequence, List, Optional
+from splunk_add_on_ucc_framework.global_config import OSDependentLibraryConfig
 
 logger = logging.getLogger("ucc_gen")
 
@@ -62,6 +64,7 @@ def install_python_libraries(
     ucc_lib_target: str,
     python_binary_name: str,
     includes_ui: bool = False,
+    os_libraries: Optional[List[OSDependentLibraryConfig]] = None,
 ) -> None:
     path_to_requirements_file = os.path.join(source_path, "lib", "requirements.txt")
     if os.path.isfile(path_to_requirements_file):
@@ -82,6 +85,12 @@ def install_python_libraries(
             ucc_lib_target,
             python_binary_name,
         )
+
+        if os_libraries:
+            install_os_dependent_libraries(
+                ucc_lib_target, python_binary_name, os_libraries
+            )
+
         packages_to_remove = (
             "setuptools",
             "bin",
@@ -152,3 +161,34 @@ def remove_execute_bit(installation_path: str) -> None:
             logger.info(f"  fixing {o} execute bit")
             current_permissions = stat.S_IMODE(os.lstat(o).st_mode)
             os.chmod(o, current_permissions & no_exec)
+
+
+def install_os_dependent_libraries(
+    ucc_lib_target: str, installer: str, os_libraries: List[OSDependentLibraryConfig]
+) -> None:
+    for package in os_libraries:
+        target_path = os.path.join(ucc_lib_target, os.path.normpath(package.target))
+
+        if not os.path.exists(target_path):
+            os.makedirs(target_path)
+
+        pip_download_command = (
+            f"{installer} "
+            f"-m pip "
+            f"install "
+            f"{package.deps_flag} "
+            f"--platform {package.platform} "
+            f"--python-version {package.python_version} "
+            f"--target {target_path}"
+            f" --only-binary=:all: "
+            f"{package.name}=={package.version}"
+        )
+
+        logger.info(f"Executing: {pip_download_command}")
+        try:
+            _subprocess_call(pip_download_command, "pip download")
+        except CouldNotInstallRequirements:
+            logger.error(
+                "Downloading process failed. Please verify parameters in the globalConfig.json file."
+            )
+            sys.exit("Package building process interrupted.")

splunk_add_on_ucc_framework/schema/schema.json

@@ -1589,6 +1589,46 @@
         "checkForUpdates": {
           "type": "boolean",
           "default": true
+        },
+        "os-dependentLibraries": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "name": {
+                "type": "string"
+              },
+              "version": {
+                "type": "string"
+              },
+              "dependencies": {
+                "type": "boolean"
+              },
+              "platform": {
+                "type": "string"
+              },
+              "python_version": {
+                "type": "string"
+              },
+              "target": {
+                "type": "string"
+              },
+              "os": {
+                "anyOf": [
+                {
+                  "const": "linux"
+                },
+                {
+                  "const": "windows"
+                },
+                {
+                  "const": "darwin"
+                }
+              ]
+              }
+            },
+            "required": ["name", "version", "platform", "python_version", "target", "os"]
+          }
         }
       },
       "required": ["displayName", "name", "restRoot", "version"],