feat: OpenAPI generator for UCC

docs/index.md

@@ -27,6 +27,7 @@ which is a new UI framework based on React.
 
 * Generate UI (`appserver` folder)
 * Generate Python REST handlers to support UI CRUD operations (`bin` folder)
+* Generate OpenAPI description document (`static/openapi.json` file)
 * Generate UI-related `.conf` files (`web.conf`, `restmap.conf`)
 * Generate `README` folder (with `.conf.spec` files)
 * Generate other `.conf` files (`inputs.conf`)

docs/openapi.md

@@ -0,0 +1,61 @@
+# OpenAPI description document
+
+`ucc-gen` command is executed with `--openapi` argument enabled by default.
+There have to be defined valid `globalConfi.json` and `app.manifest` to have OpenAPI description document (`static/openapi.json` file) generated.
+
+To disable OpenAPI functionality for `ucc-gen` run, call it with `--openapi=0` argument.
+
+## How to find the document?
+
+Once `ucc-gen` command is executed, OpenAPI description document is located in output `static` subdirectory.
+
+When add-on is installed to Splunk instance, it is exposed via web and management interface, so is available under following addresses accordingly:
+
+* \[protocol\]://\[domain\]:\[port\]/en-GB/splunkd/__raw/servicesNS/\[user\]/\[appname\]/static/openapi.json
+
+(eg. http://localhost:8000/en-GB/splunkd/__raw/servicesNS/admin/Splunk_TA_cisco_meraki/static/openapi.json)
+
+* https://\[domain\]:\[port\]/servicesNS/\[user\]/\[appname\]/static/openapi.json
+
+(eg. https://localhost:8089/servicesNS/admin/Splunk_TA_cisco_meraki/static/openapi.json)
+
+All security rules are applied so user has to be authenticated and authorised to be able to have access to the document.
+
+See the following resources for more information on working with the Splunk REST API (eg. how to authenticate):
+
+* [REST API User Manual](http://docs.splunk.com/Documentation/Splunk/9.0.3/RESTUM/RESTusing)
+* [REST API Tutorials](http://docs.splunk.com/Documentation/Splunk/9.0.3/RESTTUT/RESTconfigurations)
+
+## Where it can be used?
+
+OpenAPI Description document can be used to create:
+
+* interactive documentation that generates simple curl requests to all documented endpoints (check [this section](#how-to-get-curl-commands-and-use-them) for relevant instruction)
+* automation that uses the simple requests to create more complex solutions such as:
+    * orchestration
+    * mass load or migration
+    * automated tests
+
+Check [swagger](https://swagger.io/) or [other tools](https://github.com/OAI/OpenAPI-Specification/blob/main/IMPLEMENTATIONS.md) for more possibilities.
+
+## How to get curl commands and use them?
+
+### Prerequisites
+
+* docker running
+* Splunk with your add-on installed
+
+### Instruction
+
+1. Run in terminal: `docker run -p 8081:8080 swaggerapi/swagger-editor`
+2. Open SwaggerEditor in web browser (http://localhost:8081/) and load the OpenAPI description document (File > Import file)
+3. Check domain and port values for your Splunk instance and Authorize
+4. Select method-path pair (eg. GET - /splunk_ta_snow_settings/logging ) and "Try it out"
+5. Define parameters and "Execute"
+6. Copy curl value, paste to your terminal, ADD `-k` PARAMETER, and run
+
+> Note: Check [Swagger Editor documentation](https://swagger.io/tools/swagger-editor/) in case of any question related to the tool
+
+### Troubleshooting
+
+Are you sure you added `-k` parameter to curl command?

example/globalConfig.json

@@ -161,7 +161,7 @@
     "meta": {
         "name": "Splunk_TA_dummy_data",
         "restRoot": "Splunk_TA_dummy_data",
-        "version": "5.20.0R770b015b",
+        "version": "5.20.0R9f9c8fed",
         "displayName": "Splunk_TA_dummy_data",
         "schemaVersion": "0.0.3"
     }

mkdocs.yml

@@ -28,6 +28,7 @@ nav:
   - Components: "components.md"
   - Tabs: "tabs.md"
   - Additional packaging: "additional_packaging.md"
+  - OpenAPI: "openapi.md"
   - Modular Input Type column: "modular_input_type_column.md"
   - Custom UI Extensions:
     - Custom Hook: "custom_ui_extensions/custom_hook.md"

pyproject.toml

@@ -42,6 +42,7 @@ dunamai = "^1.9.0"
 jsonschema = "^4.4.0"
 PyYAML = "^6.0"
 cookiecutter = "^2.1.1"
+openapi3 = "^1.7.0"
 
 [tool.poetry.dev-dependencies]
 pytest = "^6.2"

splunk_add_on_ucc_framework/__init__.py

@@ -34,5 +34,6 @@ def generate(
     ta_version=None,
     outputdir=None,
     python_binary_name="python3",
+    openapi=True,
 ):
-    build.generate(source, config, ta_version, outputdir, python_binary_name)
+    build.generate(source, config, ta_version, outputdir, python_binary_name, openapi)

splunk_add_on_ucc_framework/app_manifest.py

@@ -15,6 +15,7 @@
 #
 
 import json
+from typing import Dict, List
 import warnings
 
 APP_MANIFEST_FILE_NAME = "app.manifest"
@@ -44,6 +45,19 @@ def get_title(self) -> str:
     def get_description(self) -> str:
         return self._manifest["info"]["description"]
 
+    def get_license_name(self) -> str:
+        return self._manifest["info"]["license"]["name"]
+
+    def get_license_uri(self) -> str:
+        return self._manifest["info"]["license"]["uri"]
+
+    def get_authors(self) -> List[Dict[str, str]]:
+        return self._manifest["info"]["author"]
+
+    @property
+    def manifest(self) -> Dict:
+        return self._manifest
+
     def read(self, content: str) -> None:
         try:
             self._manifest = json.loads(content)

splunk_add_on_ucc_framework/commands/build.py

@@ -14,24 +14,26 @@
 # limitations under the License.
 #
 import configparser
+import json
 import logging
 import os
 import shutil
 import sys
 from typing import Optional
 
 from jinja2 import Environment, FileSystemLoader
+from openapi3 import OpenAPI
 
 from splunk_add_on_ucc_framework import (
     __version__,
     app_conf,
-    app_manifest,
     exceptions,
     global_config_update,
     global_config_validator,
     meta_conf,
     utils,
 )
+from splunk_add_on_ucc_framework import app_manifest as app_manifest_lib
 from splunk_add_on_ucc_framework import global_config as global_config_lib
 from splunk_add_on_ucc_framework.commands.rest_builder import (
     global_config_builder_schema,
@@ -43,6 +45,10 @@
     install_python_libraries,
 )
 from splunk_add_on_ucc_framework.start_alert_build import alert_build
+from splunk_add_on_ucc_framework.commands.openapi_generator import (
+    ucc_to_oas,
+)
+
 
 logger = logging.getLogger("ucc_gen")
 
@@ -416,7 +422,12 @@ def _get_addon_version(addon_version: Optional[str]) -> str:
 
 
 def generate(
-    source, config_path, addon_version, outputdir=None, python_binary_name="python3"
+    source,
+    config_path,
+    addon_version,
+    outputdir=None,
+    python_binary_name="python3",
+    openapi=True,
 ):
     logger.info(f"ucc-gen version {__version__} is used")
     logger.info(f"Python binary name to use: {python_binary_name}")
@@ -430,21 +441,21 @@ def generate(
     os.makedirs(os.path.join(outputdir))
     logger.info(f"Cleaned out directory {outputdir}")
     app_manifest_path = os.path.abspath(
-        os.path.join(source, app_manifest.APP_MANIFEST_FILE_NAME),
+        os.path.join(source, app_manifest_lib.APP_MANIFEST_FILE_NAME),
     )
     with open(app_manifest_path) as manifest_file:
         app_manifest_content = manifest_file.read()
-    manifest = app_manifest.AppManifest()
+    app_manifest = app_manifest_lib.AppManifest()
     try:
-        manifest.read(app_manifest_content)
-    except app_manifest.AppManifestFormatException:
+        app_manifest.read(app_manifest_content)
+    except app_manifest_lib.AppManifestFormatException:
         logger.error(
             f"Manifest file @ {app_manifest_path} has invalid format.\n"
-            f"Please refer to {app_manifest.APP_MANIFEST_WEBSITE}.\n"
+            f"Please refer to {app_manifest_lib.APP_MANIFEST_WEBSITE}.\n"
             f'Lines with comments are supported if they start with "#".\n'
         )
         sys.exit(1)
-    ta_name = manifest.get_addon_name()
+    ta_name = app_manifest.get_addon_name()
     if not config_path:
         is_global_config_yaml = False
         config_path = os.path.abspath(os.path.join(source, "..", "globalConfig.json"))
@@ -484,17 +495,18 @@ def generate(
         global_config_file = (
             "globalConfig.yaml" if is_global_config_yaml else "globalConfig.json"
         )
+        output_global_config_path = os.path.join(
+            outputdir,
+            ta_name,
+            "appserver",
+            "static",
+            "js",
+            "build",
+            global_config_file,
+        )
         shutil.copyfile(
             config_path,
-            os.path.join(
-                outputdir,
-                ta_name,
-                "appserver",
-                "static",
-                "js",
-                "build",
-                global_config_file,
-            ),
+            output_global_config_path,
         )
         ucc_lib_target = os.path.join(outputdir, ta_name, "lib")
         logger.info(f"Install add-on requirements into {ucc_lib_target} from {source}")
@@ -557,18 +569,18 @@ def generate(
         version_file.write("\n")
         version_file.write(addon_version)
 
-    manifest.update_addon_version(addon_version)
+    app_manifest.update_addon_version(addon_version)
     output_manifest_path = os.path.abspath(
-        os.path.join(outputdir, ta_name, app_manifest.APP_MANIFEST_FILE_NAME)
+        os.path.join(outputdir, ta_name, app_manifest_lib.APP_MANIFEST_FILE_NAME)
     )
     with open(output_manifest_path, "w") as manifest_file:
-        manifest_file.write(str(manifest))
+        manifest_file.write(str(app_manifest))
 
     app_config = app_conf.AppConf()
     path = os.path.join(outputdir, ta_name, "default", "app.conf")
     app_config.read(path)
     app_config.update(
-        addon_version, ta_name, manifest.get_description(), manifest.get_title()
+        addon_version, ta_name, app_manifest.get_description(), app_manifest.get_title()
     )
     with open(path, "w") as app_conf_fd:
         app_config.write(app_conf_fd)
@@ -586,3 +598,18 @@ def generate(
         from additional_packaging import additional_packaging
 
         additional_packaging(ta_name)
+
+    if os.path.isfile(config_path) and openapi:
+        logger.info("Generating OpenAPI file")
+        open_api_object = ucc_to_oas.transform(global_config, app_manifest)
+        open_api = OpenAPI(open_api_object.json)
+
+        output_openapi_folder = os.path.abspath(
+            os.path.join(outputdir, ta_name, "static")
+        )
+        output_openapi_path = os.path.join(output_openapi_folder, "openapi.json")
+        if not os.path.isdir(output_openapi_folder):
+            os.makedirs(os.path.join(output_openapi_folder))
+            logger.info(f"Creating {output_openapi_folder} folder")
+        with open(output_openapi_path, "w") as openapi_file:
+            json.dump(open_api.raw_element, openapi_file, indent=4)

splunk_add_on_ucc_framework/commands/openapi_generator/__init__.py

@@ -0,0 +1,15 @@
+#
+# Copyright 2021 Splunk Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#

splunk_add_on_ucc_framework/commands/openapi_generator/json_to_object.py

@@ -0,0 +1,52 @@
+#
+# Copyright 2021 Splunk Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from typing import Any, Dict
+
+
+class DataClasses:
+    def __init__(self, json: Dict) -> None:
+        self._json = json
+        self.__dict__.update(self._iterator(self._json))
+        #   __dict__ contains references to base object
+        #   update, do not override!
+
+    def __getattr__(self, name: str):
+        _name = f"_{name}"
+        if _name in self.__dict__:
+            return self.__dict__[_name]
+        else:
+            raise AttributeError()
+            #   hasattr expects AttributeError exception
+
+    def _iteration_manager(self, element: Any) -> Any:
+        if isinstance(element, dict):
+            return DataClasses(element)
+        elif isinstance(element, list):
+            return self._list_iterator(element)
+        else:
+            return element
+
+    def _list_iterator(self, _list: list) -> list:
+        return_list = []
+        for i in _list:
+            return_list.append(self._iteration_manager(i))
+        return return_list
+
+    def _iterator(self, json: Any):
+        d = json
+        for k, v in json.items():
+            d[k] = self._iteration_manager(v)
+        return d