diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..cfc8666 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,114 @@ +name: CI + +# Controls when the action will run. +on: + push: + branches: ["**"] + pull_request: + branches: [main] + release: + # A release, pre-release, or draft of a release is published. + types: [published] + # Allows you to run this workflow manually from the Actions tab. + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel. +jobs: + # The introduction just shows some useful informations. + intro: + # The type of runner that the job will run on. + runs-on: ubuntu-latest + # Steps represent a sequence of tasks that will be executed as part of the job. + steps: + - run: echo "The job was automatically triggered by a ${{ github.event_name }} event." + - run: echo "The name of the branch is ${{ github.ref }} and the repository is ${{ github.repository }}." + + lint: + # The type of runner that the job will run on. + runs-on: ubuntu-latest + needs: intro + strategy: + matrix: + python-version: ["3.9", "3.10", "3.11"] + + # Steps represent a sequence of tasks that will be executed as part of the job. + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pylint pytest pytest-cov toml + pip install . + + - name: Analysing the code with pylint + run: | + pylint ./src/pySupersetCli + + test: + # The type of runner that the job will run on. + runs-on: ubuntu-latest + needs: intro + strategy: + matrix: + python-version: ["3.9", "3.10", "3.11"] + + # Steps represent a sequence of tasks that will be executed as part of the job. + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pytest + pip install . + + - name: Test with pytest + run: | + pytest --verbose tests + + coverage: + # The type of runner that the job will run on. + runs-on: ubuntu-latest + needs: intro + strategy: + matrix: + python-version: ["3.9", "3.10", "3.11"] + + # Steps represent a sequence of tasks that will be executed as part of the job. + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pylint pytest pytest-cov toml + pip install . + + - name: Create coverage report + run: | + pytest tests -v --cov=./src/pySupersetCli --cov-report=html:coverage_report + + - name: Upload coverage report + uses: actions/upload-artifact@v4 + with: + name: coverage-report-${{ matrix.python-version }} + path: coverage_report/ diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a68c88a --- /dev/null +++ b/.gitignore @@ -0,0 +1,170 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# Exported files +*export*.json + +# Certificates +*.pem +*.crt + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +# Development Scripts +dev/ \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..23fd35f --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "editor.formatOnSave": true +} \ No newline at end of file diff --git a/.vscode/templates.code-snippets b/.vscode/templates.code-snippets new file mode 100644 index 0000000..40ea7a6 --- /dev/null +++ b/.vscode/templates.code-snippets @@ -0,0 +1,76 @@ +{ + // Place your pySupersetCli workspace snippets here. Each snippet is defined under a snippet name and has a scope, prefix, body and + // description. Add comma separated ids of the languages where the snippet is applicable in the scope field. If scope + // is left empty or omitted, the snippet gets applied to all languages. The prefix is what is + // used to trigger the snippet and the body will be expanded and inserted. Possible variables are: + // $1, $2 for tab stops, $0 for the final cursor position, and ${1:label}, ${2:another} for placeholders. + // Placeholders with the same ids are connected. + // Example: + // "Print to console": { + // "scope": "javascript,typescript", + // "prefix": "log", + // "body": [ + // "console.log('$1');", + // "$2" + // ], + // "description": "Log output to console" + // } + "Python Template": { + "prefix": "template_python", + "scope": "python", + "body": [ + "\"\"\"${1:DocString}\"\"\"", + "", + "# BSD 3-Clause License", + "#", + "# Copyright (c) 2024, NewTec GmbH", + "#", + "# Redistribution and use in source and binary forms, with or without", + "# modification, are permitted provided that the following conditions are met:", + "#", + "# 1. Redistributions of source code must retain the above copyright notice, this", + "# list of conditions and the following disclaimer.", + "#", + "# 2. Redistributions in binary form must reproduce the above copyright notice,", + "# this list of conditions and the following disclaimer in the documentation", + "# and/or other materials provided with the distribution.", + "#", + "# 3. Neither the name of the copyright holder nor the names of its", + "# contributors may be used to endorse or promote products derived from", + "# this software without specific prior written permission.", + "#", + "# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"", + "# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE", + "# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICU5LAR PURPOSE ARE", + "# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE", + "# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL", + "# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR", + "# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER", + "# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,", + "# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE", + "# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.", + "", + "################################################################################", + "# Imports", + "################################################################################", + "", + "################################################################################", + "# Variables", + "################################################################################", + "", + "################################################################################", + "# Classes", + "################################################################################", + "", + "################################################################################", + "# Functions", + "################################################################################", + "", + "################################################################################", + "# Main", + "################################################################################", + "" + ], + "description": "Python Template" + } +} \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..b12a3e2 --- /dev/null +++ b/README.md @@ -0,0 +1,62 @@ +# pySupersetCli + +CLI tool for easy usage of the Superset API. + +[![License](https://img.shields.io/badge/license-bsd-3.svg)](https://choosealicense.com/licenses/bsd-3-clause/) [![Repo Status](https://www.repostatus.org/badges/latest/wip.svg)](https://www.repostatus.org/#wip) [![CI](https://github.com/NewTec-GmbH/pySupersetCli/actions/workflows/ci.yml/badge.svg)](https://github.com/NewTec-GmbH/pySupersetCli/actions/workflows/ci.yml) + +* [Installation](#installation) +* [Overview](#overview) +* [Usage](#usage) +* [Commands](#commands) +* [Examples](#examples) +* [Used Libraries](#used-libraries) +* [Issues, Ideas And Bugs](#issues-ideas-and-bugs) +* [License](#license) +* [Contribution](#contribution) + +## Installation + +```cmd +git clone https://github.com/NewTec-GmbH/pySupersetCli.git +cd pySupersetCli +pip install . +``` + +## Overview + +WIP + +## Usage + +Show help information: + +```cmd +pySupersetCli --help +``` + +## Commands + +WIP + +## Examples + +Check out the all the [Examples](./examples) on how to use the pySupersetCli tool. + +## Used Libraries + +Used 3rd party libraries which are not part of the standard Python package: + +* [toml](https://github.com/uiri/toml) - Parsing [TOML](https://en.wikipedia.org/wiki/TOML) - MIT License + +## Issues, Ideas And Bugs + +If you have further ideas or you found some bugs, great! Create an [issue](https://github.com/NewTec-GmbH/pySupersetCli/issues) or if you are able and willing to fix it by yourself, clone the repository and create a pull request. + +## License + +The whole source code is published under [BSD-3-Clause](https://github.com/NewTec-GmbH/pySupersetCli/blob/main/LICENSE). +Consider the different licenses of the used third party libraries too! + +## Contribution + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, shall be licensed as above, without any additional terms or conditions. diff --git a/design/README.md b/design/README.md new file mode 100644 index 0000000..e69de29 diff --git a/examples/.gitkeep b/examples/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..bddb7b4 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,47 @@ +[build-system] +requires = ["setuptools", "setuptools-scm", "wheel", "toml"] +build-backend = "setuptools.build_meta" + +[project] +name = "pySupersetCli" +version = "1.0.0" +description = "CLI tool for easy usage of the Superset API." +readme = "README.md" +requires-python = ">=3.9" +authors = [ + { name = "Gabryel Reyes", email = "gabryel.reyes@newtec.de" }, + { name = "Juliane Kerpe", email = "juliane.kerpe@newtec.de" } +] +license = {text = "BSD 3-Clause"} +classifiers = [ + "License :: OSI Approved :: BSD 3-Clause", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11" +] + +dependencies = [ + "toml>=0.10.2", + "requests>=2.32.3", + "pandas>=2.2.2" +] + +[project.optional-dependencies] +test = [ + "pytest > 5.0.0", + "pytest-cov[all]" +] + +[project.urls] +documentation = "https://github.com/NewTec-GmbH/pySupersetCli" +repository = "https://github.com/NewTec-GmbH/pySupersetCli" +tracker = "https://github.com/NewTec-GmbH/pySupersetCli/issues" + +[project.scripts] +pySupersetCli = "pySupersetCli.__main__:main" + +[tool.pytest.ini_options] +pythonpath = [ + "src" +] diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..cc6a85b --- /dev/null +++ b/setup.cfg @@ -0,0 +1,45 @@ +[metadata] +name = pySupersetCli +version = attr: pySupersetCli.version.__version__ +description = CLI tool for easy usage of the Superset API. +long_description = file: README.md +long_description_content_type = text/markdown; charset=UTF-8 +url = https://github.com/NewTec-GmbH/pySupersetCli +author = Gabryel Reyes +author_email = gabryel.reyes@newtec.de +license = BSD 3-Clause +license_files = LICENSE +classifiers = + License :: OSI Approved :: BSD 3-Clause + Operating System :: OS Independent + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 +project_urls = + Documentation = https://github.com/NewTec-GmbH/pySupersetCli + Source = https://github.com/NewTec-GmbH/pySupersetCli + Tracker = https://github.com/NewTec-GmbH/pySupersetCli/issues + +[options] +package_dir= + =src +packages = find: +zip_safe = False +platforms = any +include_package_data = True +install_requires = + toml>=0.10.2 + requests>=2.32.3 + pandas>=2.2.2 +python_requires = >=3.9 +setup_requires = + setuptools_scm + wheel + toml + +[options.packages.find] +where=src + +[options.entry_points] +console_scripts = + pySupersetCli = pySupersetCli.__main__:main diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..dc16c3c --- /dev/null +++ b/setup.py @@ -0,0 +1,53 @@ +""" Tool setup """ +# BSD 3-Clause License +# +# Copyright (c) 2024, NewTec GmbH +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICU5LAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +################################################################################ +# Imports +################################################################################ +import setuptools + +################################################################################ +# Variables +################################################################################ + +################################################################################ +# Classes +################################################################################ + +################################################################################ +# Functions +################################################################################ + +################################################################################ +# Main +################################################################################ + +if __name__ == "__main__": + setuptools.setup() diff --git a/src/pySupersetCli/__init__.py b/src/pySupersetCli/__init__.py new file mode 100644 index 0000000..7def378 --- /dev/null +++ b/src/pySupersetCli/__init__.py @@ -0,0 +1,32 @@ +"""__init__""" # pylint: disable=invalid-name + +# BSD 3-Clause License +# +# Copyright (c) 2024, NewTec GmbH +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from .version import __version__, __author__, __email__, __repository__, __license__ diff --git a/src/pySupersetCli/__main__.py b/src/pySupersetCli/__main__.py new file mode 100644 index 0000000..37311b6 --- /dev/null +++ b/src/pySupersetCli/__main__.py @@ -0,0 +1,206 @@ +"""The main module with the program entry point.""" + +# BSD 3-Clause License +# +# Copyright (c) 2024, NewTec GmbH +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICU5LAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +################################################################################ +# Imports +################################################################################ + +import sys +import argparse +import logging + +from pySupersetCli.version import __version__, __author__, __email__, __repository__, __license__ +from pySupersetCli.ret import Ret +from pySupersetCli.superset import Superset +from pySupersetCli.cmd_upload import register as cmd_upload_register + + +################################################################################ +# Variables +################################################################################ + +# Register a command here! +_COMMAND_REG_LIST = [ + cmd_upload_register +] + +PROG_NAME = "pySupersetCli" +PROG_DESC = "CLI tool for easy usage of the Superset API." +PROG_COPYRIGHT = f"Copyright (c) 2024 NewTec GmbH - {__license__}" +PROG_GITHUB = f"Find the project on GitHub: {__repository__}" +PROG_EPILOG = f"{PROG_COPYRIGHT} - {PROG_GITHUB}" + +LOG: logging.Logger = logging.getLogger(__name__) + +################################################################################ +# Classes +################################################################################ + +################################################################################ +# Functions +################################################################################ + + +def add_parser() -> argparse.ArgumentParser: + """ Add parser for command line arguments and + set the execute function of each + cmd module as callback for the subparser command. + Return the parser after all the modules have been registered + and added their subparsers. + + + Returns: + argparse.ArgumentParser: The parser object for commandline arguments. + """ + parser = argparse.ArgumentParser(prog=PROG_NAME, + description=PROG_DESC, + epilog=PROG_EPILOG) + + required_arguments = parser.add_argument_group('required arguments') + + required_arguments.add_argument('-u', + '--user', + type=str, + metavar='', + required=True, + help="The user to authenticate with the Superset server.") + + required_arguments.add_argument('-p', + '--password', + type=str, + metavar='', + required=True, + help="The password to authenticate with the Superset server.") + + required_arguments.add_argument('-s', + '--server', + type=str, + metavar='', + required=True, + help="The Superset server URL to connect to.") + + parser.add_argument("--version", + action="version", + version="%(prog)s " + __version__) + + parser.add_argument("-v", + "--verbose", + action="store_true", + help="Print full command details before executing the command.\ + Enables logs of type INFO and WARNING.") + + parser.add_argument("--no_ssl", + action="store_true", + help="Disables SSL certificate verification.") + + parser.add_argument("--basic_auth", + action="store_true", + help="Use basic authentication instead of LDAP.") + + return parser + + +def main() -> Ret: + """ The program entry point function. + + Returns: + int: System exit status. + """ + ret_status = Ret.OK + commands = [] + + # Create the main parser and add the subparsers. + parser = add_parser() + subparser = parser.add_subparsers(required=True, dest="cmd") + + # Register all commands. + for cmd_register in _COMMAND_REG_LIST: + cmd_par_dict = cmd_register(subparser) + commands.append(cmd_par_dict) + + # Parse the command line arguments. + args = parser.parse_args() + + # Check if the command line arguments are valid. + if args is None: + ret_status = Ret.ERROR_ARGPARSE + parser.print_help() + else: + # If the verbose flag is set, change the default logging level. + if args.verbose: + logging.basicConfig(level=logging.INFO) + LOG.info("Program arguments: ") + for arg in vars(args): + LOG.info("* %s = %s", arg, vars(args)[arg]) + + # Create Superset client. + try: + verify_ssl = not args.no_ssl + provider = Superset.Provider.LDAP + + if args.basic_auth: + provider = Superset.Provider.DB + + # pylint: disable=unused-variable + client = Superset(args.server, + args.user, + args.password, + provider, + verify_ssl=verify_ssl) + + except RuntimeError as e: + LOG.error("Failed to create Superset client: %s", e) + ret_status = Ret.ERROR_LOGIN + else: + handler = None + + # Find the command handler. + for command in commands: + if command["name"] == args.cmd: + handler = command["handler"] + break + + # Execute the command. + if handler is not None: + ret_status = handler(args, client) + else: + LOG.error("Command '%s' not found!", args.cmd) + ret_status = Ret.ERROR_INVALID_ARGUMENTS + + return ret_status + +################################################################################ +# Main +################################################################################ + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/pySupersetCli/cmd_upload.py b/src/pySupersetCli/cmd_upload.py new file mode 100644 index 0000000..feb9ccc --- /dev/null +++ b/src/pySupersetCli/cmd_upload.py @@ -0,0 +1,172 @@ +"""Upload a JSON file to a Superset instance.""" + +# BSD 3-Clause License +# +# Copyright (c) 2024, NewTec GmbH +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICU5LAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +################################################################################ +# Imports +################################################################################ + +import os +import argparse +import logging +import json +import pandas as pd +from pySupersetCli.ret import Ret +from pySupersetCli.superset import Superset + +################################################################################ +# Variables +################################################################################ + +LOG: logging.Logger = logging.getLogger(__name__) +_CMD_NAME = "upload" +_TEMP_FILE_NAME = "./temp.csv" +DATE_COLUMN = "date" + +################################################################################ +# Classes +################################################################################ + +################################################################################ +# Functions +################################################################################ + + +def register(subparser) -> dict: + """ Register subparser commands. + + Args: + subparser (obj): the command subparser provided via __main__.py + + Returns: + obj: the command parser of this module + """ + cmd_dict: dict = { + "name": _CMD_NAME, + "handler": _execute + } + + sub_parser_search: argparse.ArgumentParser = \ + subparser.add_parser(_CMD_NAME, + help="Upload a JSON file to a Superset instance.") + + required_subarguments = sub_parser_search.add_argument_group( + 'required arguments') + + required_subarguments.add_argument('-d', + '--database', + type=int, + metavar='', + required=True, + help="The primary key of the database to " + + "upload the JSON file to.") + + required_subarguments.add_argument('-t', + '--table', + type=str, + metavar='', + required=True, + help="The name of the table to upload the JSON file to.") + + required_subarguments.add_argument('-f', + '--file', + type=str, + metavar='', + required=True, + help="The JSON input file to upload.") + + return cmd_dict + + +def _execute(args, superset_client: Superset) -> Ret: + """ This function serves as entry point for the command. + It will be stored as callback for this module's subparser command. + + Args: + args (obj): The command line arguments. + superset_client (obj): The Superset client object. + + Returns: + Ret: The status of the command execution. + """ + + return_status = Ret.OK + + if ("" != args.table) and ("" != args.file) and (None is not superset_client): + try: + if args.file.endswith(".json") is False: + raise ValueError( + "Invalid file format. Please provide a JSON file.") + + with open(args.file, encoding="utf-8") as json_file: + data_dict = json.load(json_file) + + if DATE_COLUMN not in data_dict: + raise ValueError( + "No 'date' column found in the JSON file.") + + # Pack the JSON data into a Pandas DataFrame. + data_frame = pd.DataFrame([data_dict]) + + # Write the DataFrame to a temporary CSV file. + data_frame.to_csv(_TEMP_FILE_NAME, encoding="UTF-8", index=False) + + with open(_TEMP_FILE_NAME, 'rb') as csv_file: + upload_file = {'file': csv_file} + upload_body = {'already_exists': 'append', + 'column_dates': [DATE_COLUMN], + 'table_name': args.table} + + # Upload the CSV file to the specified table + ret_code, ret_data = \ + superset_client.request("POST", + f"/database/{args.database}/csv_upload/", + data=upload_body, + files=upload_file) + + if ret_data.get("message") == "OK": + LOG.info("Upload successful.") + else: + LOG.error("Upload failed: [%d] %s", + ret_code, ret_data.get("message")) + return_status = Ret.ERROR_UPLOAD_FAILED + + if os.path.exists(_TEMP_FILE_NAME): + os.remove(_TEMP_FILE_NAME) + + except Exception as e: # pylint: disable=broad-except + LOG.error("Exception: %s", e) + return_status = Ret.ERROR_INVALID_ARGUMENTS + + return return_status + +################################################################################ +# Main +################################################################################ diff --git a/src/pySupersetCli/ret.py b/src/pySupersetCli/ret.py new file mode 100644 index 0000000..8a2bdc1 --- /dev/null +++ b/src/pySupersetCli/ret.py @@ -0,0 +1,60 @@ +"""This module contains general constants, used in all other modules.""" + +# BSD 3-Clause License +# +# Copyright (c) 2024, NewTec GmbH +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICU5LAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +################################################################################ +# Imports +################################################################################ +from enum import IntEnum +################################################################################ +# Variables +################################################################################ + +################################################################################ +# Classes +################################################################################ + + +class Ret(IntEnum): + """This type shall be used for return status information. + """ + OK = 0 + ERROR_LOGIN = 1 + ERROR_ARGPARSE = 2 # Must be 2 to match the argparse error code. + ERROR_INVALID_ARGUMENTS = 3 + ERROR_UPLOAD_FAILED = 4 + +################################################################################ +# Functions +################################################################################ + +################################################################################ +# Main +################################################################################ diff --git a/src/pySupersetCli/superset.py b/src/pySupersetCli/superset.py new file mode 100644 index 0000000..d729907 --- /dev/null +++ b/src/pySupersetCli/superset.py @@ -0,0 +1,226 @@ +"""Server wrapper for requests to the Superset API.""" + +# BSD 3-Clause License +# +# Copyright (c) 2024, NewTec GmbH +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICU5LAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +################################################################################ +# Imports +################################################################################ + +from dataclasses import dataclass +import logging +import requests +import urllib3 + + +################################################################################ +# Variables +################################################################################ + +LOG: logging.Logger = logging.getLogger(__name__) + +################################################################################ +# Classes +################################################################################ + + +class Superset: # pylint: disable=too-few-public-methods + """ + Wrapper of the requests module for the Superset API. + Handles the authentication and the API calls. + Implements parts of the Superset API: https://superset.apache.org/docs/api/ + """ + + @dataclass + class Provider: + """ + Enum for the supported authentication providers. + """ + DB = "db" + LDAP = "ldap" + + # pylint: disable=too-many-arguments + def __init__(self, + server_url: str, + username: str, + password: str, + provider: Provider, + verify_ssl: bool = True) -> None: + """ + Initializes the Superset object and logs in the user. + + Args: + server_url (str): The URL of the Superset server. + username (str): The username of the user. + password (str): The password of the user. + provider (Provider): The authentication provider. + verify_ssl (bool): Verify the SSL certificate of the server. + """ + self._server_url: str = f"{server_url}/api/v1" + self._access_token: str = "" + self._csrf_token: str = "" + self._cookies: dict = {} + self._timeout: int = 60 + self._verify_ssl: bool = verify_ssl + + if not self._verify_ssl: + # Disable SSL warnings if SSL verification is disabled + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + + # Login the user and retrieve the access token and the CSRF token + self._login(username, password, provider) + + def _login(self, username: str, password: str, provider: Provider) -> None: + """ + Logs in the user and retrieves the access token and the refresh token. + + Args: + username (str): The username of the user. + password (str): The password of the user. + provider (Provider): The authentication provider. + + Returns: + None + """ + login_endpoint: str = "/security/login" + crsf_token_endpoint: str = "/security/csrf_token/" + + login_body: dict = { + "password": password, + "provider": provider, + "refresh": True, + "username": username + } + + # Send the login request + ret_code, response = self.request("POST", + login_endpoint, + json=login_body) + + if requests.codes.ok != ret_code: # pylint: disable=no-member + LOG.fatal("Login failed: %s", response.get("message")) + raise RuntimeError("Login failed") + + self._access_token = response.get("access_token", "") + + # Get the CSRF token + ret_code, response = self.request("GET", crsf_token_endpoint) + + if requests.codes.ok != ret_code: # pylint: disable=no-member + LOG.fatal("Get CSRF token failed: %s", response.get("message")) + raise RuntimeError("Get CSRF token failed") + + self._csrf_token = response.get("result", "") + + if self._access_token == "" or self._csrf_token == "": + LOG.fatal("Tokens failed: Access token or CSRF token not received.") + raise RuntimeError("Tokens failed") + + def request(self, + method: str, + endpoint: str, + **request_kwargs) -> tuple[int, dict]: + """ + Sends a request to the Superset API. + + Args: + method (str): The HTTP method of the request. + endpoint (str): The endpoint of the request after '/api/v1'. + data (dict): The data of the request. + request_kwargs (dict): Additional keyword arguments for the request. + Can be any accepted by the Requests module. + + Returns: + tuple[int, dict]: The response code and the response data. + """ + + url: str = f"{self._server_url}{endpoint}" + headers: dict = {} + response_code: int = 0 + reponse_data: dict = {} + + # If already logged in, add the access token to the headers + if self._access_token != "": + headers = { + 'Authorization': f'Bearer {self._access_token}', + 'referer': self._server_url, + 'X-CSRFToken': self._csrf_token + } + + try: + # Send the request + response: requests.Response = requests.request( + method=method, + url=url, + headers=headers, + timeout=self._timeout, + verify=self._verify_ssl, + cookies=self._cookies, + allow_redirects=False, + ** request_kwargs) + + response_code = response.status_code + reponse_data = response.json() + + if response.cookies: + self._cookies = response.cookies.get_dict() + + LOG.info("Request: %s %s", method, url) + LOG.info("Response Code: %s", response_code) + + # Check if the token has expired + if (requests.codes.unauthorized == response_code) and \ + (reponse_data.get('message') == "Token has expired"): # pylint: disable=no-member + LOG.error("Refreshing token is not implemented.") + + except requests.exceptions.JSONDecodeError as e: + LOG.error("JSON decode error: %s", e) + + except requests.exceptions.Timeout as e: + LOG.error("Timeout error: %s", e) + + except requests.exceptions.SSLError as e: + LOG.error("SSL error: %s", e) + if self._verify_ssl is True: + LOG.error("If you trust the server you are connecting to (%s), " + + "consider deactivating SSL verification.", self._server_url) + + except requests.exceptions.RequestException as e: + LOG.error("Request error: %s", e) + + return (response_code, reponse_data) + + +################################################################################ +# Functions +################################################################################ + +################################################################################ +# Main +################################################################################ diff --git a/src/pySupersetCli/version.py b/src/pySupersetCli/version.py new file mode 100644 index 0000000..fa5ad48 --- /dev/null +++ b/src/pySupersetCli/version.py @@ -0,0 +1,118 @@ +"""This module provides version and author information.""" + +# BSD 3-Clause License +# +# Copyright (c) 2024, NewTec GmbH +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +################################################################################ +# Imports +################################################################################ +import importlib.metadata as meta +import os +import sys +import toml + +################################################################################ +# Variables +################################################################################ + +__version__ = "???" +__author__ = "???" +__email__ = "???" +__repository__ = "???" +__license__ = "???" + +################################################################################ +# Classes +################################################################################ + +################################################################################ +# Functions +################################################################################ + + +def resource_path(relative_path): + """ Get the absolute path to the resource, works for dev and for PyInstaller """ + try: + # PyInstaller creates a temp folder and stores path in _MEIPASS + # pylint: disable=protected-access + # pylint: disable=no-member + base_path = sys._MEIPASS + except Exception: # pylint: disable=broad-except + base_path = os.path.abspath(".") + + return os.path.join(base_path, relative_path) + + +def init_from_metadata(): + """Initialize dunders from importlib.metadata + Requires that the package was installed. + + Returns: + list: Tool related information + """ + + my_metadata = meta.metadata('pySupersetCli') + + return \ + my_metadata['Version'], \ + my_metadata['Author'], \ + my_metadata['Author-email'], \ + my_metadata['Project-URL'].replace("repository, ", ""), \ + my_metadata['License'] + + +def init_from_toml(): + """Initialize dunders from pypackage.toml file + + Tried if package wasn't installed. + + Returns: + list: Tool related information + """ + + toml_file = resource_path("pyproject.toml") + data = toml.load(toml_file) + + return \ + data["project"]["version"], \ + data["project"]["authors"][0]["name"], \ + data["project"]["authors"][0]["email"], \ + data["project"]["urls"]["repository"], \ + data["project"]["license"]["text"] + +################################################################################ +# Main +################################################################################ + + +try: + __version__, __author__, __email__, __repository__, __license__ = init_from_metadata() + +except meta.PackageNotFoundError: + __version__, __author__, __email__, __repository__, __license__ = init_from_toml() diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_empty.py b/tests/test_empty.py new file mode 100644 index 0000000..90e261e --- /dev/null +++ b/tests/test_empty.py @@ -0,0 +1,8 @@ +"""Tests +""" + + +def test_do_nothing(): + """The test case does nothing. Its just used to suppress any error + in the test to avoid that the CI alerts. + """ diff --git a/tools/create_executable.bat b/tools/create_executable.bat new file mode 100644 index 0000000..84a578e --- /dev/null +++ b/tools/create_executable.bat @@ -0,0 +1,2 @@ +rem Please run this script from the root path. ".\tools\create_executable.bat" +pyinstaller --noconfirm --onefile --console --name "pySupersetCli" --add-data "./pyproject.toml;." "./src/pySupersetCli/__main__.py" \ No newline at end of file