pyproject.toml

[tool.poetry]
name = "meltano"
version = "2.20.0"
description = "Meltano is your CLI for ELT+: Open Source, Flexible, and Scalable. Move, transform, and test your data with confidence using a streamlined data engineering workflow you’ll love."
authors = ["Meltano <hello@meltano.com>"]
license = "MIT"
readme = "README.md"
repository = "https://github.com/meltano/meltano"
classifiers = [
  "Development Status :: 5 - Production/Stable",
  "Programming Language :: Python :: 3 :: Only",
  "Programming Language :: Python :: 3.7",
  "Programming Language :: Python :: 3.8",
  "Programming Language :: Python :: 3.9",
  "Programming Language :: Python :: 3.10",
  "Programming Language :: Python :: 3.11",
  "License :: OSI Approved :: MIT License",
  "Operating System :: OS Independent",
]
packages = [
  { include = "meltano", from = "src", format = "wheel" },
  { include = "src", format = "sdist" },
  { include = "tests", format = "sdist"},
  { include = "scripts/build.py", format = "sdist"},
]
# Explicitly include non-Python files and gitignored files:
include = [
  "src/meltano/api/static/css/*",
  "src/meltano/api/static/js/*",
  "src/meltano/api/templates/embed.html",
  "src/meltano/api/templates/webapp.html",
  "src/meltano/core/tracking/.release_marker",
  "src/meltano/schema/*",
]
keywords = [
  "Meltano",
  "ELT",
  "Data integration",
  "singer-io",
  "dbt",
]
documentation = "https://docs.meltano.com"
homepage = "https://meltano.com"

[tool.poetry.urls]
"Changelog" = "https://github.com/meltano/meltano/blob/main/CHANGELOG.md"
"Issue Tracker" = "https://github.com/meltano/meltano/issues"
"Slack" = "https://meltano.com/slack"
"Twitter" = "https://twitter.com/meltanodata/"
"Youtube" = "https://www.youtube.com/meltano"

[tool.poetry.dependencies]
aiodocker = "^0.21.0"
aiohttp = "^3.4.4"
alembic = "^1.11.1"
atomicwrites = "^1.2.1"
authlib = "^1.2.1"
azure-common = {version = "^1.1.28", optional = true}
azure-core = {version = "^1.28.0", optional = true}
azure-storage-blob = {version = "^12.14.1", optional = true}
backoff = "^2.1.2"
bcrypt = "^3.2.0" # Needs to be installed explicitly so it can be used as a backend for passlib
boto3 = {version = "^1.28.7", optional = true}
cached-property = "^1" # Remove after Python 3.7 support is dropped
check-jsonschema = "^0.22.0"
click = "^8.1"
click-default-group = "^1.2.1"
click-didyoumean = "^0.3.0"
croniter = "^1.4.1"
email-validator = "^1.1.2"
fasteners = "^0.18"
flask = "^2.1"
flask-cors = "^3.0.7"
flask-executor = "^0.10"
flask-login = "==0.6.1" # https://github.com/meltano/meltano/issues/6548
flask-restful = "^0.3.10"
flask-sqlalchemy = "^2.4.4"
flatten-dict = "^0"
google-cloud-storage = {version = ">=1.31.0", optional = true}
gunicorn = "^20.1.0"
importlib-resources = {version = "^5.10.1", optional = true, python = "<3.9"}
jsonschema = "^4.17"
meltano-flask-security = "^0.1.0"
packaging = "^23.0"
psutil = "^5.9.5"
psycopg2-binary = "^2.8.5"
pyhumps = "^3.0.0"
pymssql = {version = "^2.2.5", optional = true}
python = ">=3.7,<3.12"
python-dotenv = "^0.21.1"
python-gitlab = "^3.15.0"
pyyaml = "^6.0.1"
requests = "^2.31.0"
rich = "^13.4.2"
"ruamel.yaml" = "^0.17.21"
smart-open = {version = "^6.2.0"}
smtpapi = "^0.4.1"
snowplow-tracker = "^1.0.1"
sqlalchemy = "^1.3.19"
structlog = "^23.1.0"
tzlocal = "^5.0.1"
uvicorn = {extras = ["standard"], version = "^0.17.6"}
virtualenv = "^20.24.1"
werkzeug = ">=2.1,<=2.1.3"

[tool.poetry.extras]
azure = ["azure-storage-blob", "azure-common", "azure-core"]
gcs = ["google-cloud-storage"]
mssql = ["pymssql"]
s3 = ["boto3"]

[tool.poetry.group.dev.dependencies]
black = "^23.3.0"
bumpversion = "^0.6.0"
colorama = "^0.4.4"
commitizen-version-bump = {git = "https://github.com/meltano/commitizen-version-bump.git", branch = "main"}
coverage = {extras = ["toml"], version = "^7.2.7"}
freezegun = "^1.2.2"
mock = "^5.0.2"
mypy = "^1.4"
pre-commit = "^2.9.2"
pytest = "^7.4.0"
pytest-asyncio = "^0.21.1"
pytest-cov = "^4.1.0"
pytest-docker = "^2.0"
pytest-order = "^1.0"
pytest-randomly = "^3.12"
pytest-structlog = "^0.6"
pytest-xdist = "^3.3"
requests-mock = "^1.11.0"
setproctitle = "^1.3" # Used by pytest-xdist to aid with handling resource intensive processes.
types-croniter = "^1.4.0"
types-jsonschema = "^4.17.0.9"
types-psutil = "^5.9.5.15"
types-pyyaml = "^6.0.12.10"
types-requests = "^2.31.0"

[tool.poetry.scripts]
meltano = "meltano.cli:main"

[tool.black]
line-length = 88

[tool.flakeheaven]
accept_encodings = "utf-8"
baseline = "flakeheaven-baseline"
docstring_style = "google"
doctests = true
enable_extensions = "G"
exclude = [
  ".nox/",
  ".venv",
  "venv",
  "*.md",
  "src/webapp/node_modules/",
  "src/meltano/migrations/",
  "src/meltano/api/",
  "src/meltano/cli/ui.py",
]
format = "colored"
inline_quotes = "double"
max_complexity = 6
max_imports = 20
max_line_length = 88
show_source = true
statistics = true

[tool.flakeheaven.plugins]
"flake8-*" = [
  "+*",
]
"flake8-isort" = [
  "-*",
]
"flake8-commas" = [
  "-*",
]
"flake8-bandit" = [
  "+*",
  "-S404", # Allow subprocess module to be used
  "-S310", # Allow `urllib.open`
  "-S603", # Allow `subprocess.run(..., shell=False)`
]
"flake8-string-format" = [
  "-P101", # Allow format strings with unindex parameters (because the pyupgrade pre-commit hook enforces their usage)
]
"flake8-rst-docstrings" = [
  "-*",
]
mccabe = [
  "+*",
]
"pep8-naming" = [
  "+*",
  "-N818", # Allow Exceptions to have suffix 'Exception' rather than 'Error'
]
pycodestyle = [
  "+*",
  "-E203", # Let Black put whitespace before :
  "-W503", # Allow for line breaks where Black decides are valid
]
pyflakes = [
  "+*",
]
"wemake-python-styleguide" = [
  "+*",
  "-WPS100",
  "-WPS110", # Allow blacklisted variable names that can be clear enough in method context
  "-WPS111",
  "-WPS112",
  "-WPS115", # Allow upper-case constants in Enum subclasses
  "-WPS201",
  "-WPS202", # Allow many module members
  "-WPS211", # Allow "too many"
  "-WPS212", # Ignore 'too many return statements' - noqa parse inconsistent between py38 vs prior
  "-WPS220",
  "-WPS221",
  "-WPS226",
  "-WPS229",
  "-WPS231",
  "-WPS232",
  "-WPS237",
  "-WPS300", # Allow local folder imports (import .module)
  "-WPS305", # Allow f-strings
  "-WPS306", # Ignore missing base class, required by pyupgrade: https://github.com/asottile/pyupgrade#rewrites-class-declaration
  "-WPS309", # Allow reversed compare order
  "-WPS316",
  "-WPS317", # Allow "incorrect multi-line parameters", since Black forces them to be "incorrect"
  "-WPS324", # Allow inconsistent returns - permitted because of too many false positives
  "-WPS326", # Allow syntax-level (process when first parsed, rather than at every run) string concatenation
  "-WPS332", # Allow assignment expressions (walrus operator :=)
  "-WPS337",
  "-WPS338", # Allow 'incorrect' order of methods in a class
  "-WPS348", # Allow `.` at beginning of line to accommodate Black formatting of multiline chained function calls
  "-WPS352",
  "-WPS354", # Allow consecutive yield expressions
  "-WPS402", # Allow "overuse" of noqa comments.
  "-WPS404",
  "-WPS407",
  "-WPS410", # Allow metadata variable (__all__)
  "-WPS412",
  "-WPS414", # Allow unpacking assignments
  "-WPS420", # Allow `pass` keywords in `except` bodies to prevent falling through to another `except`
  "-WPS429",
  "-WPS430",
  "-WPS431", # Allow nested classes
  "-WPS433",
  "-WPS440", # Allow block variable overlap, e.g. multiple for-loops that use the same name for their loop variables
  "-WPS441", # Allow control variables to be used after their block, because this rule has too many false-positives
  "-WPS454",
  "-WPS458", # Import name collisions - disabled because it raises many false positives
  "-WPS461", # Allow all inline ignores
  "-WPS501",
  "-WPS507",
  "-WPS509", # Allow "incorrect" ternary expressions
  "-WPS528", # Allow iterating over keys in a dict, then using them to access values
  "-WPS529", # Allow use of `dict_object[key]` (i.e. `dict.__getitem__`) instead of `dict.get`
  "-WPS600", # Allow subclassing a builtin, such as 'str' (used for serializable Enums)
  "-WPS602", # Allow using @staticmethod
  "-WPS615", # Allow "unpythonic" getters and setters
]

[tool.flakeheaven.exceptions."tests/*"]
"flake8-bandit" = [
  "-S101", # Don't warn on use of asserts in tests
]
# Don't require docstrings in tests
"flake8-darglint" = [
  "-DAR101",
  "-DAR201",
  "-DAR301",
]
"flake8-docstrings" = [
  "-D100",
  "-D101",
  "-D102",
  "-D103",
  "-D104",
  "-D105",
  "-D107",
]
"flake8-string-format" = [
  "-P103", # String does contain unindexed parameters
]
"wemake-python-styleguide" = [
  "-WPS114", # Allow underscored number name pattern (e.g. tap_covid_19)
  "-WPS204", # Don't warn on overused expressions in test methods
  "-WPS211", # Don't warn on too many arguments in test methods
  "-WPS210", # Don't warn on too many local variables in test methods
  "-WPS218", # Don't warn on too many `assert` statements in test methods
  "-WPS213", # Found too many expressions
  "-WPS214", # Don't warn on too many methods in test modules
  "-WPS217", # Don't warn about there being too many await expressions
  "-WPS226", # Don't warn on too many string constants in test methods
  "-WPS316", # Context manager with too many assignments
  "-WPS442", # Allow outer scope name shadowing for fixtures
  "-WPS430", # Allow nested functions in test methods
  "-WPS432", # Allow magic numbers
  "-WPS436", # Allow protected module import (_pytest.monkeypatch)
  "-WPS437", # Allow protected attribute usage in test methods
  "-WPS201", # Found module with too many imports
  "-WPS425", # Found boolean non-keyword argument
  "-WPS118", # Found too long name
  "-WPS342", # Avoid false positive for implicit raw strings
]

[tool.flakeheaven.exceptions."!tests/*"]
"flake8-debugger" = [
  "-T100",
]

[tool.flakeheaven.exceptions."scripts/*"]
"wemake-python-styleguide" = [
  "-WPS421", # Allow for print function calls
]

[tool.flakeheaven.exceptions."src/meltano/cli/*"]
"flake8-docstrings" = [
  "-D301", # Allow backslashes in dosctrings
]
"flake8-darglint" = [
  "-*", # Don't require docstrings in click commands
]
"wemake-python-styleguide" = [
  "-WPS210", # Allow many local variables, since these aid in string building
  "-WPS211", # Allow many arguments, since these often represent CLI arguments
  "-WPS213", # Allow many expressions, since string building requires them
  "-WPS216", # Allow many decorators, since `click` uses them to define the CLI
  "-WPS202", # Allow many module members
  "-WPS204", # Allow 'overuse' of expressions (e.g. ctx.obj["-settings"])
]

[tool.flakeheaven.exceptions."src/meltano/migrations/versions/*"]
"flake8-docstrings" = [
  "-D101", # Don't class docstrings in migrations
  "-D103", # Don't require function docstrings in migrations
  "-D400", # Don't require periods in the first lines of dosctrings
]
"pep8-naming" = [
  "-N806", # Allow uppercase variable names
]
"wemake-python-styleguide" = [
  "-WPS102", # Allow module names starting with numbers
  "-WPS118", # Allow long filenames
  "-WPS204", # Allow expression "-overuse"
  "-WPS226", # Allow string constant "-overuse"
  "-WPS432", # Allow "-magic" numbers
]

[tool.flakeheaven.exceptions."src/meltano/core/job/job.py"]
"wemake-python-styleguide" = [
  "-WPS601", # Class attributes can be used as instance attributes in sqlalchemy.declarative model
]

[tool.flakeheaven.exceptions."src/meltano/core/project_files.py"]
"wemake-python-styleguide" = [
  "-WPS226", # Dict manipulation requires lots of string constant references
]

[tool.flakeheaven.exceptions."src/meltano/core/project_init_service.py"]
"wemake-python-styleguide" = [
  "-WPS226", # Found string constant over-use: blue > 3
  "-WPS213", # Found too many expressions: 27 > 9
]

[tool.flakeheaven.exceptions."src/**/__init__.py"]
pyflakes = [
  "-F401", # Allow for using __init__.py to promote imports to the module namespace
]

[tool.flakeheaven.exceptions."tests/**/__init__.py"]
pyflakes = [
  "-F401", # Allow for using __init__.py to promote imports to the module namespace
]
# Don't require docstrings in tests
"flake8-darglint" = [
  "-DAR101",
  "-DAR201",
  "-DAR301",
]
"flake8-docstrings" = [
  "-D100",
  "-D101",
  "-D102",
  "-D103",
  "-D104",
]

[tool.flakeheaven.exceptions."src/meltano/__init__.py"]
"wemake-python-styleguide" = [
  "-WPS412", # Found `__init__.py` module with logic
  "-WPS410", # Found wrong metadata variable
]

[tool.flakeheaven.exceptions."src/meltano/core/block/extract_load.py"]
"wemake-python-styleguide" = [
  "-WPS201", # Found module with too many imports: 21 > 20
]

[tool.flakeheaven.exceptions."src/meltano/core/logging/utils.py"]
"wemake-python-styleguide" = [
  "-WPS100", # Found wrong module name
]

[tool.flakeheaven.exceptions."src/meltano/core/cli_messages.py"]
# Special exceptions for cli magic
"wemake-python-styleguide" = [
  "-WPS114",
  "-WPS360",
]
pycodestyle = [
  "-W291",
  "-W293",
]

[tool.flakeheaven.exceptions."src/meltano/core/settings_store.py"]
"wemake-python-styleguide" = [
  "-WPS204", # Found overused expression: allow settings store implementations to all return vars with the same names
]

[tool.flakeheaven.exceptions."src/meltano/core/utils/__init__.py"]
"flake8-docstrings" = [
  "-D103", # Many of these utility functions are descriptively named and self-documenting
]

[tool.flakeheaven.exceptions."src/meltano/core/state_store/*"]
"wemake-python-styleguide" = [
  "-WPS428", # Ignore ... in ABC
]

[tool.flakeheaven.exceptions."src/meltano/core/state_store/__init__.py"]
"wemake-python-styleguide" = [
  "-WPS320", # Ignore multi-line function annotations
]

[tool.pytest.ini_options]
addopts = "--cov=meltano --cov=tests --doctest-modules --order-scope=module -ra"
asyncio_mode = "auto"
log_cli = 1
log_cli_format = "%(message)s"
log_cli_level = "CRITICAL"
log_file = "pytest.log"
log_file_date_format = "%Y-%m-%d %H:%M:%S"
log_file_format = "%(asctime)s [%(levelname)8s] %(message)s (%(filename)s:%(lineno)s)"
log_file_level = "DEBUG"
markers = [
  "backend(type): backend specific test",
  "meta: meta test",
  "concurrent: test requires true concurrency",
  "slow: slow test",
]
testpaths = "tests"

[tool.coverage.run]
branch = true
concurrency = [
  "multiprocessing",
  "thread",
]
parallel = true
source = ["meltano"]

[tool.coverage.paths]
source = [
  "src/meltano",
  "**/site-packages/meltano",
  "**/site-packages/meltano*/meltano",
]

[tool.coverage.report]
exclude_lines = [
  "pragma: no cover",
  '''if (t\.)?TYPE_CHECKING:''',
  "\\.\\.\\.",
]
precision = 2
show_missing = true
skip_covered = true

[tool.commitizen]
name = "cz_version_bump"
tag_format = "v$major.$minor.$patch$prerelease"
version = "2.20.0"
version_files = [
  "pyproject.toml:^version =",
  "src/meltano/__init__.py:^__version__ =",
  'src/webapp/package.json:^  "version":',
  'docs/package.json:^  "version":',
]

[tool.mypy]
files = "src"
plugins = [
  "sqlalchemy.ext.mypy.plugin",
]
warn_unused_configs = true
warn_unused_ignores = true

# The following whitelist is used to allow for incremental adoption
# of MyPy. Modules should be removed from this whitelist as and when
# their respective type errors have been addressed. No new modules
# should be added to this whitelist.
[[tool.mypy.overrides]]
ignore_errors = true
module = [
  "meltano.api.*",
  "meltano.cli.*",
  "meltano.oauth.*",
  # Way too many modules at the root of meltano.core to tackle them all at once  # so listing them individually here.
  "meltano.core.connection_service",
  "meltano.core.db",
  "meltano.core.elt_context",
  "meltano.core.extract_utils",
  "meltano.core.extractor",
  "meltano.core.meltano_invoker",
  "meltano.core.migration_service",
  "meltano.core.models",
  "meltano.core.plugin_discovery_service",
  "meltano.core.plugin_install_service",
  "meltano.core.plugin_invoker",
  "meltano.core.plugin_test_service",
  "meltano.core.project",
  "meltano.core.project_add_service",
  "meltano.core.project_files",
  "meltano.core.project_plugins_service",
  "meltano.core.project_settings_service",
  "meltano.core.select_service",
  "meltano.core.settings_service",
  "meltano.core.settings_store",
  "meltano.core.sqlalchemy",
  "meltano.core.task_sets_service",
  "meltano.core.transform_add_service",
  # Meltano Core submodules
  "meltano.core.behavior.*",
  "meltano.core.block.*",
  "meltano.core.compiler.*",
  "meltano.core.job.*",
  "meltano.core.legacy_tracking.*",
  "meltano.core.logging.job_logging_service",
  "meltano.core.logging.output_logger",
  "meltano.core.logging.utils",
  "meltano.core.m5o.*",
  "meltano.core.plugin.*",
  "meltano.core.runner.*",
  "meltano.core.sql.*",
  "meltano.core.tracking.*",
  "meltano.core.utils.*",
]

[build-system]
requires = ["poetry-core==1.5.0"]
build-backend = "build:api"
backend-path = ["scripts"]

[tool.ruff]
target-version = "py37"
line-length = 88
exclude = [
  "src/meltano/migrations/*",
  "src/webapp/*",
  "src/meltano/api/*",
  "src/meltano/cli/ui.py",
]
ignore = [
  "ANN101", # Missing type annotation for `self` in method
  "ANN102", # Missing type annotation for `cls` in class method
  "PT004",  # Add leading underscore to fixtures that do not return anything
  "UP026",  # Replace `mock` import with `unittest.mock` - remove once Python 3.7 support is dropped
]
select = [
  "ARG", # flake8-unused-arguments
  "B",   # flake8-bugbear
  "C4",  # flake8-comprehensions
  "COM", # flake8-commas
  "E",   # pycodestyle (error)
  "F",   # pyflakes
  "I",   # isort
  "ISC", # flake8-implicit-str-concat
  "PT",  # flake8-pytest-style
  "RSE", # flake8-raise
  "SIM", # flake8-simplify
  "TID", # flake8-tidy-imports
  "UP",  # pyupgrade
  "W",   # pycodestyle (warning)
]

[tool.ruff.per-file-ignores]
"__init__.py" = [
  "F401", # Permit unused imports in `__init__.py` files
]

[tool.ruff.flake8-annotations]
allow-star-arg-any = true
mypy-init-return = true
suppress-dummy-args = true

[tool.ruff.flake8-pytest-style]
parametrize-values-type = "tuple"

[tool.ruff.isort]
known-first-party = ["asserts", "fixtures", "meltano"]
required-imports = ["from __future__ import annotations"]

[tool.ruff.pydocstyle]
convention = "google"

[tool.ruff.flake8-tidy-imports.banned-api]
"meltano.cli.cli.command".msg = "Use `click.command` and `meltano.cli.cli.add_command` instead"
"meltano.cli.cli.group".msg = "Use `click.group` and `meltano.cli.cli.add_command` instead"