Skip to content

Commit

Permalink
Align configurations with UCX project (#96)
Browse files Browse the repository at this point in the history 
- Fix pylint config and warnings
- Fix github actions workflows
- Fix `pyproject.toml`
  • Loading branch information
@nfx
nfx authored Apr 23, 2024
1 parent 43add0b commit a5a8563
Show file tree
Hide file tree
Showing 8 changed files with 149 additions and 93 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/acceptance.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ jobs:
run: pip install hatch==1.9.4

- name: Acceptance
uses: databrickslabs/sandbox/acceptance@acceptance/v0.1.4
uses: databrickslabs/sandbox/acceptance@acceptance/v0.2.1
with:
vault_uri: ${{ secrets.VAULT_URI }}
env:
Expand Down
23 changes: 23 additions & 0 deletions .github/workflows/no-cheat.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
name: no-cheat

on:
pull_request:
types: [opened, synchronize]

jobs:
no-pylint-disable:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4.1.3
with:
fetch-depth: 0

- name: Verify no lint is disabled in the new code
run: |
NEW_CODE=$(git diff origin/main..$(git branch --show-current) | grep -e '^+')
CHEAT=$(echo "${NEW_CODE}" | grep '# pylint: disable' | grep -v "CHEAT" | wc -c)
if [ "${CHEAT}" -ne 0 ]; then
echo "Do not cheat the linter: ${CHEAT}"
exit 1
fi
19 changes: 2 additions & 17 deletions .github/workflows/push.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@ jobs:
- name: Publish test coverage
uses: codecov/codecov-action@v4
with:
token: ${{ secrets.CODECOV_TOKEN }}

fmt:
runs-on: ubuntu-latest
Expand All @@ -51,20 +53,3 @@ jobs:

- name: Fail on differences
run: git diff --exit-code

no-lint-disabled:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4.1.3
with:
fetch-depth: 0

- name: Verify no lint disabled in the new code
run: |
NEW_CODE=$(git diff origin/main..$(git branch --show-current) | grep -e '^+')
CHEAT=$(echo "${NEW_CODE}" | grep '# pylint: disable' | grep -v "CHEAT" | wc -c)
if [ "${CHEAT}" -ne 0 ]; then
echo "Do not cheat the linter: ${CHEAT}"
exit 1
fi
109 changes: 75 additions & 34 deletions pyproject.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,18 +37,19 @@ path = "src/databricks/labs/blueprint/__about__.py"
[tool.hatch.envs.default]
dependencies = [
"databricks-labs-blueprint[yaml]",
"coverage[toml]>=6.5",
"pytest",
"pylint",
"pytest-xdist",
"pytest-cov>=4.0.0,<5.0.0",
"pytest-mock>=3.0.0,<4.0.0",
"pytest-timeout",
"ruff>=0.0.243",
"isort>=2.5.0",
"mypy",
"types-PyYAML",
"types-requests"
"coverage[toml]~=7.4.4",
"mypy~=1.9.0",
"pylint~=3.1.0",
"pylint-pytest==2.0.0a0",
"databricks-labs-pylint~=0.3.0",
"pytest~=8.1.0",
"pytest-cov~=4.1.0",
"pytest-mock~=3.14.0",
"pytest-timeout~=2.3.1",
"pytest-xdist~=3.5.0",
"ruff~=0.3.4",
"types-PyYAML~=6.0.12",
"types-requests~=2.31.0",
]

python="3.10"
Expand All @@ -62,7 +63,7 @@ coverage = "pytest -n 2 --cov src tests/unit --timeout 30 --cov-report=html -
integration = "pytest -n 10 --cov src tests/integration --durations 20"
fmt = ["isort .",
"ruff format",
"ruff . --fix",
"ruff check . --fix",
"mypy .",
"pylint --output-format=colorized -j 0 src"]
verify = ["black --check .",
Expand All @@ -88,7 +89,7 @@ cache-dir = ".venv/ruff-cache"
target-version = "py310"
line-length = 120

[tool.ruff.isort]
[tool.ruff.lint.isort]
known-first-party = ["databricks.labs.blueprint"]

[tool.coverage.run]
Expand All @@ -108,11 +109,6 @@ exclude_lines = [
# Sources https://google.github.io/styleguide/pylintrc
# License: https://github.com/google/styleguide/blob/gh-pages/LICENSE

# Analyse import fallback blocks. This can be used to support both Python 2 and 3
# compatible code, which means that the block might have code that exists only in
# one or another interpreter, leading to false positives when analysed.
# analyse-fallback-blocks =

# Clear in-memory caches upon conclusion of linting. Useful if running pylint in
# a server-like mode.
# clear-cache-post-run =
Expand All @@ -132,11 +128,6 @@ exclude_lines = [
# for backward compatibility.)
# extension-pkg-whitelist =

# Return non-zero exit code if any of these messages/categories are detected,
# even if score is above --fail-under value. Syntax same as enable. Messages
# specified are enabled, while categories only check already-enabled messages.
# fail-on =

# Specify a score threshold under which the program will exit with error.
fail-under = 10.0

Expand Down Expand Up @@ -168,7 +159,7 @@ ignore-patterns = ["^\\.#"]
# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
# number of processors available to use, and will cap the count on Windows to
# avoid hangs.
# jobs =
jobs = 0

# Control the amount of potential inferred values when inferring a single object.
# This can help the performance when dealing with large functions or complex,
Expand All @@ -177,7 +168,27 @@ limit-inference-results = 100

# List of plugins (as comma separated values of python module names) to load,
# usually to register additional checkers.
load-plugins = ["pylint.extensions.check_elif", "pylint.extensions.bad_builtin", "pylint.extensions.docparams", "pylint.extensions.for_any_all", "pylint.extensions.set_membership", "pylint.extensions.code_style", "pylint.extensions.overlapping_exceptions", "pylint.extensions.typing", "pylint.extensions.redefined_variable_type", "pylint.extensions.comparison_placement", "pylint.extensions.broad_try_clause", "pylint.extensions.dict_init_mutate", "pylint.extensions.consider_refactoring_into_while_condition"]
load-plugins = [
"databricks.labs.pylint.all",
"pylint_pytest",
"pylint.extensions.bad_builtin",
"pylint.extensions.broad_try_clause",
"pylint.extensions.check_elif",
"pylint.extensions.code_style",
"pylint.extensions.confusing_elif",
"pylint.extensions.comparison_placement",
"pylint.extensions.consider_refactoring_into_while_condition",
"pylint.extensions.dict_init_mutate",
"pylint.extensions.docparams",
"pylint.extensions.dunder",
"pylint.extensions.for_any_all",
"pylint.extensions.mccabe",
"pylint.extensions.overlapping_exceptions",
"pylint.extensions.private_import",
"pylint.extensions.redefined_variable_type",
"pylint.extensions.set_membership",
"pylint.extensions.typing",
]

# Pickle collected data for later comparisons.
persistent = true
Expand Down Expand Up @@ -217,7 +228,7 @@ attr-naming-style = "snake_case"
# Regular expression matching correct attribute names. Overrides attr-naming-
# style. If left empty, attribute names will be checked with the set naming
# style.
attr-rgx = "[a-z_][a-z0-9_]{2,}$"
attr-rgx = "[a-z_][a-z0-9_]{1,}$"

# Bad variable names which should always be refused, separated by a comma.
bad-names = ["foo", "bar", "baz", "toto", "tutu", "tata"]
Expand All @@ -232,7 +243,7 @@ class-attribute-naming-style = "any"
# Regular expression matching correct class attribute names. Overrides class-
# attribute-naming-style. If left empty, class attribute names will be checked
# with the set naming style.
class-attribute-rgx = "([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$"
class-attribute-rgx = "([A-Za-z_][A-Za-z0-9_]{1,30}|(__.*__))$"

# Naming style matching correct class constant names.
class-const-naming-style = "UPPER_CASE"
Expand Down Expand Up @@ -265,10 +276,21 @@ function-naming-style = "snake_case"

# Regular expression matching correct function names. Overrides function-naming-
# style. If left empty, function names will be checked with the set naming style.
function-rgx = "[a-z_][a-z0-9_]{2,30}$"
function-rgx = "[a-z_][a-z0-9_]{2,}$"

# Good variable names which should always be accepted, separated by a comma.
good-names = ["i", "j", "k", "ex", "Run", "_"]
good-names = [
"i", "j", "k", "v", # use for loops
"f", # use for file handles
"df", # use for pyspark.sql.DataFrame
"ex", "e", # use for exceptions
"fn", "cb", # use for callbacks
"_", "ok", # use for ignores
"a", # use for databricks.sdk.AccountClient
"w", "ws", # use for databricks.sdk.WorkspaceClient
"me", # use for current user
"T" # use for type variables
]

# Good variable names regexes, separated by a comma. If names match any regex,
# they will always be accepted
Expand All @@ -290,7 +312,8 @@ method-naming-style = "snake_case"

# Regular expression matching correct method names. Overrides method-naming-
# style. If left empty, method names will be checked with the set naming style.
method-rgx = "[a-z_][a-z0-9_]{2,}$"
# Special exception is for `visit_` methods related to AST visitors.
method-rgx = "(([a-z_][a-z0-9_]{2,})|(visit_.*))$"

# Naming style matching correct module names.
module-naming-style = "snake_case"
Expand Down Expand Up @@ -459,7 +482,7 @@ known-third-party = ["enchant"]
[tool.pylint.logging]
# The type of string formatting that logging methods do. `old` means using %
# formatting, `new` is for `{}` formatting.
logging-format-style = "old"
logging-format-style = "new"

# Logging modules to check that the string format arguments are in logging
# function parameter format.
Expand All @@ -479,7 +502,25 @@ confidence = ["HIGH", "CONTROL_FLOW", "INFERENCE", "INFERENCE_FAILURE", "UNDEFIN
# --enable=similarities". If you want to run only the classes checker, but have
# no Warning level messages displayed, use "--disable=all --enable=classes
# --disable=W".
disable = ["raw-checker-failed", "bad-inline-option", "locally-disabled", "file-ignored", "suppressed-message", "deprecated-pragma", "use-implicit-booleaness-not-comparison-to-string", "use-implicit-booleaness-not-comparison-to-zero", "consider-using-augmented-assign", "prefer-typing-namedtuple", "attribute-defined-outside-init", "invalid-name", "missing-module-docstring", "missing-class-docstring", "missing-function-docstring", "protected-access", "too-few-public-methods", "line-too-long", "too-many-lines", "trailing-whitespace", "missing-final-newline", "trailing-newlines", "bad-indentation", "unnecessary-semicolon", "multiple-statements", "superfluous-parens", "mixed-line-endings", "unexpected-line-ending-format", "fixme", "consider-using-assignment-expr", "logging-fstring-interpolation", "consider-using-any-or-all"]
disable = [
"prefer-typing-namedtuple",
"attribute-defined-outside-init",
"missing-module-docstring",
"missing-class-docstring",
"missing-function-docstring",
"too-few-public-methods",
"line-too-long",
"trailing-whitespace",
"missing-final-newline",
"trailing-newlines",
"unnecessary-semicolon",
"mixed-line-endings",
"unexpected-line-ending-format",
"fixme",
"consider-using-assignment-expr",
"logging-fstring-interpolation",
"consider-using-any-or-all"
]

# Enable the message, report, category or checker with the given id(s). You can
# either give multiple identifier separated by comma (,) or put this option
Expand Down Expand Up @@ -522,7 +563,7 @@ default-docstring-type = "default"

[tool.pylint.refactoring]
# Maximum number of nested blocks for function / method body
max-nested-blocks = 5
max-nested-blocks = 3

# Complete name of functions that never returns. When checking for inconsistent-
# return-statements if a never returning function is called then it will be
Expand Down
14 changes: 7 additions & 7 deletions src/databricks/labs/blueprint/__main__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
blueprint = App(__file__)
logger = get_logger(__file__)

main_py_file = '''from databricks.sdk import AccountClient, WorkspaceClient
MAIN_PY_FILE = '''from databricks.sdk import AccountClient, WorkspaceClient
from databricks.labs.blueprint.entrypoint import get_logger
from databricks.labs.blueprint.cli import App
Expand All @@ -38,7 +38,7 @@ def workspaces(a: AccountClient):
'''

labs_yml_file = """---
LABS_YML_FILE = """---
name: __app__
description: Common libraries for Databricks Labs
install:
Expand Down Expand Up @@ -92,10 +92,10 @@ def init_project(target):
relative_file_name = current.as_posix().replace("blueprint", project_name)
dst_file = dst_dir / relative_file_name
dst_file.parent.mkdir(exist_ok=True, parents=True)
with current.open("r", encoding=sys.getdefaultencoding()) as r, dst_file.open("w") as w:
content = r.read().replace("blueprint", project_name)
with current.open("r", encoding=sys.getdefaultencoding()) as src, dst_file.open("w") as dst:
content = src.read().replace("blueprint", project_name)
content = content.replace("databricks-sdk", "databricks-labs-blueprint")
w.write(content)
dst.write(content)
continue
virtual_env_marker = current / "pyvenv.cfg"
if virtual_env_marker.exists():
Expand All @@ -107,13 +107,13 @@ def init_project(target):
inner_package_dir = dst_dir / "src" / "databricks" / "labs" / project_name
inner_package_dir.mkdir(parents=True, exist_ok=True)
with (inner_package_dir / "__main__.py").open("w") as f:
f.write(main_py_file.replace("__app__", project_name))
f.write(MAIN_PY_FILE.replace("__app__", project_name))
with (inner_package_dir / "__init__.py").open("w") as f:
f.write(f"from databricks.labs.{project_name}.__about__ import __version__")
with (inner_package_dir / "__about__.py").open("w") as f:
f.write('__version__ = "0.0.0"\n')
with (dst_dir / "labs.yml").open("w") as f:
f.write(labs_yml_file.replace("__app__", project_name))
f.write(LABS_YML_FILE.replace("__app__", project_name))
with (dst_dir / "CODEOWNERS").open("w") as f:
f.write(f"* @nfx\n/src @databrickslabs/{project_name}-write\n/tests @databrickslabs/{project_name}-write\n")
with (dst_dir / "CHANGELOG.md").open("w") as f:
Expand Down
Loading

0 comments on commit a5a8563

Please sign in to comment.