diff --git a/.github/workflows/test_common.yml b/.github/workflows/test_common.yml index 585f2adafc..75f5ac18a2 100644 --- a/.github/workflows/test_common.yml +++ b/.github/workflows/test_common.yml @@ -96,11 +96,11 @@ jobs: run: poetry install --no-interaction --with sentry-sdk - run: | - poetry run pytest tests/common tests/normalize tests/reflection tests/load/test_dummy_client.py tests/extract/test_extract.py tests/extract/test_sources.py tests/pipeline/test_pipeline_state.py + poetry run pytest tests/common tests/normalize tests/reflection tests/plugins tests/load/test_dummy_client.py tests/extract/test_extract.py tests/extract/test_sources.py tests/pipeline/test_pipeline_state.py if: runner.os != 'Windows' name: Run common tests with minimum dependencies Linux/MAC - run: | - poetry run pytest tests/common tests/normalize tests/reflection tests/load/test_dummy_client.py tests/extract/test_extract.py tests/extract/test_sources.py tests/pipeline/test_pipeline_state.py -m "not forked" + poetry run pytest tests/common tests/normalize tests/reflection tests/plugins tests/load/test_dummy_client.py tests/extract/test_extract.py tests/extract/test_sources.py tests/pipeline/test_pipeline_state.py -m "not forked" if: runner.os == 'Windows' name: Run common tests with minimum dependencies Windows shell: cmd diff --git a/dlt/common/runtime/anon_tracker.py b/dlt/common/runtime/anon_tracker.py index 4e78db48e5..8e4393d53d 100644 --- a/dlt/common/runtime/anon_tracker.py +++ b/dlt/common/runtime/anon_tracker.py @@ -9,7 +9,7 @@ from dlt.common import logger from dlt.common.managed_thread_pool import ManagedThreadPool from dlt.common.configuration.specs import RuntimeConfiguration -from dlt.common.runtime.exec_info import get_execution_context, TExecutionContext +from dlt.common.runtime.exec_info import get_execution_context, TExecutionContext, run_context_name from dlt.common.runtime import run_context from dlt.common.typing import DictStrAny, StrAny from dlt.common.utils import uniq_id @@ -85,7 +85,10 @@ def track(event_category: TEventCategory, event_name: str, properties: DictStrAn properties.update({"event_category": event_category, "event_name": event_name}) try: - _send_event(f"{event_category}_{event_name}", properties, _default_context_fields()) + context = _default_context_fields() + # always refresh run context name, it may change at runtime + context["run_context"] = run_context_name() + _send_event(f"{event_category}_{event_name}", properties, context) except Exception as e: logger.debug(f"Skipping telemetry reporting: {e}") raise diff --git a/dlt/common/runtime/exec_info.py b/dlt/common/runtime/exec_info.py index b894b3aad8..1f28262129 100644 --- a/dlt/common/runtime/exec_info.py +++ b/dlt/common/runtime/exec_info.py @@ -6,7 +6,7 @@ import platform from dlt.common.runtime.typing import TExecutionContext, TVersion, TExecInfoNames -from dlt.common.typing import StrStr, StrAny, Literal, List +from dlt.common.typing import StrStr, StrAny, List from dlt.common.utils import filter_env_vars from dlt.version import __version__, DLT_PKG_NAME @@ -175,13 +175,43 @@ def is_gcp_cloud_function() -> bool: return os.environ.get("FUNCTION_NAME") is not None +def get_plus_version() -> TVersion: + "Gets dlt+ library version" + try: + from dlt_plus.version import __version__, PKG_NAME + + return TVersion(name=PKG_NAME, version=__version__) + except Exception: + return None + + +def run_context_name() -> str: + try: + from dlt.common.configuration.container import Container + from dlt.common.configuration.specs.pluggable_run_context import PluggableRunContext + + container = Container() + if PluggableRunContext in container: + return container[PluggableRunContext].context.name + + except Exception: + pass + + return "dlt" + + def get_execution_context() -> TExecutionContext: "Get execution context information" - return TExecutionContext( + context = TExecutionContext( ci_run=in_continuous_integration(), python=sys.version.split(" ")[0], cpu=multiprocessing.cpu_count(), exec_info=exec_info_names(), os=TVersion(name=platform.system(), version=platform.release()), library=TVersion(name=DLT_PKG_NAME, version=__version__), + run_context=run_context_name(), ) + if plus_version := get_plus_version(): + context["plus"] = plus_version + + return context diff --git a/dlt/common/runtime/typing.py b/dlt/common/runtime/typing.py index 8ab24bcf3a..51397c423c 100644 --- a/dlt/common/runtime/typing.py +++ b/dlt/common/runtime/typing.py @@ -25,7 +25,7 @@ class TVersion(TypedDict): version: str -class TExecutionContext(TypedDict): +class TExecutionContext(TypedDict, total=False): """TypeDict representing the runtime context info""" ci_run: bool @@ -34,3 +34,5 @@ class TExecutionContext(TypedDict): exec_info: List[TExecInfoNames] library: TVersion os: TVersion + run_context: str + plus: TVersion diff --git a/mypy.ini b/mypy.ini index 51151486b6..bb8c8ffa31 100644 --- a/mypy.ini +++ b/mypy.ini @@ -146,3 +146,6 @@ ignore_missing_imports = True [mypy-airflow.*] ignore_missing_imports = True + +[mypy-dlt_plus.*] +ignore_missing_imports = True diff --git a/tests/common/runtime/test_telemetry.py b/tests/common/runtime/test_telemetry.py index 255f76e5e4..f55b0c78f2 100644 --- a/tests/common/runtime/test_telemetry.py +++ b/tests/common/runtime/test_telemetry.py @@ -171,10 +171,13 @@ def test_track_anon_event( # verify context context = event["context"] assert context["library"] == {"name": DLT_PKG_NAME, "version": __version__} + # we assume plus is not installed + assert "plus" not in context assert isinstance(context["cpu"], int) assert isinstance(context["ci_run"], bool) assert isinstance(context["exec_info"], list) assert ["kubernetes", "codespaces"] <= context["exec_info"] + assert context["run_context"] == "dlt" def test_cleanup(environment: DictStrStr) -> None: diff --git a/tests/pipeline/cases/contracts/trace.schema.yaml b/tests/pipeline/cases/contracts/trace.schema.yaml index 1d6c31bdd7..51505bd350 100644 --- a/tests/pipeline/cases/contracts/trace.schema.yaml +++ b/tests/pipeline/cases/contracts/trace.schema.yaml @@ -75,6 +75,15 @@ tables: execution_context__library__version: data_type: text nullable: true + execution_context__plus__name: + data_type: text + nullable: true + execution_context__plus__version: + data_type: text + nullable: true + execution_context__run_context: + data_type: text + nullable: true started_at: data_type: timestamp nullable: true diff --git a/tests/plugins/dlt_example_plugin/pyproject.toml b/tests/plugins/dlt_example_plugin/pyproject.toml index 475254e591..850df1ec50 100644 --- a/tests/plugins/dlt_example_plugin/pyproject.toml +++ b/tests/plugins/dlt_example_plugin/pyproject.toml @@ -12,7 +12,7 @@ packages = [ dlt-example-plugin = "dlt_example_plugin" [tool.poetry.dependencies] -python = ">=3.8.1,<3.13" +python = ">=3.9.1,<3.14" dlt={"path"="../../../"} [build-system] diff --git a/tests/plugins/test_plugin_discovery.py b/tests/plugins/test_plugin_discovery.py index 6962e89bf7..32d0f3d79b 100644 --- a/tests/plugins/test_plugin_discovery.py +++ b/tests/plugins/test_plugin_discovery.py @@ -7,6 +7,7 @@ import importlib from dlt.common.configuration.container import Container +from dlt.common.configuration.specs.pluggable_run_context import PluggableRunContext from dlt.common.runners import Venv from dlt.common.configuration import plugins from dlt.common.runtime import run_context @@ -31,8 +32,11 @@ def plugin_install(): raise sys.path.insert(0, temp_dir) - # remove current plugin manager + # remove current plugin manager and run context + # NOTE: new run context reloads plugin manager container = Container() + if PluggableRunContext in container: + del container[PluggableRunContext] if plugins.PluginContext in container: del container[plugins.PluginContext] @@ -54,6 +58,13 @@ def test_example_plugin() -> None: assert context.data_dir == os.path.abspath(TEST_STORAGE_ROOT) +def test_plugin_execution_context() -> None: + from dlt.common.runtime.exec_info import get_execution_context + + context = get_execution_context() + assert context["run_context"] == "dlt-test" + + def test_cli_hook(script_runner: ScriptRunner) -> None: # new command result = script_runner.run(["dlt", "example", "--name", "John"])