diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4cbb4b20..7c3f45b7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -54,7 +54,8 @@ jobs: # Run Test - run: meltano install utilities sqlfluff dbt-snowflake - run: meltano run dbt-snowflake:deps - - run: meltano invoke sqlfluff lint $(python sqlfluff_file_changes.py) + - run: meltano invoke sqlfluff lint + # - run: meltano invoke sqlfluff lint $(python sqlfluff_file_changes.py) dbt_seed: runs-on: ubuntu-latest diff --git a/data/transform/models/marts/telemetry/base/unstructured_parsing/context_environment.sql b/data/transform/models/marts/telemetry/base/unstructured_parsing/context_environment.sql index 6270c226..378a200e 100644 --- a/data/transform/models/marts/telemetry/base/unstructured_parsing/context_environment.sql +++ b/data/transform/models/marts/telemetry/base/unstructured_parsing/context_environment.sql @@ -24,6 +24,7 @@ WITH base AS ( MAX(context:data:is_dev_build::STRING) AS is_dev_build, MAX(context:data:is_ci_environment::STRING) AS is_ci_environment, MAX(context:data:notable_flag_env_vars) AS notable_flag_env_vars, + MAX(context:data:notable_hashed_env_vars) AS notable_hashed_env_vars, MAX(context:data:python_version::STRING) AS python_version, MAX( context:data:python_implementation::STRING diff --git a/data/transform/models/marts/telemetry/base/unstructured_parsing/schema.yml b/data/transform/models/marts/telemetry/base/unstructured_parsing/schema.yml index 11944251..5c5ac542 100644 --- a/data/transform/models/marts/telemetry/base/unstructured_parsing/schema.yml +++ b/data/transform/models/marts/telemetry/base/unstructured_parsing/schema.yml @@ -100,7 +100,7 @@ models: tests: - not_null - accepted_values: - values: ['1-0-0', '1-1-0', '1-2-0'] + values: ['1-0-0', '1-1-0', '1-2-0', '1-3-0'] - name: context_exception description: Snowplow exception contexts parsed. diff --git a/data/transform/models/marts/telemetry/base/unstructured_parsing/unstruct_context_flattened.sql b/data/transform/models/marts/telemetry/base/unstructured_parsing/unstruct_context_flattened.sql index 18568dd6..2ff2d8b1 100644 --- a/data/transform/models/marts/telemetry/base/unstructured_parsing/unstruct_context_flattened.sql +++ b/data/transform/models/marts/telemetry/base/unstructured_parsing/unstruct_context_flattened.sql @@ -19,6 +19,8 @@ SELECT context_environment.windows_edition, context_environment.is_dev_build, context_environment.is_ci_environment, + context_environment.notable_flag_env_vars, + context_environment.notable_hashed_env_vars, context_environment.python_version, context_environment.python_implementation, context_environment.system_name, diff --git a/data/transform/models/marts/telemetry/base/unstructured_parsing/unstruct_event_flattened.sql b/data/transform/models/marts/telemetry/base/unstructured_parsing/unstruct_event_flattened.sql index 9d030193..2b25f5ee 100644 --- a/data/transform/models/marts/telemetry/base/unstructured_parsing/unstruct_event_flattened.sql +++ b/data/transform/models/marts/telemetry/base/unstructured_parsing/unstruct_event_flattened.sql @@ -27,6 +27,8 @@ SELECT unstruct_context_flattened.windows_edition, unstruct_context_flattened.is_dev_build, unstruct_context_flattened.is_ci_environment, + unstruct_context_flattened.notable_flag_env_vars, + unstruct_context_flattened.notable_hashed_env_vars, unstruct_context_flattened.python_version, unstruct_context_flattened.python_implementation, unstruct_context_flattened.system_name, @@ -75,6 +77,8 @@ SELECT unstruct_context_flattened.windows_edition, unstruct_context_flattened.is_dev_build, unstruct_context_flattened.is_ci_environment, + unstruct_context_flattened.notable_flag_env_vars, + unstruct_context_flattened.notable_hashed_env_vars, unstruct_context_flattened.python_version, unstruct_context_flattened.python_implementation, unstruct_context_flattened.system_name, @@ -123,6 +127,8 @@ SELECT unstruct_context_flattened.windows_edition, unstruct_context_flattened.is_dev_build, unstruct_context_flattened.is_ci_environment, + unstruct_context_flattened.notable_flag_env_vars, + unstruct_context_flattened.notable_hashed_env_vars, unstruct_context_flattened.python_version, unstruct_context_flattened.python_implementation, unstruct_context_flattened.system_name, @@ -171,6 +177,8 @@ SELECT unstruct_context_flattened.windows_edition, unstruct_context_flattened.is_dev_build, unstruct_context_flattened.is_ci_environment, + unstruct_context_flattened.notable_flag_env_vars, + unstruct_context_flattened.notable_hashed_env_vars, unstruct_context_flattened.python_version, unstruct_context_flattened.python_implementation, unstruct_context_flattened.system_name, @@ -191,7 +199,8 @@ SELECT FROM {{ ref('event_telemetry_state_change') }} LEFT JOIN {{ ref('unstruct_context_flattened') }} ON - event_telemetry_state_change.event_id = unstruct_context_flattened.event_id + event_telemetry_state_change.event_id + = unstruct_context_flattened.event_id -- The original implementation didn't have a project_uuid so they aren't -- useful for anything and should be excluded. WHERE unstruct_context_flattened.project_uuid IS NOT NULL @@ -227,6 +236,8 @@ SELECT unstruct_context_flattened.windows_edition, unstruct_context_flattened.is_dev_build, unstruct_context_flattened.is_ci_environment, + unstruct_context_flattened.notable_flag_env_vars, + unstruct_context_flattened.notable_hashed_env_vars, unstruct_context_flattened.python_version, unstruct_context_flattened.python_implementation, unstruct_context_flattened.system_name, diff --git a/data/transform/models/marts/telemetry/base/unstructured_parsing/unstruct_exec_flattened.sql b/data/transform/models/marts/telemetry/base/unstructured_parsing/unstruct_exec_flattened.sql index 90fb8b00..6aafb480 100644 --- a/data/transform/models/marts/telemetry/base/unstructured_parsing/unstruct_exec_flattened.sql +++ b/data/transform/models/marts/telemetry/base/unstructured_parsing/unstruct_exec_flattened.sql @@ -63,6 +63,8 @@ SELECT ) AS environment_name_hash, MAX(base.client_uuid) AS client_uuid, MAX(base.is_ci_environment) AS is_ci_environment, + MAX(base.notable_flag_env_vars) AS notable_flag_env_vars, + MAX(base.notable_hashed_env_vars) AS notable_hashed_env_vars, MAX(base.num_cpu_cores) AS num_cpu_cores, MAX( base.python_implementation diff --git a/data/transform/models/marts/telemetry/fact_el_executions.sql b/data/transform/models/marts/telemetry/fact_el_executions.sql index adeaf5f2..df132851 100644 --- a/data/transform/models/marts/telemetry/fact_el_executions.sql +++ b/data/transform/models/marts/telemetry/fact_el_executions.sql @@ -37,7 +37,8 @@ pair_roll_up AS ( GROUP BY 1, 2 -- Exclude edge cases: 2 extractors, 2 loaders, etc. HAVING - extractor_plugin_exec_pk IS NOT NULL AND loader_plugin_exec_pk IS NOT NULL + extractor_plugin_exec_pk IS NOT NULL + AND loader_plugin_exec_pk IS NOT NULL ) SELECT