Add version selector method (#7330)

Add version selector method: "latest", "prerelease", "old", and "none" selectors supported

.changes/unreleased/Features-20230412-011138.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Add version selector method
+time: 2023-04-12T01:11:38.093684-04:00
+custom:
+  Author: MichelleArk
+  Issue: "7199"

core/dbt/graph/selector_methods.py

@@ -17,7 +17,9 @@
     SourceDefinition,
     ResultNode,
     ManifestNode,
+    ModelNode,
 )
+from dbt.contracts.graph.unparsed import UnparsedVersion
 from dbt.contracts.state import PreviousState
 from dbt.exceptions import (
     DbtInternalError,
@@ -48,6 +50,7 @@ class MethodName(StrEnum):
     Result = "result"
     SourceStatus = "source_status"
     Wildcard = "wildcard"
+    Version = "version"
 
 
 def is_selected_node(
@@ -639,6 +642,38 @@ def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[Uniqu
                 yield node
 
 
+class VersionSelectorMethod(SelectorMethod):
+    def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]:
+        for node, real_node in self.parsed_nodes(included_nodes):
+            if isinstance(real_node, ModelNode):
+                if selector == "latest":
+                    if real_node.is_latest_version:
+                        yield node
+                elif selector == "prerelease":
+                    if (
+                        real_node.version
+                        and real_node.latest_version
+                        and UnparsedVersion(v=real_node.version)
+                        > UnparsedVersion(v=real_node.latest_version)
+                    ):
+                        yield node
+                elif selector == "old":
+                    if (
+                        real_node.version
+                        and real_node.latest_version
+                        and UnparsedVersion(v=real_node.version)
+                        < UnparsedVersion(v=real_node.latest_version)
+                    ):
+                        yield node
+                elif selector == "none":
+                    if real_node.version is None:
+                        yield node
+                else:
+                    raise DbtRuntimeError(
+                        f'Invalid version type selector {selector}: expected one of: "latest", "prerelease", "old", or "none"'
+                    )
+
+
 class MethodManager:
     SELECTOR_METHODS: Dict[MethodName, Type[SelectorMethod]] = {
         MethodName.FQN: QualifiedNameSelectorMethod,
@@ -657,6 +692,7 @@ class MethodManager:
         MethodName.Metric: MetricSelectorMethod,
         MethodName.Result: ResultSelectorMethod,
         MethodName.SourceStatus: SourceStatusSelectorMethod,
+        MethodName.Version: VersionSelectorMethod,
     }
 
     def __init__(

test/unit/test_graph_selector_methods.py

@@ -42,6 +42,7 @@
     StateSelectorMethod,
     ExposureSelectorMethod,
     MetricSelectorMethod,
+    VersionSelectorMethod,
 )
 import dbt.exceptions
 import dbt.contracts.graph.nodes
@@ -608,7 +609,7 @@ def versioned_model_v1(seed):
         sources=[],
         path="subdirectory/versioned_model_v1.sql",
         version=1,
-        latest_version=1,
+        latest_version=2,
     )
 
 
@@ -623,7 +624,22 @@ def versioned_model_v2(seed):
         sources=[],
         path="subdirectory/versioned_model_v2.sql",
         version=2,
-        latest_version=1,
+        latest_version=2,
+    )
+
+
+@pytest.fixture
+def versioned_model_v3(seed):
+    return make_model(
+        "pkg",
+        "versioned_model",
+        'select * from {{ ref("seed") }}',
+        config_kwargs={"materialized": "table"},
+        refs=[seed],
+        sources=[],
+        path="subdirectory/versioned_model_v3.sql",
+        version="3",
+        latest_version=2,
     )
 
 
@@ -703,6 +719,7 @@ def manifest(
     union_model,
     versioned_model_v1,
     versioned_model_v2,
+    versioned_model_v3,
     ext_source_2,
     ext_source_other,
     ext_source_other_2,
@@ -729,6 +746,7 @@ def manifest(
         union_model,
         versioned_model_v1,
         versioned_model_v2,
+        versioned_model_v3,
         ext_model,
         table_id_unique,
         table_id_not_null,
@@ -789,6 +807,7 @@ def test_select_fqn(manifest):
         "union_model",
         "versioned_model.v1",
         "versioned_model.v2",
+        "versioned_model.v3",
         "table_model",
         "table_model_py",
         "table_model_csv",
@@ -804,6 +823,7 @@ def test_select_fqn(manifest):
     assert search_manifest_using_method(manifest, method, "versioned_model") == {
         "versioned_model.v1",
         "versioned_model.v2",
+        "versioned_model.v3",
     }
     assert search_manifest_using_method(manifest, method, "versioned_model.v1") == {
         "versioned_model.v1"
@@ -980,6 +1000,7 @@ def test_select_package(manifest):
         "union_model",
         "versioned_model.v1",
         "versioned_model.v2",
+        "versioned_model.v3",
         "table_model",
         "table_model_py",
         "table_model_csv",
@@ -1030,6 +1051,7 @@ def test_select_config_materialized(manifest):
         "union_model",
         "versioned_model.v1",
         "versioned_model.v2",
+        "versioned_model.v3",
         "mynamespace.union_model",
     }
 
@@ -1106,6 +1128,27 @@ def test_select_test_type(manifest):
     assert search_manifest_using_method(manifest, method, "data") == {"view_test_nothing"}
 
 
+def test_select_version(manifest):
+    methods = MethodManager(manifest, None)
+    method = methods.get_method("version", [])
+    assert isinstance(method, VersionSelectorMethod)
+    assert method.arguments == []
+    assert search_manifest_using_method(manifest, method, "latest") == {"versioned_model.v2"}
+    assert search_manifest_using_method(manifest, method, "old") == {"versioned_model.v1"}
+    assert search_manifest_using_method(manifest, method, "prerelease") == {"versioned_model.v3"}
+    assert search_manifest_using_method(manifest, method, "none") == {
+        "table_model_py",
+        "union_model",
+        "view_model",
+        "mynamespace.ephemeral_model",
+        "table_model_csv",
+        "ephemeral_model",
+        "mynamespace.union_model",
+        "table_model",
+        "ext_model",
+    }
+
+
 def test_select_exposure(manifest):
     exposure = make_exposure("test", "my_exposure")
     manifest.exposures[exposure.unique_id] = exposure

tests/functional/graph_selection/fixtures.py

@@ -48,6 +48,16 @@
     - name: gender
       tests:
       - unique
+  - name: versioned
+    latest_version: 2
+    versions:
+      - v: 1
+      - v: 2
+      - v: 3
+      - v: 4.5
+      - v: "5.0"
+      - v: 21
+      - v: "test"
 
 sources:
   - name: raw
@@ -61,6 +71,7 @@
     depends_on:
       - ref('users')
       - ref('users_rollup')
+      - ref('versioned', v=3)
     owner:
       email: nope@example.com
   - name: seed_ml_exposure
@@ -194,6 +205,7 @@ def models(self):
             "patch_path_selection_schema.yml": patch_path_selection_schema_yml,
             "base_users.sql": base_users_sql,
             "users.sql": users_sql,
+            "versioned_v3.sql": base_users_sql,
             "users_rollup.sql": users_rollup_sql,
             "users_rollup_dependency.sql": users_rollup_dependency_sql,
             "emails.sql": emails_sql,

tests/functional/graph_selection/test_graph_selection.py

@@ -245,6 +245,7 @@ def test_exposure_parents(self, project):
             "test.unique_users_rollup_gender",
             "test.users",
             "test.users_rollup",
+            "test.versioned.v3",
         ]
         results = run_dbt(["run", "-m", "+exposure:user_exposure"], expect_pass=False)
         check_result_nodes_by_name(

tests/functional/graph_selection/test_group_selection.py

@@ -41,6 +41,7 @@ def models(self):
             "base_users.sql": base_users_sql,
             "users.sql": users_sql,
             "users_rollup.sql": users_rollup_sql,
+            "versioned_v3.sql": base_users_sql,
             "users_rollup_dependency.sql": users_rollup_dependency_sql,
             "emails.sql": emails_sql,
             "emails_alt.sql": emails_alt_sql,
@@ -112,6 +113,7 @@ def test_select_group_and_children_selector_str(self, project):  # noqa
             "test.users",
             "test.users_rollup",
             "test.users_rollup_dependency",
+            "test.versioned.v3",
         ]
         results = run_dbt(["ls", "--selector", "users_grouped_childrens_parents"])
         assert sorted(results) == expected

tests/functional/graph_selection/test_version_selection.py

@@ -0,0 +1,111 @@
+import pytest
+
+from dbt.tests.util import run_dbt, read_file
+from tests.functional.graph_selection.fixtures import (
+    schema_yml,
+    base_users_sql,
+    users_sql,
+    users_rollup_sql,
+    properties_yml,
+)
+
+
+selectors_yml = """
+            selectors:
+              - name: version_specified_as_string_str
+                definition: version:latest
+              - name: version_specified_as_string_dict
+                definition:
+                  method: version
+                  value: latest
+              - name: version_childrens_parents
+                definition:
+                  method: version
+                  value: latest
+                  childrens_parents: true
+"""
+
+
+class TestVersionSelection:
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {
+            "schema.yml": schema_yml,
+            "versioned_v1.sql": users_sql,
+            "versioned_v2.sql": users_sql,
+            "versioned_v3.sql": users_sql,
+            "versioned_v4.5.sql": users_sql,
+            "versioned_v5.0.sql": users_sql,
+            "versioned_v21.sql": users_sql,
+            "versioned_vtest.sql": users_sql,
+            "base_users.sql": base_users_sql,
+            "users.sql": users_sql,
+            "users_rollup.sql": users_rollup_sql,
+        }
+
+    @pytest.fixture(scope="class")
+    def seeds(self, test_data_dir):
+        # Read seed file and return
+        seeds = {"properties.yml": properties_yml}
+        for seed_file in ["seed.csv", "summary_expected.csv"]:
+            seeds[seed_file] = read_file(test_data_dir, seed_file)
+        return seeds
+
+    @pytest.fixture(scope="class")
+    def selectors(self):
+        return selectors_yml
+
+    def test_select_none_versions(self, project):
+        results = run_dbt(["ls", "--select", "version:none"])
+        assert sorted(results) == [
+            "test.base_users",
+            "test.unique_users_id",
+            "test.unique_users_rollup_gender",
+            "test.users",
+            "test.users_rollup",
+        ]
+
+    def test_select_latest_versions(self, project):
+        results = run_dbt(["ls", "--select", "version:latest"])
+        assert sorted(results) == ["test.versioned.v2"]
+
+    def test_select_old_versions(self, project):
+        results = run_dbt(["ls", "--select", "version:old"])
+        assert sorted(results) == ["test.versioned.v1"]
+
+    def test_select_prerelease_versions(self, project):
+        results = run_dbt(["ls", "--select", "version:prerelease"])
+        assert sorted(results) == [
+            "test.versioned.v21",
+            "test.versioned.v3",
+            "test.versioned.v4.5",
+            "test.versioned.v5.0",
+            "test.versioned.vtest",
+        ]
+
+    def test_select_version_selector_str(self, project):
+        results = run_dbt(["ls", "--selector", "version_specified_as_string_str"])
+        assert sorted(results) == ["test.versioned.v2"]
+
+    def test_select_version_selector_dict(self, project):
+        results = run_dbt(["ls", "--selector", "version_specified_as_string_dict"])
+        assert sorted(results) == ["test.versioned.v2"]
+
+    def test_select_models_by_version_and_children(self, project):  # noqa
+        results = run_dbt(["ls", "--models", "+version:latest+"])
+        assert sorted(results) == ["test.base_users", "test.versioned.v2"]
+
+    def test_select_version_and_children(self, project):  # noqa
+        expected = ["source:test.raw.seed", "test.base_users", "test.versioned.v2"]
+        results = run_dbt(["ls", "--select", "+version:latest+"])
+        assert sorted(results) == expected
+
+    def test_select_group_and_children_selector_str(self, project):  # noqa
+        expected = ["source:test.raw.seed", "test.base_users", "test.versioned.v2"]
+        results = run_dbt(["ls", "--selector", "version_childrens_parents"])
+        assert sorted(results) == expected
+
+    # 2 versions
+    def test_select_models_two_versions(self, project):
+        results = run_dbt(["ls", "--models", "version:latest version:old"])
+        assert sorted(results) == ["test.versioned.v1", "test.versioned.v2"]