Add SemanticModel Node Type

core/dbt/contracts/files.py

@@ -228,6 +228,7 @@ class SchemaSourceFile(BaseSourceFile):
     groups: List[str] = field(default_factory=list)
     # node patches contain models, seeds, snapshots, analyses
     ndp: List[str] = field(default_factory=list)
+    semantic_models: List[str] = field(default_factory=list)
     # any macro patches in this file by macro unique_id.
     mcp: Dict[str, str] = field(default_factory=dict)
     # any source patches in this file. The entries are package, name pairs

core/dbt/contracts/graph/manifest.py

@@ -25,20 +25,21 @@
 from dbt.contracts.publication import ProjectDependencies, PublicationConfig, PublicModel
 
 from dbt.contracts.graph.nodes import (
-    Macro,
+    BaseNode,
     Documentation,
-    SourceDefinition,
-    GenericTestNode,
     Exposure,
-    Metric,
+    GenericTestNode,
+    GraphMemberNode,
     Group,
-    UnpatchedSourceDefinition,
+    Macro,
     ManifestNode,
-    GraphMemberNode,
-    ResultNode,
-    BaseNode,
     ManifestOrPublicNode,
+    Metric,
     ModelNode,
+    ResultNode,
+    SemanticModel,
+    SourceDefinition,
+    UnpatchedSourceDefinition,
 )
 from dbt.contracts.graph.unparsed import SourcePatch, NodeVersion, UnparsedVersion
 from dbt.contracts.graph.manifest_upgrade import upgrade_manifest_json
@@ -689,6 +690,7 @@ class Manifest(MacroMethods, DataClassMessagePackMixin, dbtClassMixin):
     public_nodes: MutableMapping[str, PublicModel] = field(default_factory=dict)
     project_dependencies: Optional[ProjectDependencies] = None
     publications: MutableMapping[str, PublicationConfig] = field(default_factory=dict)
+    semantic_models: MutableMapping[str, SemanticModel] = field(default_factory=dict)
 
     _doc_lookup: Optional[DocLookup] = field(
         default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None}
@@ -1212,6 +1214,11 @@ def add_doc(self, source_file: SourceFile, doc: Documentation):
         self.docs[doc.unique_id] = doc
         source_file.docs.append(doc.unique_id)
 
+    def add_semantic_model(self, source_file: SchemaSourceFile, semantic_model: SemanticModel):
+        _check_duplicates(semantic_model, self.semantic_models)
+        self.semantic_models[semantic_model.unique_id] = semantic_model
+        source_file.semantic_models.append(semantic_model.unique_id)
+
     # end of methods formerly in ParseResult
 
     # Provide support for copy.deepcopy() - we just need to avoid the lock!
@@ -1311,6 +1318,9 @@ class WritableManifest(ArtifactMixin):
     public_nodes: Mapping[UniqueID, PublicModel] = field(
         metadata=dict(description=("The public models used in the dbt project"))
     )
+    semantic_models: Mapping[UniqueID, SemanticModel] = field(
+        metadata=dict(description=("The semantic models defined in the dbt project"))
+    )
     metadata: ManifestMetadata = field(
         metadata=dict(
             description="Metadata about the manifest",

core/dbt/contracts/graph/nodes.py

@@ -6,29 +6,23 @@
 import hashlib
 
 from mashumaro.types import SerializableType
-from typing import (
-    Optional,
-    Union,
-    List,
-    Dict,
-    Any,
-    Sequence,
-    Tuple,
-    Iterator,
-)
+from typing import Optional, Union, List, Dict, Any, Sequence, Tuple, Iterator, Protocol
 
 from dbt.dataclass_schema import dbtClassMixin, ExtensibleDbtClassMixin
 
 from dbt.clients.system import write_file
 from dbt.contracts.files import FileHash
 from dbt.contracts.graph.unparsed import (
+    Dimension,
     Docs,
+    Entity,
     ExposureType,
     ExternalTable,
     FreshnessThreshold,
     HasYamlMetadata,
     MacroArgument,
     MaturityType,
+    Measure,
     MetricFilter,
     MetricTime,
     Owner,
@@ -62,12 +56,6 @@
     EmptySnapshotConfig,
     SnapshotConfig,
 )
-import sys
-
-if sys.version_info >= (3, 8):
-    from typing import Protocol
-else:
-    from typing_extensions import Protocol
 
 
 # =====================================================================
@@ -552,6 +540,30 @@ def depends_on_macros(self):
         return self.depends_on.macros
 
 
+@dataclass
+class FileSlice(dbtClassMixin, Replaceable):
+    """Provides file slice level context about what something was created from.
+
+    Implementation of the dbt-semantic-interfaces `FileSlice` protocol
+    """
+
+    filename: str
+    content: str
+    start_line_number: int
+    end_line_number: int
+
+
+@dataclass
+class Metadata(dbtClassMixin, Replaceable):
+    """Provides file context about what something was created from.
+
+    Implementation of the dbt-semantic-interfaces `Metadata` protocol
+    """
+
+    repo_file_path: str
+    file_slice: FileSlice
+
+
 # ====================================
 # CompiledNode subclasses
 # ====================================
@@ -1399,6 +1411,28 @@ class Group(BaseNode):
     resource_type: NodeType = field(metadata={"restrict": [NodeType.Group]})
 
 
+# ====================================
+# SemanticModel and related classes
+# ====================================
+
+
+@dataclass
+class NodeRelation(dbtClassMixin, Replaceable):
+    alias: str
+    schema_name: str
+    relation_name: str
+    database: Optional[str] = None
+
+
+@dataclass
+class SemanticModel(GraphNode):
+    description: Optional[str]
+    node_relation: NodeRelation
+    entities: Sequence[Entity]
+    measures: Sequence[Measure]
+    dimensions: Sequence[Dimension]
+
+
 # ====================================
 # Patches
 # ====================================

core/dbt/contracts/graph/unparsed.py

@@ -661,6 +661,62 @@ def validate(cls, data):
             raise ValidationError("Group owner must have at least one of 'name' or 'email'.")
 
 
+#
+# semantic interfaces unparsed objects
+#
+
+
+@dataclass
+class Entity(dbtClassMixin, Replaceable):
+    name: str
+    type: str  # actually an enum
+    description: Optional[str] = None
+    role: Optional[str] = None
+    expr: Optional[str] = None
+
+
+@dataclass
+class MeasureAggregationParameters(dbtClassMixin, Replaceable):
+    percentile: Optional[float] = None
+    use_discrete_percentile: bool = False
+    use_approximate_percentile: bool = False
+
+
+@dataclass
+class Measure(dbtClassMixin, Replaceable):
+    name: str
+    agg: str  # actually an enum
+    description: Optional[str] = None
+    create_metric: Optional[bool] = None
+    expr: Optional[str] = None
+    agg_params: Optional[MeasureAggregationParameters] = None
+    non_additive_dimension: Optional[Dict[str, Any]] = None  # TODO: Refine type as class?
+    agg_time_dimension: Optional[str] = None
+
+
+@dataclass
+class Dimension(dbtClassMixin, Replaceable):
+    name: str
+    type: str  # actually an enum
+    description: Optional[str] = None
+    is_partition: Optional[bool] = False
+    type_params: Optional[Dict[str, Any]] = field(
+        default_factory=dict
+    )  # TODO: Refine type as class?
+    expr: Optional[str] = None
+    # TODO metadata: Optional[Metadata] (this would actually be the YML for the dimension)
+
+
+@dataclass
+class UnparsedSemanticModel(dbtClassMixin, Replaceable):
+    name: str
+    description: Optional[str]
+    model: str  # looks like "ref(...)"
+    entities: List[Entity] = field(default_factory=list)
+    measures: List[Measure] = field(default_factory=list)
+    dimensions: List[Dimension] = field(default_factory=list)
+
+
 def normalize_date(d: Optional[datetime.date]) -> Optional[datetime.datetime]:
     """Convert date to datetime (at midnight), and add local time zone if naive"""
     if d is None:

core/dbt/node_types.py

@@ -33,6 +33,7 @@ class NodeType(StrEnum):
     Exposure = "exposure"
     Metric = "metric"
     Group = "group"
+    SemanticModel = "semantic model"
 
     @classmethod
     def executable(cls) -> List["NodeType"]:

core/dbt/parser/schema_yaml_readers.py

@@ -1,8 +1,13 @@
 from dbt.parser.schemas import YamlReader, SchemaParser
 from dbt.parser.common import YamlBlock
 from dbt.node_types import NodeType
-from dbt.contracts.graph.unparsed import UnparsedExposure, UnparsedMetric, UnparsedGroup
-from dbt.contracts.graph.nodes import Exposure, Metric, Group
+from dbt.contracts.graph.unparsed import (
+    UnparsedExposure,
+    UnparsedGroup,
+    UnparsedMetric,
+    UnparsedSemanticModel,
+)
+from dbt.contracts.graph.nodes import Exposure, Group, Metric, NodeRelation, SemanticModel
 from dbt.exceptions import DbtInternalError, YamlParseDictError, JSONValidationError
 from dbt.context.providers import generate_parse_exposure, generate_parse_metrics
 from dbt.contracts.graph.model_config import MetricConfig, ExposureConfig
@@ -269,3 +274,47 @@ def parse(self):
                 raise YamlParseDictError(self.yaml.path, self.key, data, exc)
 
             self.parse_group(unparsed)
+
+
+class SemanticModelParser(YamlReader):
+    def __init__(self, schema_parser: SchemaParser, yaml: YamlBlock):
+        super().__init__(schema_parser, yaml, "semantic_models")
+        self.schema_parser = schema_parser
+        self.yaml = yaml
+
+    def parse_semantic_model(self, unparsed: UnparsedSemanticModel):
+        package_name = self.project.project_name
+        unique_id = f"{NodeType.SemanticModel}.{package_name}.{unparsed.name}"
+        path = self.yaml.path.relative_path
+
+        fqn = self.schema_parser.get_fqn_prefix(path)
+        fqn.append(unparsed.name)
+
+        parsed = SemanticModel(
+            description=unparsed.description,
+            fqn=fqn,
+            name=unparsed.name,
+            node_relation=NodeRelation(
+                alias="", database="", relation_name="", schema_name=""
+            ),  # TODO: arguments
+            original_file_path=self.yaml.path.original_file_path,
+            package_name=package_name,
+            path=path,
+            resource_type=NodeType.SemanticModel,
+            unique_id=unique_id,
+            entities=unparsed.entities,
+            measures=unparsed.measures,
+            dimensions=unparsed.dimensions,
+        )
+
+        self.manifest.add_semantic_model(self.yaml.file, parsed)
+
+    def parse(self):
+        for data in self.get_key_dicts():
+            try:
+                UnparsedSemanticModel.validate(data)
+                unparsed = UnparsedSemanticModel.from_dict(data)
+            except (ValidationError, JSONValidationError) as exc:
+                raise YamlParseDictError(self.yaml.path, self.key, data, exc)
+
+            self.parse_semantic_model(unparsed)

core/dbt/parser/schemas.py

@@ -74,6 +74,7 @@
     "analyses",
     "exposures",
     "metrics",
+    "semantic_models",
 )
 
 
@@ -217,6 +218,12 @@ def parse_file(self, block: FileBlock, dct: Dict = None) -> None:
                 group_parser = GroupParser(self, yaml_block)
                 group_parser.parse()
 
+            if "semantic_models" in dct:
+                from dbt.parser.schema_yaml_readers import SemanticModelParser
+
+                semantic_model_parser = SemanticModelParser(self, yaml_block)
+                semantic_model_parser.parse()
+
 
 Parsed = TypeVar("Parsed", UnpatchedSourceDefinition, ParsedNodePatch, ParsedMacroPatch)
 NodeTarget = TypeVar("NodeTarget", UnparsedNodeUpdate, UnparsedAnalysisUpdate, UnparsedModelUpdate)

core/setup.py

@@ -70,6 +70,7 @@
         "cffi>=1.9,<2.0.0",
         "pyyaml>=5.3",
         "urllib3~=1.0",
+        "dbt-semantic-interfaces==0.1.0.dev3",
     ],
     zip_safe=False,
     classifiers=[