From 1761f7f38140a247afdaf6e4e1eae51d4ca80ff5 Mon Sep 17 00:00:00 2001
From: Emily Rockman <emily.rockman@dbtlabs.com>
Date: Mon, 7 Aug 2023 15:24:16 -0500
Subject: [PATCH] add semantic models

---
 src/app/components/graph/graph-launcher.js    |  4 ++
 src/app/components/model_tree/model_tree.html | 11 +++
 src/app/components/references/index.js        | 10 +--
 src/app/components/search/search.js           |  2 +
 src/app/docs/index.js                         |  1 +
 src/app/docs/metric.js                        |  6 +-
 src/app/docs/semantic_model.html              | 71 +++++++++++++++++++
 src/app/docs/semantic_model.js                | 46 ++++++++++++
 src/app/graph/index.js                        |  2 +
 src/app/index.routes.js                       |  9 +++
 src/app/main/index.js                         |  1 +
 src/app/services/code_service.js              |  2 +-
 src/app/services/graph.service.js             | 13 +++-
 src/app/services/node_selection_service.js    |  9 ++-
 src/app/services/project_service.js           | 56 ++++++++++++++-
 src/app/services/selector_matcher.js          | 26 ++++++-
 src/app/services/selector_methods.test.js     | 15 ++++
 17 files changed, 266 insertions(+), 18 deletions(-)
 create mode 100644 src/app/docs/semantic_model.html
 create mode 100644 src/app/docs/semantic_model.js

diff --git a/src/app/components/graph/graph-launcher.js b/src/app/components/graph/graph-launcher.js
index 5ef43c6f0..8545f7dd3 100644
--- a/src/app/components/graph/graph-launcher.js
+++ b/src/app/components/graph/graph-launcher.js
@@ -155,6 +155,8 @@ angular
                     var nodes = graph.showFullGraph('exposure:' + node.name);
                 } else if (node && node.resource_type == 'metric') {
                     var nodes = graph.showFullGraph('metric:' + node.name);
+                } else if (node && node.resource_type == 'semantic_model') {
+                    var nodes = graph.showFullGraph('semantic_model:' + node.name);
                 } else {
                     var nodes = graph.showFullGraph(node_name);
                 }
@@ -170,6 +172,8 @@ angular
                     var nodes = graph.showVerticalGraph('exposure:' + node.name, true);
                 } else if (node && node.resource_type == 'metric') {
                     var nodes = graph.showVerticalGraph('metric:' + node.name, true);
+                } else if (node && node.resource_type == 'semantic_model') {
+                    var nodes = graph.showVerticalGraph('semantic_model:' + node.name, true);
                 } else {
                     var nodes = graph.showVerticalGraph(node.name, true);
                 }
diff --git a/src/app/components/model_tree/model_tree.html b/src/app/components/model_tree/model_tree.html
index 76ab733d1..db7f31e16 100644
--- a/src/app/components/model_tree/model_tree.html
+++ b/src/app/components/model_tree/model_tree.html
@@ -78,6 +78,17 @@
                     </ul>
                     <br />
                 </div>
+                
+                <div ng-show="tree.semantic_models.length > 0">
+                    <strong>Semantic Models</strong>
+                    <ul style="display: block">
+                        <model-tree-line
+                            item="item"
+                            resource-type="semantic_model"
+                            ng-repeat="item in tree.semantic_models"></model-tree-line>
+                    </ul>
+                    <br />
+                </div>
 
                 <strong>Projects</strong>
                 <ul>
diff --git a/src/app/components/references/index.js b/src/app/components/references/index.js
index 5642fc928..df0a7eb55 100644
--- a/src/app/components/references/index.js
+++ b/src/app/components/references/index.js
@@ -40,13 +40,15 @@ angular
                 } else if (type == 'analysis') {
                     return 'Analyses';
                 } else if (type == 'macro') {
-                      return 'Macros';
+                    return 'Macros';
                 } else if (type == 'exposure') {
-                      return 'Exposures';
+                    return 'Exposures';
                 } else if (type == 'metric') {
-                      return 'Metrics';
+                    return 'Metrics';
+                } else if (type == 'semantic_model') {
+                    return 'Semantic Models';
                 } else if (type == 'operation') {
-                        return 'Operations';
+                    return 'Operations';
                 } else {
                     return 'Nodes';
                 }
diff --git a/src/app/components/search/search.js b/src/app/components/search/search.js
index e96b9c198..c6bee4423 100644
--- a/src/app/components/search/search.js
+++ b/src/app/components/search/search.js
@@ -48,6 +48,8 @@ angular
                     return model.package_name + "." + model.name;
                 } else if (model.resource_type == 'metric') {
                     return model.label;
+                } else if (model.resource_type == 'semantic_model') {
+                    return model.name;
                 } else if (model.resource_type == 'exposure') {
                     return model.label;
                 } else if (model.resource_type == 'model' && model.version != null) {
diff --git a/src/app/docs/index.js b/src/app/docs/index.js
index 1435e2dbe..19c501f6e 100644
--- a/src/app/docs/index.js
+++ b/src/app/docs/index.js
@@ -8,4 +8,5 @@ require('./macro');
 require('./analysis');
 require('./exposure');
 require('./metric');
+require('./semantic_model');
 require('./operation');
diff --git a/src/app/docs/metric.js b/src/app/docs/metric.js
index dc871bf6f..74bfdc144 100644
--- a/src/app/docs/metric.js
+++ b/src/app/docs/metric.js
@@ -23,9 +23,9 @@ angular
         $scope.parents = dag_utils.getParents(project, metric);
         $scope.parentsLength = $scope.parents.length;
 
-        $scope.versions = {
-            'Definition': codeService.generateMetricSQL($scope.metric)
-        }
+        // $scope.versions = {
+        //     'Definition': codeService.generateMetricSQL($scope.metric)
+        // }
 
         const metric_type =  $scope.metric.type === 'expression'
             ? 'Expression metric'
diff --git a/src/app/docs/semantic_model.html b/src/app/docs/semantic_model.html
new file mode 100644
index 000000000..71c346b3d
--- /dev/null
+++ b/src/app/docs/semantic_model.html
@@ -0,0 +1,71 @@
+<style>
+/* TODO */
+.section-target {
+    top: -8em;
+}
+
+.noflex {
+    flex: 0 0 160px !important;
+}
+
+.highlight {
+    color: #24292e;
+    background-color: white;
+}
+
+</style>
+
+<div class='app-scroll'>
+    <div class="app-links app-sticky">
+        <div class="app-title">
+            <div class="app-frame app-pad app-flush-bottom">
+
+                <h1>
+                    <span class="break">{{ semantic_model.name }}</span>
+                    <small>semantic_model</small>
+
+                    <div class='clearfix'></div>
+                </h1>
+
+            </div>
+        </div>
+        <div class="app-frame app-pad-h">
+            <ul class="nav nav-tabs">
+                <li ui-sref-active='active'><a ui-sref="dbt.semantic_model({'#': 'details'})">Details</a></li>
+                <li ui-sref-active='active'><a ui-sref="dbt.semantic_model({'#': 'description'})">Description</a></li>
+                <li ui-sref-active='active' ng-show = "parentsLength != 0"><a ui-sref="dbt.semantic_model({'#': 'depends_on'})">Depends On</a></li>
+            </ul>
+        </div>
+    </div>
+    <div class="app-details">
+        <div class="app-frame app-pad">
+
+            <section class="section">
+                <div class="section-target" id="details"></div>
+                <table-details model="semantic_model" extras="extra_table_fields" />
+            </section>
+
+            <section class="section">
+                <div class="section-target" id="description"></div>
+                <div class="section-content">
+                    <h6>Description</h6>
+                    <div class="panel">
+                        <div class="panel-body">
+                            <div ng-if="semantic_model.description" class="model-markdown" marked="semantic_model.description"></div>
+                            <div ng-if="!semantic_model.description">This {{ semantic_model.resource_type }} is not currently documented</div>
+                        </div>
+                    </div>
+                </div>
+            </section>
+
+            <section class="section" ng-show = "parentsLength != 0">
+                <div class="section-target" id="depends_on"></div>
+                <div class="section-content">
+                    <h6>Depends On</h6>
+                    <reference-list references="parents" node="semantic_model" />
+                </div>
+            </section>
+
+        </div>
+    </div>
+</div>
diff --git a/src/app/docs/semantic_model.js b/src/app/docs/semantic_model.js
new file mode 100644
index 000000000..e2df61a17
--- /dev/null
+++ b/src/app/docs/semantic_model.js
@@ -0,0 +1,46 @@
+'use strict';
+
+const angular = require('angular');
+const dag_utils = require('./dag_utils')
+require("./styles.css");
+
+angular
+.module('dbt')
+.controller('SemanticModelCtrl', ['$scope', '$state', 'project', 'code', '$anchorScroll', '$location',
+            function($scope, $state, projectService, codeService, $anchorScroll, $location) {
+
+    $scope.model_uid = $state.params.unique_id;
+    $scope.project = projectService;
+
+    $scope.codeService = codeService;
+    $scope.extra_table_fields = [];
+    $scope.versions = {};
+
+    $scope.semantic_model = {};
+    projectService.ready(function(project) {
+        let semantic_model = project.nodes[$scope.model_uid];
+        $scope.semantic_model = semantic_model;
+        $scope.parents = dag_utils.getParents(project, semantic_model);
+        $scope.parentsLength = $scope.parents.length;
+
+        // $scope.versions = {
+        //     'Definition': codeService.generateSemanticModelSQL($scope.semantic_model)
+        // }
+
+        const semantic_model_type =  $scope.semantic_model.type === 'expression'
+            ? 'Expression semantic_model'
+            : 'Aggregate semantic_model';
+
+        $scope.extra_table_fields = [
+            {
+                name: "Semantic Model Type",
+                value: semantic_model_type,
+            },
+            {
+                name: "Semantic Model name",
+                value: $scope.semantic_model.name
+            }
+        ]
+
+    })
+}]);
diff --git a/src/app/graph/index.js b/src/app/graph/index.js
index 9e298f32e..78255ecd7 100644
--- a/src/app/graph/index.js
+++ b/src/app/graph/index.js
@@ -23,6 +23,8 @@ angular
                 return 'exposure:' + node.name;
             } else if (node && node.resource_type == 'metric') {
                 return 'metric:' + node.name;
+            } else if (node && node.resource_type == 'semantic_model') {
+                return 'semantic_model:' + node.name;
             } else if (node && node.resource_type == 'model' && node.version != null) {
                 return node.label;
             } else if (node.name)  {
diff --git a/src/app/index.routes.js b/src/app/index.routes.js
index ee4f720ca..00e73bae2 100644
--- a/src/app/index.routes.js
+++ b/src/app/index.routes.js
@@ -18,6 +18,7 @@ const templates = {
     macro: require('./docs/macro.html'),
     exposure: require('./docs/exposure.html'),
     metric: require('./docs/metric.html'),
+    semantic_model: require('./docs/semantic_model.html'),
     operation: require('./docs/operation.html'),
 }
 
@@ -133,6 +134,14 @@ angular
                 unique_id: {type: 'string'}
             },
         })
+        .state('dbt.semantic_model', {
+            url: 'semantic_model/:unique_id?section&' + graph_params,
+            controller: 'SemanticModelCtrl',
+            templateUrl: templates.semantic_model,
+            params: {
+                unique_id: {type: 'string'}
+            },
+        })
         .state('dbt.operation', {
             url: 'operation/:unique_id?section&' + graph_params,
             controller: 'OperationCtrl',
diff --git a/src/app/main/index.js b/src/app/main/index.js
index 88f4f5203..484dcb1b9 100644
--- a/src/app/main/index.js
+++ b/src/app/main/index.js
@@ -87,6 +87,7 @@ angular
         $scope.tree.sources = tree.sources;
         $scope.tree.exposures = tree.exposures;
         $scope.tree.metrics = tree.metrics;
+        $scope.tree.semantic_models = tree.semantic_models;
         $scope.tree.groups = tree.groups;
 
         setTimeout(function() {
diff --git a/src/app/services/code_service.js b/src/app/services/code_service.js
index c30d2cb8b..400ddc52e 100644
--- a/src/app/services/code_service.js
+++ b/src/app/services/code_service.js
@@ -61,7 +61,7 @@ angular
         query.push("from " + rel)
         return query.join("\n");
     }
-
+    // TODO: revisit this
     service.generateMetricSQL = function(metric) {
         if (metric.calculation_method == 'derived') {
             return "-- derived\n" + metric.expression;
diff --git a/src/app/services/graph.service.js b/src/app/services/graph.service.js
index 5d1d86fa2..43c7935e0 100644
--- a/src/app/services/graph.service.js
+++ b/src/app/services/graph.service.js
@@ -197,6 +197,13 @@ angular
                         'border-color': '#ff5688',
                     }
                 },
+                {
+                    selector: 'node[resource_type="semantic_model"]',
+                    style: {
+                        'background-color': '#ffa8c2',
+                        'border-color': '#ffa8c2',
+                    }
+                },
                 {
                     selector: 'node[language="python"]',
                     style: {
@@ -372,8 +379,8 @@ angular
 
 
         _.each(_.filter(service.manifest.nodes, function(node) {
-            // operation needs to be a graph type so that the parent/child mpa can be resolved even though we won't be displaying it
-            var is_graph_type = _.includes(['model', 'seed', 'source', 'snapshot', 'analysis', 'exposure', 'metric', 'operation'], node.resource_type);
+            // operation needs to be a graph type so that the parent/child map can be resolved even though we won't be displaying it
+            var is_graph_type = _.includes(['model', 'seed', 'source', 'snapshot', 'analysis', 'exposure', 'metric', 'semantic_model', 'operation'], node.resource_type);
             var is_singular_test = node.resource_type == 'test' && !node.hasOwnProperty('test_metadata');
             return is_graph_type || is_singular_test;
         }), function(node) {
@@ -394,7 +401,7 @@ angular
                 var parent_node = service.manifest.nodes[parent];
                 var child_node = service.manifest.nodes[child];
 
-                if (!_.includes(['model', 'source', 'seed', 'snapshot', 'metric'], parent_node.resource_type)) {
+                if (!_.includes(['model', 'source', 'seed', 'snapshot', 'metric', 'semantic_model'], parent_node.resource_type)) {
                     return;
                 } else if (child_node.resource_type == 'test' && child_node.hasOwnProperty('test_metadata')) {
                     return;
diff --git a/src/app/services/node_selection_service.js b/src/app/services/node_selection_service.js
index 84e24477a..4ecca497c 100644
--- a/src/app/services/node_selection_service.js
+++ b/src/app/services/node_selection_service.js
@@ -20,7 +20,8 @@ angular
             'test',
             'analysis',
             'exposure',
-            'metric'
+            'metric',
+            'semantic_model'
         ],
         depth: 1,
     };
@@ -36,7 +37,7 @@ angular
         options: {
             packages: [],
             tags: [null],
-            resource_types: ['model', 'seed', 'snapshot', 'source', 'test', 'analysis', 'exposure', 'metric'],
+            resource_types: ['model', 'seed', 'snapshot', 'source', 'test', 'analysis', 'exposure', 'metric', 'semantic_model'],
         }
     };
 
@@ -60,6 +61,8 @@ angular
             include_selection = '+exposure:' + node.name;
         } else if (node && node.resource_type == 'metric') {
             include_selection = '+metric:' + node.name;
+        } else if (node && node.resource_type == 'semantic_model') {
+            include_selection = '+semantic_model:' + node.name;
         } else if (node && _.includes(['analysis', 'test'], node.resource_type)) {
             include_selection = '+' + node.name;
         } else {
@@ -92,7 +95,7 @@ angular
         if (node.resource_type == 'source') {
             pre += "source:"
             node_name = node.source_name + "." + node.name;
-        } else if (['exposure', 'metric'].indexOf(node.resource_type) > -1) {
+        } else if (['exposure', 'metric', 'semantic_model'].indexOf(node.resource_type) > -1) {
             pre += node.resource_type + ":"
             node_name = node.name;
         } else {
diff --git a/src/app/services/project_service.js b/src/app/services/project_service.js
index 10ac8016c..3ec2620a4 100644
--- a/src/app/services/project_service.js
+++ b/src/app/services/project_service.js
@@ -162,6 +162,12 @@ angular
                 service.files.manifest.nodes[node.unique_id] = node;
             });
 
+            // Add semantic models back into nodes to make site logic work
+            _.each(service.files.manifest.semantic_models, function(node) {
+                service.files.manifest.nodes[node.unique_id] = node;
+                node.label = node.name;
+            });
+
             var adapter = service.files.manifest.metadata.adapter_type;
             var macros = clean_project_macros(service.files.manifest.macros, adapter);
             service.files.manifest.macros = macros;
@@ -255,7 +261,7 @@ angular
             });
 
             var search_nodes = _.filter(service.project.nodes, function(node) {
-                return _.includes(['model', 'source', 'seed', 'snapshot', 'analysis', 'exposure', 'metric'], node.resource_type);
+                return _.includes(['model', 'source', 'seed', 'snapshot', 'analysis', 'exposure', 'metric', 'semantic_model'], node.resource_type);
             });
 
             service.project.searchable = _.filter(search_nodes.concat(search_macros), function(obj) {
@@ -383,7 +389,7 @@ angular
                     return true;
                 }
 
-                var accepted = ['snapshot', 'source', 'seed', 'model', 'analysis', 'exposure', 'metric'];
+                var accepted = ['snapshot', 'source', 'seed', 'model', 'analysis', 'exposure', 'metric', 'semantic_model'];
                 return _.includes(accepted, node.resource_type);
             })
 
@@ -399,6 +405,9 @@ angular
             
             var metrics = _.values(service.project.metrics);
             service.tree.metrics = buildMetricTree(metrics, select);
+            
+            var semantic_models = _.values(service.project.semantic_models);
+            service.tree.semantic_models = buildSemanticModelTree(semantic_models, select);
 
             cb(service.tree);
         });
@@ -430,6 +439,7 @@ angular
         service.updateSelectedInTree(select, service.tree.sources);
         service.updateSelectedInTree(select, service.tree.exposures);
         service.updateSelectedInTree(select, service.tree.metrics);
+        service.updateSelectedInTree(select, service.tree.semantic_models);
 
         return service.tree;
     }
@@ -573,6 +583,46 @@ angular
         return metrics
     }
 
+    function buildSemanticModelTree(nodes, select) {
+        var semantic_models = {}
+
+        _.each(nodes, function(node) {
+            var name = node.name;
+
+            var project = node.package_name;
+
+            var is_active = node.unique_id == select;
+
+            if (!semantic_models[project]) {
+                semantic_models[project] = {
+                    type: "folder",
+                    name: project,
+                    active: is_active,
+                    items: []
+                };
+            } else if (is_active) {
+                semantic_models[project].active = true;
+            }
+
+            semantic_models[project].items.push({
+                type: 'file',
+                name: node.name,
+                node: node,
+                active: is_active,
+                unique_id: node.unique_id,
+                node_type: 'semantic_model'
+            })
+        });
+
+        var semantic_models = _.sortBy(_.values(semantic_models), 'name');
+
+        _.each(semantic_models, function(semantic_model) {
+            semantic_models.items = _.sortBy(semantic_models.items, 'name');
+        });
+
+        return semantic_models
+    }
+
     function consolidateAdapterMacros(macros, adapter) {
         var adapter_macros = {};
         _.each(macros, function(macro) {
@@ -619,7 +669,7 @@ angular
 
         _.each(nodes.concat(macros), function(node) {
             var show = _.get(node, ['docs', 'show'], true);
-            if (node.resource_type == 'source' || node.resource_type == 'exposure' || node.resource_type == 'metric') {
+            if (node.resource_type == 'source' || node.resource_type == 'exposure' || node.resource_type == 'metric' || node.resource_type == 'semantic_model') {
                 // no sources in the model tree, sorry
                 return;
             } else if (!show) {
diff --git a/src/app/services/selector_matcher.js b/src/app/services/selector_matcher.js
index 772b29c4e..7e79a0815 100644
--- a/src/app/services/selector_matcher.js
+++ b/src/app/services/selector_matcher.js
@@ -9,6 +9,7 @@ var SELECTOR_TYPE = {
     SOURCE: 'source',
     EXPOSURE: 'exposure',
     METRIC: 'metric',
+    SEMANTIC_MODEL: 'semantic_model',
     GROUP: 'group',
     PATH: 'path',
     FILE: 'file',
@@ -25,6 +26,7 @@ NODE_MATCHERS[SELECTOR_TYPE.TAG] = getNodesByTag;
 NODE_MATCHERS[SELECTOR_TYPE.SOURCE] = getNodesBySource;
 NODE_MATCHERS[SELECTOR_TYPE.EXPOSURE] = getNodesByExposure;
 NODE_MATCHERS[SELECTOR_TYPE.METRIC] = getNodesByMetric;
+NODE_MATCHERS[SELECTOR_TYPE.SEMANTIC_MODEL] = getNodesBySemanticModel;
 NODE_MATCHERS[SELECTOR_TYPE.GROUP] = getNodesByGroup;
 NODE_MATCHERS[SELECTOR_TYPE.PATH] = getNodesByPath;
 NODE_MATCHERS[SELECTOR_TYPE.FILE] = getNodesByFile;
@@ -93,7 +95,8 @@ function getNodesByFQN(elements, qualified_name) {
           !fqn || 
           node.resource_type == 'source' || 
           node.resource_type == 'exposure' || 
-          node.resource_type == 'metric'
+          node.resource_type == 'metric' || 
+          node.resource_type == 'semantic_model'
         ) {
             return;
         }
@@ -313,6 +316,27 @@ function getNodesByMetric(elements, metric) {
     return nodes;
 }
 
+function getNodesBySemanticModel(elements, semantic_model) {
+    var nodes = [];
+    _.each(elements, function(node_obj) {
+        var node = node_obj.data;
+
+        if (node.resource_type != 'semantic_model') {
+            return;
+        }
+
+        var semantic_model_name = node.name;
+        var selected_semantic_model_name = semantic_model;
+
+        if (selected_semantic_model_name == '*') {
+            nodes.push(node_obj.data);
+        } else if (selected_semantic_model_name == semantic_model_name) {
+            nodes.push(node_obj.data);
+        }
+    })
+    return nodes;
+}
+
 function getNodesByGroup(elements, group) {
     var nodes = [];
     _.each(elements, function(node_obj) {
diff --git a/src/app/services/selector_methods.test.js b/src/app/services/selector_methods.test.js
index bdb93c834..feceb2c9a 100644
--- a/src/app/services/selector_methods.test.js
+++ b/src/app/services/selector_methods.test.js
@@ -176,6 +176,21 @@ test("Test parsing specs (explicit metric)", () => {
     });
 })
 
+test("Test parsing specs (explicit semantic model)", () => {
+    expect(
+        selectors.parseSpec('+semantic_model:a')
+    ).toStrictEqual({
+        select_at: false,
+        select_children: false,
+        select_parents: true,
+        parents_depth: null,
+        children_depth: null,
+        selector_type: 'semantic_model',
+        selector_value: 'a',
+        raw: '+semantic_model:a',
+    });
+})
+
 test("Test parsing specs (scoped fqn)", () => {
     expect(
         selectors.parseSpec('a.b.c+')