From 9286fea71a5d63b8af33336b8fb81d3604e5cc96 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Mon, 17 Feb 2020 22:41:52 -0500 Subject: [PATCH 1/2] wip --- src/app/components/code_block/code_block.html | 22 ++++++ src/app/components/code_block/code_block.js | 52 +++++++++++++ .../column_details/column_details.html | 4 +- src/app/components/index.js | 3 + src/app/components/macro_arguments/index.html | 73 ++++++++++++++++++ src/app/components/macro_arguments/index.js | 21 +++++ src/app/components/references/index.html | 23 ++++++ src/app/components/references/index.js | 55 +++++++++++++ src/app/docs/index.js | 1 + src/app/docs/macro.html | 77 +++++++++++++++++++ src/app/docs/macro.js | 60 +++++++++++++++ src/app/index.routes.js | 10 ++- src/app/services/project_service.js | 75 ++++++++++++++++-- 13 files changed, 469 insertions(+), 7 deletions(-) create mode 100644 src/app/components/code_block/code_block.html create mode 100644 src/app/components/code_block/code_block.js create mode 100644 src/app/components/macro_arguments/index.html create mode 100644 src/app/components/macro_arguments/index.js create mode 100644 src/app/components/references/index.html create mode 100644 src/app/components/references/index.js create mode 100644 src/app/docs/macro.html create mode 100644 src/app/docs/macro.js diff --git a/src/app/components/code_block/code_block.html b/src/app/components/code_block/code_block.html new file mode 100644 index 00000000..2d5bdf51 --- /dev/null +++ b/src/app/components/code_block/code_block.html @@ -0,0 +1,22 @@ +
Code
+
+
+ +
+ 

+        

+    

+

diff --git a/src/app/components/code_block/code_block.js b/src/app/components/code_block/code_block.js
new file mode 100644
index 00000000..3f3c578a
--- /dev/null
+++ b/src/app/components/code_block/code_block.js
@@ -0,0 +1,52 @@
+'use strict';
+
+const template = require('./code_block.html');
+
+angular
+.module('dbt')
+.directive('codeBlock', ['code', function(codeService) {
+    return {
+        scope: {
+            versions: '=',
+            default: '<',
+        },
+        restrict: 'E',
+        templateUrl: template,
+        link: function(scope) {
+            scope.selected_version = scope.default;
+            scope.raw_source = null;
+            scope.source = null;
+
+            function updateTo(name) {
+                scope.raw_source = scope.versions[name];
+                scope.source = codeService.highlightSql(scope.raw_source);
+            }
+
+            scope.setSelected = function(name) {
+                scope.selected_version = name;
+                updateTo(name);
+            }
+
+            scope.titleCase = function(name) {
+                return name.charAt(0).toUpperCase() + name.substring(1);
+            }
+
+            scope.copied = false;
+            scope.copy_to_clipboard = function() {
+                codeService.copy_to_clipboard(scope.raw_source)
+                scope.copied = true;
+                setTimeout(function() {
+                    scope.$apply(function() {
+                       scope.copied = false;
+                    })
+                }, 1000);
+            }
+
+            scope.$watch('versions', function(nv, ov) {
+                if (nv) {
+                    scope.setSelected(scope.default);
+                }
+            })
+        }
+    }
+}]);
diff --git a/src/app/components/column_details/column_details.html b/src/app/components/column_details/column_details.html
index 09f95af0..d737c5c0 100644
--- a/src/app/components/column_details/column_details.html
+++ b/src/app/components/column_details/column_details.html
@@ -3,7 +3,9 @@
         

             Column information is not available for this seed
         

-        

+        

             
diff --git a/src/app/components/index.js b/src/app/components/index.js
index ab3929c4..2d01c094 100644
--- a/src/app/components/index.js
+++ b/src/app/components/index.js
@@ -6,3 +6,6 @@ require('./model_tree/model_tree_line.js');
 require('./search/search.js');
 require('./table_details/table_details.js');
 require('./column_details/column_details.js');
+require('./code_block/code_block.js');
+require('./macro_arguments/');
+require('./references/');
diff --git a/src/app/components/macro_arguments/index.html b/src/app/components/macro_arguments/index.html
new file mode 100644
index 00000000..b24b385d
--- /dev/null
+++ b/src/app/components/macro_arguments/index.html
@@ -0,0 +1,73 @@
+
+
+

+    

+        

+            Details are not available for this macro
+        

+        

+            

                 
                     

+                
+                    
+                        
+                        
+                        
+                        
+                    
+                
+                
+                    
+                        
+                        
+                        
+                        
+                    
+                    
+                        
+                    
+                
+            
ArgumentTypeDescriptionMore?

+                            

+                                {{ arg.name }}
+                            

+                        		
+                            {{ arg.type }}

+                        		
+                            {{ arg.description }}
+                        
+                            
+                                
+                                    
+                                
+                                
+                                    
+                                
+                            
+                        

+                            

+                                

+                                    
Description

+                                    
+                                

+                            

+                        

+        

+    

+
diff --git a/src/app/components/macro_arguments/index.js b/src/app/components/macro_arguments/index.js
new file mode 100644
index 00000000..12ddc11c
--- /dev/null
+++ b/src/app/components/macro_arguments/index.js
@@ -0,0 +1,21 @@
+'use strict';
+
+const template = require('./index.html');
+
+angular
+.module('dbt')
+.directive('macroArguments', [function() {
+    return {
+        scope: {
+            macro: '=',
+        },
+        templateUrl: template,
+        link: function(scope) {
+
+            _.each(scope.macro.arguments, function(arg) {
+                arg.expanded = false;
+            })
+        }
+    }
+}]);
+
diff --git a/src/app/components/references/index.html b/src/app/components/references/index.html
new file mode 100644
index 00000000..3a59e88c
--- /dev/null
+++ b/src/app/components/references/index.html
@@ -0,0 +1,23 @@
+

+    

+        No resources reference this macro
+    

+    

+        
+        

+            
+        

+    

+

diff --git a/src/app/components/references/index.js b/src/app/components/references/index.js
new file mode 100644
index 00000000..277b7552
--- /dev/null
+++ b/src/app/components/references/index.js
@@ -0,0 +1,55 @@
+'use strict';
+
+const template = require('./index.html');
+
+angular
+.module('dbt')
+.directive('referenceList', ["$state", function($state) {
+    return {
+        scope: {
+            references: '=',
+        },
+        restrict: 'E',
+        templateUrl: template,
+        link: function(scope) {
+            scope.selected_type = null;
+
+            scope.setType = function(type) {
+                scope.selected_type = type;
+                scope.nodes = scope.references[scope.selected_type];
+            }
+
+            scope.getNodeUrl = function(node) {
+                var state = 'dbt.' + node.resource_type;
+                return $state.href(state, {
+                    unique_id: node.unique_id,
+                    '#': null
+                })
+            }
+
+            scope.mapResourceType = function(type) {
+                if (type == 'model') {
+                    return 'Models';
+                } else if (type == 'test') {
+                    return 'Tests';
+                } else if (type == 'snapshot') {
+                    return 'Snapshots'
+                } else if (type == 'analysis') {
+                    return 'Analyses';
+                } else {
+                    return 'Nodes';
+                }
+            }
+
+            scope.$watch('references', function(nv) {
+                if (nv && _.size(nv) > 0) {
+                    scope.selected_type = _.keys(nv)[0];
+                    scope.has_references = true;
+                    scope.nodes = scope.references[scope.selected_type];
+                } else {
+                    scope.has_references = false;
+                }
+            })
+        }
+    }
+}]);
diff --git a/src/app/docs/index.js b/src/app/docs/index.js
index 63804a4c..5d54c661 100644
--- a/src/app/docs/index.js
+++ b/src/app/docs/index.js
@@ -3,3 +3,4 @@ require('./model');
 require('./source');
 require('./seed');
 require('./snapshot');
+require('./macro');
diff --git a/src/app/docs/macro.html b/src/app/docs/macro.html
new file mode 100644
index 00000000..1ee29033
--- /dev/null
+++ b/src/app/docs/macro.html
@@ -0,0 +1,77 @@
+
+
+

+    

+        

+            

+                

+                    {{ macro.package_name }}.{{ macro.name }}
+                    adapter macro
+                    macro
+                

+            

+        

+        

+            
+        

+    

+    

+        

+            

+                

+                

+                    
Description

+                    

+                        

+                            

+                            
This {{ macro.resource_type }} is not currently documented

+                        

+                    

+                

+            

+
+            

+                

+                

+                    
Arguments

+                    
+                

+            

+
+            

+                

+                

+                    
Referenced By

+                    
+                

+            

+
+            

+                

+                

+                    
+                

+            

+        

+    

+

diff --git a/src/app/docs/macro.js b/src/app/docs/macro.js
new file mode 100644
index 00000000..22874ceb
--- /dev/null
+++ b/src/app/docs/macro.js
@@ -0,0 +1,60 @@
+'use strict';
+
+const angular = require('angular');
+const hljs = require('highlight.js/lib/highlight.js');
+const $ = require("jquery");
+
+hljs.initHighlightingOnLoad();
+hljs.initLineNumbersOnLoad();
+
+require("./styles.css");
+
+const _ = require('underscore');
+
+angular
+.module('dbt')
+.controller('MacroCtrl', ['$scope', '$state', 'project', 'code', '$transitions', '$anchorScroll', '$location',
+            function($scope, $state, projectService, codeService, $transitions, $anchorScroll, $location) {
+
+    $scope.model_uid = $state.params.unique_id;
+    $scope.tab = $state.params.tab;
+    $scope.project = projectService;
+    $scope.codeService = codeService;
+
+    function getReferences(project, macro) {
+        var references = _.filter(project.nodes, function(node) {
+            if (node.depends_on && node.depends_on.macros && node.depends_on.macros.length) {
+                if (_.contains(node.depends_on.macros, macro.unique_id)) {
+                    return true;
+                }
+            }
+            return false;
+        });
+
+        // TODO : include macros?
+        return _.groupBy(references, 'resource_type');
+    }
+
+    $scope.macro = {};
+    projectService.ready(function(project) {
+        var macro = project.macros[$scope.model_uid];
+        $scope.macro = macro;
+        $scope.references = getReferences(project, macro);
+
+        // adapter macros
+        if ($scope.macro.is_adapter_macro) {
+            var adapter = project.metadata.adapter_type;
+            $scope.versions = macro.impls;
+            if (macro.impls[adapter]) {
+                $scope.default_version = adapter;
+            } else if (macro.impls['default']) {
+                $scope.default_version = 'default'
+            } else {
+                $scope.default_version = _.keys(macro.impls)[0];
+            }
+        } else {
+            $scope.default_version = "Source"
+            $scope.versions = {Source: $scope.macro.macro_sql};
+        }
+    })
+}]);
diff --git a/src/app/index.routes.js b/src/app/index.routes.js
index aee30024..9788e7a8 100644
--- a/src/app/index.routes.js
+++ b/src/app/index.routes.js
@@ -13,6 +13,7 @@ const templates = {
     source: require('./docs/source.html'),
     snapshot: require('./docs/snapshot.html'),
     seed: require('./docs/seed.html'),
+    macro: require('./docs/macro.html'),
 }
 
 angular
@@ -79,5 +80,12 @@ angular
                 source: {type: 'string'}
             },
         })
-
+        .state('dbt.macro', {
+            url: 'macro/:unique_id?section',
+            controller: 'MacroCtrl',
+            templateUrl: templates.macro,
+            params: {
+                unique_id: {type: 'string'}
+            },
+        })
 }])
diff --git a/src/app/services/project_service.js b/src/app/services/project_service.js
index 0408c0a8..cabecf05 100644
--- a/src/app/services/project_service.js
+++ b/src/app/services/project_service.js
@@ -222,6 +222,8 @@ angular
                 return _.includes(['model', 'source', 'seed', 'snapshot'], node.resource_type);
             });
 
+            // TODO : include macros
+
             service.loaded.resolve();
         });
     }
@@ -286,7 +288,7 @@ angular
 
     service.getModelTree = function(select, cb) {
         service.loaded.promise.then(function() {
-            //var models = _.filter(service.project.nodes, {resource_type: 'model'});
+            var macros = _.values(service.project.macros);
             var nodes = _.filter(service.project.nodes, function(node) {
                 // only grab custom data tests
                 if (node.resource_type == 'test' && !_.includes(node.tags, 'schema')) {
@@ -296,8 +298,10 @@ angular
                 var accepted = ['snapshot', 'source', 'seed', 'model'];
                 return _.includes(accepted, node.resource_type);
             })
+
+            var adapter = service.project.metadata.adapter_type;
             service.tree.database = buildDatabaseTree(nodes, select);
-            service.tree.project = buildProjectTree(nodes, select);
+            service.tree.project = buildProjectTree(nodes, macros, select, adapter);
 
             var sources = _.filter(service.project.nodes, {resource_type: 'source'});
             service.tree.sources = buildSourceTree(sources, select);
@@ -388,10 +392,65 @@ angular
         return sources
     }
 
-    function buildProjectTree(nodes, select) {
+    function consolidateAdapterMacros(macros, adapter) {
+        var adapter_macros = {};
+        _.each(macros, function(macro) {
+            if (macro.macro_sql.match(/{{\s*adapter_macro\([^)]+\)\s+}}/)) {
+                macro.impls = {"Adapter Macro": macro.macro_sql};
+                macro.is_adapter_macro = true;
+                adapter_macros[macro.name] = macro;
+            }
+        });
+
+        // ideally we would not need to do this!
+        var databases = [
+            'postgres',
+            'redshift',
+            'bigquery',
+            'snowflake',
+            'spark',
+            'presto',
+            'default',
+        ];
+
+        var to_return = _.values(adapter_macros);
+        var extras = _.filter(macros, function(macro) {
+            var parts = macro.name.split("__");
+            var head = parts.shift();
+
+            var macro_name = parts.join("__");
+            if (databases.indexOf(head) >= 0 && adapter_macros[macro_name]) {
+                adapter_macros[macro_name].impls[head] = macro.macro_sql;
+                return false;
+            }
+            return true;
+        });
+
+        return to_return.concat(extras);
+    }
+
+    function buildProjectTree(nodes, macros, select, adapter) {
         var tree = {};
 
-        _.each(nodes, function(node) {
+        var nodes = nodes || [];
+        var all_macros = macros || [];
+
+        var package_macros = {};
+        _.each(all_macros, function(macro) {
+            if (!package_macros[macro.package_name]) {
+                package_macros[macro.package_name] = {}
+            }
+
+            package_macros[macro.package_name][macro.name] = macro
+        });
+
+        var macros = [];
+        _.each(package_macros, function(package_macros, package_name) {
+            var pkg_macros = consolidateAdapterMacros(package_macros, adapter);
+            macros = macros.concat(pkg_macros);
+        });
+
+        _.each(nodes.concat(macros), function(node) {
             if (node.resource_type == 'source') {
                 // no sources in the model tree, sorry
                 return;
@@ -407,7 +466,12 @@ angular
             var is_active = node.unique_id == select;
 
             var dirpath = _.initial(path);
-            var fname = _.last(path);
+
+            if (node.resource_type == 'macro') {
+                var fname = node.name;
+            } else {
+                var fname = _.last(path);
+            }
 
             var cur_dir = tree;
             _.each(dirpath, function(dir) {
@@ -497,6 +561,7 @@ angular
     service.init = function() {
         service.loadProject()
 
+        // TODO : Do these need to be here ...?
         var models = _.filter(service.project.nodes, {resource_type: 'model'});
         service.tree.database = buildDatabaseTree(models);
         service.tree.project = buildProjectTree(models);

From b267f20a7853cd2ffbe3a0077844347caa059de6 Mon Sep 17 00:00:00 2001
From: Drew Banin 
Date: Sun, 1 Mar 2020 20:40:26 -0500
Subject: [PATCH 2/2] macro search, fix console errors

---
 src/app/components/references/index.html |  2 +-
 src/app/components/search/search.js      |  2 +
 src/app/graph/index.js                   |  4 +-
 src/app/services/project_service.js      | 81 ++++++++++++++----------
 4 files changed, 53 insertions(+), 36 deletions(-)

diff --git a/src/app/components/references/index.html b/src/app/components/references/index.html
index 3a59e88c..f650a7ca 100644
--- a/src/app/components/references/index.html
+++ b/src/app/components/references/index.html
@@ -13,7 +13,7 @@
             
         
         

-            
    
+            
      
                 
    • 
                     {{ node.name }}
                 
      • 
diff --git a/src/app/components/search/search.js b/src/app/components/search/search.js
index 64c95fd0..f187feed 100644
--- a/src/app/components/search/search.js
+++ b/src/app/components/search/search.js
@@ -31,6 +31,8 @@ angular
             scope.getModelName = function(model) {
                 if (model.resource_type == 'source') {
                     return model.source_name + "." + model.name;
+                } else if (model.resource_type == 'macro') {
+                    return model.package_name + "." + model.name;
                 } else {
                     return model.name;
                 }
diff --git a/src/app/graph/index.js b/src/app/graph/index.js
index c02534fc..116c5d39 100644
--- a/src/app/graph/index.js
+++ b/src/app/graph/index.js
@@ -32,7 +32,9 @@ angular
         }, function(nv, ov) {
             if (nv && nv != ov) {
                 projectService.find_by_id(nv, function(node) {
-                    if (graph.orientation == 'sidebar') {
+                    if (!node) {
+                        // pass - it's a macro or some other non-project resource
+                    } else if (graph.orientation == 'sidebar') {
                         graph.showVerticalGraph(getNodeSelector(node), false);
                     } else {
                         graph.showFullGraph(getNodeSelector(node));
diff --git a/src/app/services/project_service.js b/src/app/services/project_service.js
index cabecf05..135f1ff8 100644
--- a/src/app/services/project_service.js
+++ b/src/app/services/project_service.js
@@ -148,6 +148,10 @@ angular
                 }
             });
 
+            var adapter = service.files.manifest.metadata.adapter_type;
+            var macros = clean_project_macros(service.files.manifest.macros, adapter);
+            service.files.manifest.macros = macros;
+
             var project = incorporate_catalog(service.files.manifest, service.files.catalog);
             var compiled_project = incorporate_run_results(project, service.files.run_results);
 
@@ -218,12 +222,15 @@ angular
             service.project = compiled_project;
 
             // performance hack
-            service.project.searchable = _.filter(service.project.nodes, function(node) {
-                return _.includes(['model', 'source', 'seed', 'snapshot'], node.resource_type);
+            var search_macros = _.filter(service.project.macros, function(macro) {
+                return !macro.is_adapter_macro_impl;
             });
 
-            // TODO : include macros
+            var search_nodes = _.filter(service.project.nodes, function(node) {
+                return _.includes(['model', 'source', 'seed', 'snapshot'], node.resource_type);
+            });
 
+            service.project.searchable = search_nodes.concat(search_macros);
             service.loaded.resolve();
         });
     }
@@ -236,10 +243,15 @@ angular
 
     function fuzzySearchObj(val, obj) {
         var objects = [];
-        var search_keys = {'name':'string', 'description':'string', 'columns':'object', 'tags': 'array'};
-        
+        var search_keys = {
+            'name':'string',
+            'description':'string',
+            'columns':'object',
+            'tags': 'array',
+            'arguments': 'array',
+        };
         var search = new RegExp(val, "i")
-        
+
         for (var i in search_keys) {
             if (!obj[i]) {
                continue;
@@ -253,7 +265,7 @@ angular
                 }
             } else if (search_keys[i] === 'array') {
                 for (var tag of obj[i]) {
-                    if (tag.toLowerCase().indexOf(val.toLowerCase()) != -1) {
+                    if (JSON.stringify(tag).toLowerCase().indexOf(val.toLowerCase()) != -1) {
                         objects.push({key: i, value: val});
                     }
                 }
@@ -286,6 +298,30 @@ angular
         return res;
     }
 
+    function clean_project_macros(macros, adapter) {
+        var all_macros = macros || [];
+
+        var package_macros = {};
+        _.each(all_macros, function(macro) {
+            if (!package_macros[macro.package_name]) {
+                package_macros[macro.package_name] = {}
+            }
+
+            package_macros[macro.package_name][macro.name] = macro
+        });
+
+        var macros = [];
+        _.each(package_macros, function(package_macros, package_name) {
+            if (package_name == 'dbt' || package_name == 'dbt_' + adapter) {
+                return
+            }
+            var pkg_macros = consolidateAdapterMacros(package_macros, adapter);
+            macros = macros.concat(pkg_macros);
+        });
+
+        return _.indexBy(macros, 'unique_id');
+    }
+
     service.getModelTree = function(select, cb) {
         service.loaded.promise.then(function() {
             var macros = _.values(service.project.macros);
@@ -299,9 +335,8 @@ angular
                 return _.includes(accepted, node.resource_type);
             })
 
-            var adapter = service.project.metadata.adapter_type;
             service.tree.database = buildDatabaseTree(nodes, select);
-            service.tree.project = buildProjectTree(nodes, macros, select, adapter);
+            service.tree.project = buildProjectTree(nodes, macros, select);
 
             var sources = _.filter(service.project.nodes, {resource_type: 'source'});
             service.tree.sources = buildSourceTree(sources, select);
@@ -421,6 +456,7 @@ angular
             var macro_name = parts.join("__");
             if (databases.indexOf(head) >= 0 && adapter_macros[macro_name]) {
                 adapter_macros[macro_name].impls[head] = macro.macro_sql;
+                macro.is_adapter_macro_impl = true;
                 return false;
             }
             return true;
@@ -429,26 +465,11 @@ angular
         return to_return.concat(extras);
     }
 
-    function buildProjectTree(nodes, macros, select, adapter) {
+    function buildProjectTree(nodes, macros, select) {
         var tree = {};
 
         var nodes = nodes || [];
-        var all_macros = macros || [];
-
-        var package_macros = {};
-        _.each(all_macros, function(macro) {
-            if (!package_macros[macro.package_name]) {
-                package_macros[macro.package_name] = {}
-            }
-
-            package_macros[macro.package_name][macro.name] = macro
-        });
-
-        var macros = [];
-        _.each(package_macros, function(package_macros, package_name) {
-            var pkg_macros = consolidateAdapterMacros(package_macros, adapter);
-            macros = macros.concat(pkg_macros);
-        });
+        var macros = macros || [];
 
         _.each(nodes.concat(macros), function(node) {
             if (node.resource_type == 'source') {
@@ -560,14 +581,6 @@ angular
 
     service.init = function() {
         service.loadProject()
-
-        // TODO : Do these need to be here ...?
-        var models = _.filter(service.project.nodes, {resource_type: 'model'});
-        service.tree.database = buildDatabaseTree(models);
-        service.tree.project = buildProjectTree(models);
-
-        var sources = _.filter(service.project.nodes, {resource_type: 'source'});
-        service.tree.sources = buildSourceTree(sources);
     }
 
     return service;