DBT-747: Removing the describe table statement each tables rather use the show queries to gather metadata

dbt/adapters/impala/impl.py

@@ -98,10 +98,17 @@ def list_schemas(self, database: str) -> List[str]:
     def list_relations_without_caching(
         self, schema_relation: ImpalaRelation
     ) -> List[ImpalaRelation]:
-        kwargs = {"schema_relation": schema_relation}
-
+        """
+        Get a list of Relation(table or view) by SQL directly
+        Use different SQL statement for view/table
+        Need this fix: https://issues.apache.org/jira/browse/IMPALA-3268
+        """
+        kwargs = {"schema": schema_relation}
         try:
-            results = self.execute_macro(LIST_RELATIONS_MACRO_NAME, kwargs=kwargs)
+            result_tables = self.execute_macro(
+                "impala__list_tables_without_caching", kwargs=kwargs
+            )
+            result_views = self.execute_macro("impala__list_views_without_caching", kwargs=kwargs)
         except dbt.exceptions.DbtRuntimeError as e:
             errmsg = getattr(e, "msg", "")
             if f"Database '{schema_relation}' not found" in errmsg:
@@ -111,24 +118,35 @@ def list_relations_without_caching(
                 logger.debug(f"{description} {schema_relation}: {e.msg}")
                 return []
 
+        # Impala
+        # Collect table/view separately
+        # Unfortunatly, Impala does not distincguish table/view
+        # Currently views are also listed in `show tables`
+
+        result_tables_without_view = []
+        for row in result_tables:
+            # check if this table is view
+            is_view = len(list(filter(lambda x: x["name"] == row["name"], result_views))) == 1
+            if not is_view:
+                result_tables_without_view.append(row)
+
         relations = []
-        for row in results:
-            if len(row) != 2:
-                raise dbt.exceptions.DbtRuntimeError(
-                    f'Invalid value from "show table extended ...", '
-                    f"got {len(row)} values, expected 4"
+        for row in result_tables_without_view:
+            relations.append(
+                self.Relation.create(
+                    schema=schema_relation.schema,
+                    identifier=row["name"],
+                    type="table",
+                )
+            )
+        for row in result_views:
+            relations.append(
+                self.Relation.create(
+                    schema=schema_relation.schema,
+                    identifier=row["name"],
+                    type="view",
                 )
-            _identifier = row[0]
-            _rel_type = row[1]
-
-            relation = self.Relation.create(
-                database=None,
-                schema=schema_relation.schema,
-                identifier=_identifier,
-                type=_rel_type,
-                information=_identifier,
             )
-            relations.append(relation)
 
         return relations
 

dbt/include/impala/macros/adapters.sql

@@ -125,24 +125,26 @@
     {{ return(load_result('list_schemas').table) }}
 {% endmacro %}
 
-{# Note: This function currently needs to query each object to determine its type. Depending on the schema, this function could be expensive. #}
-{% macro impala__list_relations_without_caching(relation) %}
-  {% set result_set = run_query('show tables in ' ~ relation) %}
-  {% set objects_with_type = [] %}
-
-  {%- for rs in result_set -%}
-    {% set obj_type = [] %}
-
-    {% do obj_type.append(rs[0]) %}
+{% macro impala__list_tables_without_caching(schema) %}
+  {% call statement('list_tables_without_caching', fetch_result=True) -%}
+    show tables in {{ schema }}
+  {% endcall %}
+  {% do return(load_result('list_tables_without_caching').table) %}
+{% endmacro %}
 
-    {% set obj_rel = relation.new_copy(relation.schema, rs[0]) %}
-    {% set rel_type = get_relation_type(obj_rel) %}
-    {% do obj_type.append(rel_type) %}
+{% macro impala__list_views_without_caching(schema) %}
+  {% call statement('list_views_without_caching', fetch_result=True) -%}
+    show views  in {{ schema }}
+  {% endcall %}
+  {% do return(load_result('list_views_without_caching').table) %}
+{% endmacro %}
 
-    {% do objects_with_type.append(obj_type) %}
-  {%- endfor -%}
+{% macro impala__list_relations_without_caching(relation) %}
+  {% call statement('list_relations_without_caching', fetch_result=True) -%}
+    show tables in {{relation}}
+  {% endcall %}
 
-  {{ return(objects_with_type) }}
+  {% do return(load_result('list_relations_without_caching').table) %}
 {% endmacro %}
 
 {% macro impala__create_table_as(temporary, relation, sql) -%}