WARP-25626, cherry-pick python codes of Superset commit '[wip] dashbo…

…ard builder v2 (apache#4528) c065319'

superset/config.py

@@ -373,6 +373,13 @@ class CeleryConfig(object):
 # using flask-compress
 ENABLE_FLASK_COMPRESS = True
 
+# Dashboard v1 deprecation configuration
+DASH_V2_IS_DEFAULT_VIEW_FOR_EDITORS = True
+CAN_FALLBACK_TO_DASH_V1_EDIT_MODE = True
+# these are incorporated into messages displayed to users
+PLANNED_V2_AUTO_CONVERT_DATE = None  # e.g. '2018-06-16'
+V2_FEEDBACK_URL = None  # e.g., 'https://goo.gl/forms/...'
+
 
 try:
     if CONFIG_PATH_ENV_VAR in os.environ:

superset/connectors/sqla/models.py

@@ -306,8 +306,8 @@ def description_markeddown(self):
     @property
     def link(self):
         name = escape(self.name)
-        return Markup(
-            '<a href="{self.explore_url}">{name}</a>'.format(**locals()))
+        anchor = '<a target="_blank" href="{self.explore_url}">{name}</a>'
+        return Markup(anchor.format(**locals()))
 
     @property
     def schema_perm(self):

superset/models/core.py

@@ -146,6 +146,11 @@ def datasource_link(self):
         datasource = self.datasource
         return datasource.link if datasource else None
 
+    def datasource_name_text(self):
+        # pylint: disable=no-member
+        datasource = self.datasource
+        return datasource.name if datasource else None
+
     @property
     def datasource_edit_url(self):
         # pylint: disable=no-member
@@ -184,6 +189,8 @@ def data(self):
             'slice_id': self.id,
             'slice_name': self.slice_name,
             'slice_url': self.slice_url,
+            'modified': self.modified(),
+            'changed_on': self.changed_on.isoformat(),
         }
 
     @property
@@ -342,10 +349,15 @@ def url(self):
             # add default_filters to the preselect_filters of dashboard
             json_metadata = json.loads(self.json_metadata)
             default_filters = json_metadata.get('default_filters')
-            if default_filters:
-                filters = parse.quote(default_filters.encode('utf8'))
-                return '/superset/dashboard/{}/?preselect_filters={}'.format(
-                    self.slug or self.id, filters)
+            # make sure default_filters is not empty and is valid
+            if default_filters and default_filters != '{}':
+                try:
+                    if json.loads(default_filters):
+                        filters = parse.quote(default_filters.encode('utf8'))
+                        return '/superset/dashboard/{}/?preselect_filters={}'.format(
+                            self.slug or self.id, filters)
+                except Exception:
+                    pass
         return '/superset/dashboard/{}/'.format(self.slug or self.id)
 
     @property
@@ -406,24 +418,67 @@ def import_obj(cls, dashboard_to_import, import_time=None):
         def alter_positions(dashboard, old_to_new_slc_id_dict):
             """ Updates slice_ids in the position json.
 
-            Sample position json:
+            Sample position json v1:
             [{
                 "col": 5,
                 "row": 10,
                 "size_x": 4,
                 "size_y": 2,
                 "slice_id": "3610"
             }]
+
+            Sample position json v2:
+            {
+                "DASHBOARD_VERSION_KEY": "v2",
+                "DASHBOARD_ROOT_ID": {
+                    "type": "DASHBOARD_ROOT_TYPE",
+                    "id": "DASHBOARD_ROOT_ID",
+                    "children": ["DASHBOARD_GRID_ID"]
+                },
+                "DASHBOARD_GRID_ID": {
+                    "type": "DASHBOARD_GRID_TYPE",
+                    "id": "DASHBOARD_GRID_ID",
+                    "children": ["DASHBOARD_CHART_TYPE-2"]
+                },
+                "DASHBOARD_CHART_TYPE-2": {
+                    "type": "DASHBOARD_CHART_TYPE",
+                    "id": "DASHBOARD_CHART_TYPE-2",
+                    "children": [],
+                    "meta": {
+                        "width": 4,
+                        "height": 50,
+                        "chartId": 118
+                    }
+                },
+            }
             """
-            position_array = dashboard.position_array
-            for position in position_array:
-                if 'slice_id' not in position:
-                    continue
-                old_slice_id = int(position['slice_id'])
-                if old_slice_id in old_to_new_slc_id_dict:
-                    position['slice_id'] = '{}'.format(
-                        old_to_new_slc_id_dict[old_slice_id])
-            dashboard.position_json = json.dumps(position_array)
+            position_data = json.loads(dashboard.position_json)
+            is_v2_dash = (
+                isinstance(position_data, dict) and
+                position_data.get('DASHBOARD_VERSION_KEY') == 'v2'
+            )
+            if is_v2_dash:
+                position_json = position_data.values()
+                for value in position_json:
+                    if (isinstance(value, dict) and value.get('meta') and
+                            value.get('meta').get('chartId')):
+                        old_slice_id = value.get('meta').get('chartId')
+
+                        if old_slice_id in old_to_new_slc_id_dict:
+                            value['meta']['chartId'] = (
+                                old_to_new_slc_id_dict[old_slice_id]
+                            )
+                dashboard.position_json = json.dumps(position_data)
+            else:
+                position_array = dashboard.position_array
+                for position in position_array:
+                    if 'slice_id' not in position:
+                        continue
+                    old_slice_id = int(position['slice_id'])
+                    if old_slice_id in old_to_new_slc_id_dict:
+                        position['slice_id'] = '{}'.format(
+                            old_to_new_slc_id_dict[old_slice_id])
+                dashboard.position_json = json.dumps(position_array)
 
         logging.info('Started import of the dashboard: {}'
                      .format(dashboard_to_import.to_json()))
@@ -877,43 +932,59 @@ def log_this(cls, f):
         """Decorator to log user actions"""
         @functools.wraps(f)
         def wrapper(*args, **kwargs):
-            start_dttm = datetime.now()
             user_id = None
             if g.user:
                 user_id = g.user.get_id()
             d = request.form.to_dict() or {}
+
             # request parameters can overwrite post body
             request_params = request.args.to_dict()
             d.update(request_params)
             d.update(kwargs)
+
             slice_id = d.get('slice_id')
+            dashboard_id = d.get('dashboard_id')
 
             try:
                 slice_id = int(
                     slice_id or json.loads(d.get('form_data')).get('slice_id'))
             except (ValueError, TypeError):
                 slice_id = 0
 
-            params = ''
-            try:
-                params = json.dumps(d)
-            except Exception:
-                pass
             stats_logger.incr(f.__name__)
+            start_dttm = datetime.now()
             value = f(*args, **kwargs)
+            duration_ms = (datetime.now() - start_dttm).total_seconds() * 1000
+
+            # bulk insert
+            try:
+                explode_by = d.get('explode')
+                records = json.loads(d.get(explode_by))
+            except Exception:
+                records = [d]
+
+            referrer = request.referrer[:1000] if request.referrer else None
+            logs = []
+            for record in records:
+                try:
+                    json_string = json.dumps(record)
+                except Exception:
+                    json_string = None
+                log = cls(
+                    action=f.__name__,
+                    json=json_string,
+                    dashboard_id=dashboard_id,
+                    slice_id=slice_id,
+                    duration_ms=duration_ms,
+                    referrer=referrer,
+                    user_id=user_id)
+                logs.append(log)
+
             sesh = db.session()
-            log = cls(
-                action=f.__name__,
-                json=params,
-                dashboard_id=d.get('dashboard_id'),
-                slice_id=slice_id,
-                duration_ms=(
-                                datetime.now() - start_dttm).total_seconds() * 1000,
-                referrer=request.referrer[:1000] if request.referrer else None,
-                user_id=user_id)
-            sesh.add(log)
+            sesh.bulk_save_objects(logs)
             sesh.commit()
             return value
+
         return wrapper
 
 
@@ -993,4 +1064,4 @@ def user_roles(self):
             if r.name in self.ROLES_BLACKLIST:
                 href = '{} Role'.format(r.name)
             action_list = action_list + '<li>' + href + '</li>'
-        return '<ul>' + action_list + '</ul>'
+        return '<ul>' + action_list + '</ul>'

superset/models/dashboard.py

@@ -117,10 +117,15 @@ def url(self):
             # add default_filters to the preselect_filters of dashboard
             json_metadata = json.loads(self.json_metadata)
             default_filters = json_metadata.get('default_filters')
-            if default_filters:
-                filters = parse.quote(default_filters.encode('utf8'))
-                return '/superset/dashboard/{}/?preselect_filters={}'.format(
-                    self.slug or self.id, filters)
+            # make sure default_filters is not empty and is valid
+            if default_filters and default_filters != '{}':
+                try:
+                    if json.loads(default_filters):
+                        filters = parse.quote(default_filters.encode('utf8'))
+                        return '/superset/dashboard/{}/?preselect_filters={}'.format(
+                            self.slug or self.id, filters)
+                except Exception:
+                    pass
         return '/superset/dashboard/{}/'.format(self.id)
 
     @property
@@ -174,23 +179,68 @@ def import_obj(cls, session, i_dash, solution, grant_owner_perms,
         """
         def alter_positions(dashboard, old_to_new_slc_id_dict):
             """ Updates slice_ids in the position json.
-            Sample position json:
+
+            Sample position json v1:
             [{
                 "col": 5,
                 "row": 10,
                 "size_x": 4,
                 "size_y": 2,
                 "slice_id": "3610"
             }]
+
+            Sample position json v2:
+            {
+                "DASHBOARD_VERSION_KEY": "v2",
+                "DASHBOARD_ROOT_ID": {
+                    "type": "DASHBOARD_ROOT_TYPE",
+                    "id": "DASHBOARD_ROOT_ID",
+                    "children": ["DASHBOARD_GRID_ID"]
+                },
+                "DASHBOARD_GRID_ID": {
+                    "type": "DASHBOARD_GRID_TYPE",
+                    "id": "DASHBOARD_GRID_ID",
+                    "children": ["DASHBOARD_CHART_TYPE-2"]
+                },
+                "DASHBOARD_CHART_TYPE-2": {
+                    "type": "DASHBOARD_CHART_TYPE",
+                    "id": "DASHBOARD_CHART_TYPE-2",
+                    "children": [],
+                    "meta": {
+                        "width": 4,
+                        "height": 50,
+                        "chartId": 118
+                    }
+                },
+            }
             """
-            position_array = dashboard.position_array
-            for position in position_array:
-                if 'slice_id' in position:
+            position_data = json.loads(dashboard.position_json)
+            is_v2_dash = (
+                isinstance(position_data, dict) and
+                position_data.get('DASHBOARD_VERSION_KEY') == 'v2'
+            )
+            if is_v2_dash:
+                position_json = position_data.values()
+                for value in position_json:
+                    if (isinstance(value, dict) and value.get('meta') and
+                            value.get('meta').get('chartId')):
+                        old_slice_id = value.get('meta').get('chartId')
+
+                        if old_slice_id in old_to_new_slc_id_dict:
+                            value['meta']['chartId'] = (
+                                old_to_new_slc_id_dict[old_slice_id]
+                            )
+                dashboard.position_json = json.dumps(position_data)
+            else:
+                position_array = dashboard.position_array
+                for position in position_array:
+                    if 'slice_id' not in position:
+                        continue
                     old_slice_id = int(position['slice_id'])
                     if old_slice_id in old_to_new_slc_id_dict:
-                        position['slice_id'] = \
-                            '{}'.format(old_to_new_slc_id_dict[old_slice_id])
-            dashboard.position_json = json.dumps(position_array)
+                        position['slice_id'] = '{}'.format(
+                            old_to_new_slc_id_dict[old_slice_id])
+                dashboard.position_json = json.dumps(position_array)
 
         slices = copy(i_dash.slices)
         old_to_new_slc_id_dict = {}

superset/models/helpers.py

@@ -280,7 +280,7 @@ def changed_on_(self):
         return Markup(
             '<span class="no-wrap">{}</span>'.format(self.changed_on))
 
-    @renders('changed_on')
+    @renders('modified')
     def modified(self):
         s = humanize.naturaltime(datetime.now() - self.changed_on)
         return Markup('<span class="no-wrap">{}</span>'.format(s))

superset/models/slice.py

@@ -112,6 +112,11 @@ def datasource_link(self):
         datasource = self.datasource
         return datasource.link if datasource else None
 
+    def datasource_name_text(self):
+        # pylint: disable=no-member
+        datasource = self.datasource
+        return datasource.name if datasource else None
+
     @property
     def datasource_edit_url(self):
         # pylint: disable=no-member
@@ -141,15 +146,18 @@ def data(self):
         except Exception as e:
             logging.exception(e)
             d['error'] = str(e)
-        d['slice_id'] = self.id
-        d['slice_name'] = self.slice_name
-        d['slice_url'] = self.slice_url
-        d['edit_url'] = self.edit_url
-        d['datasource'] = self.datasource_name
-        d['form_data'] = self.form_data
-        d['description'] = self.description
-        d['description_markeddown'] = self.description_markeddown
-        return d
+        return {
+            'datasource': self.datasource_name,
+            'description': self.description,
+            'description_markeddown': self.description_markeddown,
+            'edit_url': self.edit_url,
+            'form_data': self.form_data,
+            'slice_id': self.id,
+            'slice_name': self.slice_name,
+            'slice_url': self.slice_url,
+            'modified': self.modified(),
+            'changed_on': self.changed_on.isoformat(),
+        }
 
     @property
     def json_data(self):