From ca9ca995109ee5251918b9df60980bffaf9b9a24 Mon Sep 17 00:00:00 2001 From: Grace Guo Date: Sun, 16 Aug 2020 14:43:30 -0700 Subject: [PATCH] refactor: [migration] convert iframe chart into dashboard markdown component (#10590) * refactor: [migration] convert iframe chart into dashboard markdown component * remove 3 viz_types * fix comments --- UPDATING.md | 2 + .../src/dashboard/reducers/getInitialState.js | 202 +++++++++--------- .../src/explore/components/SaveModal.jsx | 8 +- .../components/controls/VizTypeControl.jsx | 3 - superset-frontend/src/explore/constants.js | 2 - .../src/visualizations/presets/MainPreset.js | 5 - ...5563a02_migrate_iframe_to_dash_markdown.py | 198 +++++++++++++++++ superset/migrations/versions/f80a3b88324b_.py | 38 ++++ superset/tasks/slack_util.py | 2 +- superset/viz.py | 19 -- 10 files changed, 340 insertions(+), 139 deletions(-) create mode 100644 superset/migrations/versions/978245563a02_migrate_iframe_to_dash_markdown.py create mode 100644 superset/migrations/versions/f80a3b88324b_.py diff --git a/UPDATING.md b/UPDATING.md index 82c36f5f236e7..054afafe2b890 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -23,6 +23,8 @@ assists people when migrating to a new version. ## Next +* [10590](https://github.com/apache/incubator-superset/pull/10590): Breaking change: this PR will convert iframe chart into dashboard markdown component, and remove all `iframe`, `separator`, and `markup` slices (and support) from Superset. If you have important data in those slices, please backup manually. + * [10562](https://github.com/apache/incubator-superset/pull/10562): EMAIL_REPORTS_WEBDRIVER is deprecated use WEBDRIVER_TYPE instead. * [10567](https://github.com/apache/incubator-superset/pull/10567): Default WEBDRIVER_OPTION_ARGS are Chrome-specific. If you're using FF, should be `--headless` only diff --git a/superset-frontend/src/dashboard/reducers/getInitialState.js b/superset-frontend/src/dashboard/reducers/getInitialState.js index 770e24963dc95..2e419dfd8c241 100644 --- a/superset-frontend/src/dashboard/reducers/getInitialState.js +++ b/superset-frontend/src/dashboard/reducers/getInitialState.js @@ -108,118 +108,116 @@ export default function (bootstrapData) { const sliceIds = new Set(); dashboard.slices.forEach(slice => { const key = slice.slice_id; - if (['separator', 'markup'].indexOf(slice.form_data.viz_type) === -1) { - const form_data = { - ...slice.form_data, - url_params: { - ...slice.form_data.url_params, - ...urlParams, - }, - }; - chartQueries[key] = { - ...chart, - id: key, - form_data, - formData: applyDefaultFormData(form_data), - }; - - slices[key] = { - slice_id: key, - slice_url: slice.slice_url, - slice_name: slice.slice_name, - form_data: slice.form_data, - edit_url: slice.edit_url, - viz_type: slice.form_data.viz_type, - datasource: slice.form_data.datasource, - description: slice.description, - description_markeddown: slice.description_markeddown, - owners: slice.owners, - modified: slice.modified, - changed_on: new Date(slice.changed_on).getTime(), - }; + const form_data = { + ...slice.form_data, + url_params: { + ...slice.form_data.url_params, + ...urlParams, + }, + }; + chartQueries[key] = { + ...chart, + id: key, + form_data, + formData: applyDefaultFormData(form_data), + }; - sliceIds.add(key); + slices[key] = { + slice_id: key, + slice_url: slice.slice_url, + slice_name: slice.slice_name, + form_data: slice.form_data, + edit_url: slice.edit_url, + viz_type: slice.form_data.viz_type, + datasource: slice.form_data.datasource, + description: slice.description, + description_markeddown: slice.description_markeddown, + owners: slice.owners, + modified: slice.modified, + changed_on: new Date(slice.changed_on).getTime(), + }; - // if there are newly added slices from explore view, fill slices into 1 or more rows - if (!chartIdToLayoutId[key] && layout[parentId]) { - if ( - newSlicesContainerWidth === 0 || - newSlicesContainerWidth + GRID_DEFAULT_CHART_WIDTH > GRID_COLUMN_COUNT - ) { - newSlicesContainer = newComponentFactory( - ROW_TYPE, - (parent.parents || []).slice(), - ); - layout[newSlicesContainer.id] = newSlicesContainer; - parent.children.push(newSlicesContainer.id); - newSlicesContainerWidth = 0; - } + sliceIds.add(key); - const chartHolder = newComponentFactory( - CHART_TYPE, - { - chartId: slice.slice_id, - }, - (newSlicesContainer.parents || []).slice(), + // if there are newly added slices from explore view, fill slices into 1 or more rows + if (!chartIdToLayoutId[key] && layout[parentId]) { + if ( + newSlicesContainerWidth === 0 || + newSlicesContainerWidth + GRID_DEFAULT_CHART_WIDTH > GRID_COLUMN_COUNT + ) { + newSlicesContainer = newComponentFactory( + ROW_TYPE, + (parent.parents || []).slice(), ); - - layout[chartHolder.id] = chartHolder; - newSlicesContainer.children.push(chartHolder.id); - chartIdToLayoutId[chartHolder.meta.chartId] = chartHolder.id; - newSlicesContainerWidth += GRID_DEFAULT_CHART_WIDTH; + layout[newSlicesContainer.id] = newSlicesContainer; + parent.children.push(newSlicesContainer.id); + newSlicesContainerWidth = 0; } - // build DashboardFilters for interactive filter features - if (slice.form_data.viz_type === 'filter_box') { - const configs = getFilterConfigsFromFormdata(slice.form_data); - let columns = configs.columns; - const labels = configs.labels; - if (preselectFilters[key]) { - Object.keys(columns).forEach(col => { - if (preselectFilters[key][col]) { - columns = { - ...columns, - [col]: preselectFilters[key][col], - }; - } - }); - } + const chartHolder = newComponentFactory( + CHART_TYPE, + { + chartId: slice.slice_id, + }, + (newSlicesContainer.parents || []).slice(), + ); - const scopesByChartId = Object.keys(columns).reduce((map, column) => { - const scopeSettings = { - ...filterScopes[key], - }; - const { scope, immune } = { - ...DASHBOARD_FILTER_SCOPE_GLOBAL, - ...scopeSettings[column], - }; + layout[chartHolder.id] = chartHolder; + newSlicesContainer.children.push(chartHolder.id); + chartIdToLayoutId[chartHolder.meta.chartId] = chartHolder.id; + newSlicesContainerWidth += GRID_DEFAULT_CHART_WIDTH; + } - return { - ...map, - [column]: { - scope, - immune, - }, - }; - }, {}); + // build DashboardFilters for interactive filter features + if (slice.form_data.viz_type === 'filter_box') { + const configs = getFilterConfigsFromFormdata(slice.form_data); + let columns = configs.columns; + const labels = configs.labels; + if (preselectFilters[key]) { + Object.keys(columns).forEach(col => { + if (preselectFilters[key][col]) { + columns = { + ...columns, + [col]: preselectFilters[key][col], + }; + } + }); + } - const componentId = chartIdToLayoutId[key]; - const directPathToFilter = (layout[componentId].parents || []).slice(); - directPathToFilter.push(componentId); - dashboardFilters[key] = { - ...dashboardFilter, - chartId: key, - componentId, - datasourceId: slice.form_data.datasource, - filterName: slice.slice_name, - directPathToFilter, - columns, - labels, - scopes: scopesByChartId, - isInstantFilter: !!slice.form_data.instant_filtering, - isDateFilter: Object.keys(columns).includes(TIME_RANGE), + const scopesByChartId = Object.keys(columns).reduce((map, column) => { + const scopeSettings = { + ...filterScopes[key], }; - } + const { scope, immune } = { + ...DASHBOARD_FILTER_SCOPE_GLOBAL, + ...scopeSettings[column], + }; + + return { + ...map, + [column]: { + scope, + immune, + }, + }; + }, {}); + + const componentId = chartIdToLayoutId[key]; + const directPathToFilter = (layout[componentId].parents || []).slice(); + directPathToFilter.push(componentId); + dashboardFilters[key] = { + ...dashboardFilter, + chartId: key, + componentId, + datasourceId: slice.form_data.datasource, + filterName: slice.slice_name, + directPathToFilter, + columns, + labels, + scopes: scopesByChartId, + isInstantFilter: !!slice.form_data.instant_filtering, + isDateFilter: Object.keys(columns).includes(TIME_RANGE), + }; } // sync layout names with current slice names in case a slice was edited diff --git a/superset-frontend/src/explore/components/SaveModal.jsx b/superset-frontend/src/explore/components/SaveModal.jsx index 45b96ffd5160d..804ebc2bc7522 100644 --- a/superset-frontend/src/explore/components/SaveModal.jsx +++ b/superset-frontend/src/explore/components/SaveModal.jsx @@ -33,8 +33,6 @@ import { CreatableSelect } from 'src/components/Select/SupersetStyledSelect'; import { t } from '@superset-ui/translation'; import ReactMarkdown from 'react-markdown'; -import { EXPLORE_ONLY_VIZ_TYPE } from '../constants'; - const propTypes = { can_overwrite: PropTypes.bool, onHide: PropTypes.func.isRequired, @@ -134,8 +132,6 @@ class SaveModal extends React.Component { this.setState({ alert: null }); } render() { - const canNotSaveToDash = - EXPLORE_ONLY_VIZ_TYPE.indexOf(this.state.vizType) > -1; return ( @@ -225,9 +221,7 @@ class SaveModal extends React.Component { id="btn_modal_save_goto_dash" bsSize="sm" disabled={ - canNotSaveToDash || - !this.state.newSliceName || - !this.state.newDashboardName + !this.state.newSliceName || !this.state.newDashboardName } onClick={this.saveOrOverwrite.bind(this, true)} > diff --git a/superset-frontend/src/explore/components/controls/VizTypeControl.jsx b/superset-frontend/src/explore/components/controls/VizTypeControl.jsx index 2c67f6837aab1..cf5ad648f3ed5 100644 --- a/superset-frontend/src/explore/components/controls/VizTypeControl.jsx +++ b/superset-frontend/src/explore/components/controls/VizTypeControl.jsx @@ -74,7 +74,6 @@ const DEFAULT_ORDER = [ 'line_multi', 'treemap', 'box_plot', - 'separator', 'sunburst', 'sankey', 'word_cloud', @@ -85,7 +84,6 @@ const DEFAULT_ORDER = [ 'bubble', 'deck_geojson', 'horizon', - 'markup', 'deck_multi', 'compare', 'partition', @@ -95,7 +93,6 @@ const DEFAULT_ORDER = [ 'world_map', 'paired_ttest', 'para', - 'iframe', 'country_map', ]; diff --git a/superset-frontend/src/explore/constants.js b/superset-frontend/src/explore/constants.js index 9ef5c9ed74675..b3d3a80043f81 100644 --- a/superset-frontend/src/explore/constants.js +++ b/superset-frontend/src/explore/constants.js @@ -71,8 +71,6 @@ export const sqlaAutoGeneratedMetricNameRegex = /^(sum|min|max|avg|count|count_d export const sqlaAutoGeneratedMetricRegex = /^(LONG|DOUBLE|FLOAT)?(SUM|AVG|MAX|MIN|COUNT)\([A-Z0-9_."]*\)$/i; export const druidAutoGeneratedMetricRegex = /^(LONG|DOUBLE|FLOAT)?(SUM|MAX|MIN|COUNT)\([A-Z0-9_."]*\)$/i; -export const EXPLORE_ONLY_VIZ_TYPE = ['separator', 'markup']; - export const TIME_FILTER_LABELS = { time_range: t('Time Range'), granularity_sqla: t('Time Column'), diff --git a/superset-frontend/src/visualizations/presets/MainPreset.js b/superset-frontend/src/visualizations/presets/MainPreset.js index cf582cb1d2e17..3fcad1ab7e982 100644 --- a/superset-frontend/src/visualizations/presets/MainPreset.js +++ b/superset-frontend/src/visualizations/presets/MainPreset.js @@ -29,9 +29,7 @@ import ForceDirectedChartPlugin from '@superset-ui/legacy-plugin-chart-force-dir import HeatmapChartPlugin from '@superset-ui/legacy-plugin-chart-heatmap'; import HistogramChartPlugin from '@superset-ui/legacy-plugin-chart-histogram'; import HorizonChartPlugin from '@superset-ui/legacy-plugin-chart-horizon'; -import IframeChartPlugin from '@superset-ui/legacy-plugin-chart-iframe'; import MapBoxChartPlugin from '@superset-ui/legacy-plugin-chart-map-box'; -import MarkupChartPlugin from '@superset-ui/legacy-plugin-chart-markup'; import PairedTTestChartPlugin from '@superset-ui/legacy-plugin-chart-paired-t-test'; import ParallelCoordinatesChartPlugin from '@superset-ui/legacy-plugin-chart-parallel-coordinates'; import PartitionChartPlugin from '@superset-ui/legacy-plugin-chart-partition'; @@ -87,12 +85,9 @@ export default class MainPreset extends Preset { new HeatmapChartPlugin().configure({ key: 'heatmap' }), new HistogramChartPlugin().configure({ key: 'histogram' }), new HorizonChartPlugin().configure({ key: 'horizon' }), - new IframeChartPlugin().configure({ key: 'iframe' }), new LineChartPlugin().configure({ key: 'line' }), new LineMultiChartPlugin().configure({ key: 'line_multi' }), new MapBoxChartPlugin().configure({ key: 'mapbox' }), - new MarkupChartPlugin().configure({ key: 'markup' }), - new MarkupChartPlugin().configure({ key: 'separator' }), new PairedTTestChartPlugin().configure({ key: 'paired_ttest' }), new ParallelCoordinatesChartPlugin().configure({ key: 'para' }), new PartitionChartPlugin().configure({ key: 'partition' }), diff --git a/superset/migrations/versions/978245563a02_migrate_iframe_to_dash_markdown.py b/superset/migrations/versions/978245563a02_migrate_iframe_to_dash_markdown.py new file mode 100644 index 0000000000000..3d03ebe70c404 --- /dev/null +++ b/superset/migrations/versions/978245563a02_migrate_iframe_to_dash_markdown.py @@ -0,0 +1,198 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Migrate iframe in dashboard to markdown component + +Revision ID: 978245563a02 +Revises: f2672aa8350a +Create Date: 2020-08-12 00:24:39.617899 + +""" +import collections +import json +import logging +import uuid +from collections import defaultdict + +from alembic import op +from sqlalchemy import and_, Column, ForeignKey, Integer, String, Table, Text +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import relationship + +from superset import db + +# revision identifiers, used by Alembic. +revision = "978245563a02" +down_revision = "f2672aa8350a" + +Base = declarative_base() + + +class Slice(Base): + """Declarative class to do query in upgrade""" + + __tablename__ = "slices" + id = Column(Integer, primary_key=True) + params = Column(Text) + viz_type = Column(String(250)) + + +dashboard_slices = Table( + "dashboard_slices", + Base.metadata, + Column("id", Integer, primary_key=True), + Column("dashboard_id", Integer, ForeignKey("dashboards.id")), + Column("slice_id", Integer, ForeignKey("slices.id")), +) + +slice_user = Table( + "slice_user", + Base.metadata, + Column("id", Integer, primary_key=True), + Column("user_id", Integer, ForeignKey("ab_user.id")), + Column("slice_id", Integer, ForeignKey("slices.id")), +) + + +class Dashboard(Base): + __tablename__ = "dashboards" + id = Column(Integer, primary_key=True) + position_json = Column(Text) + slices = relationship("Slice", secondary=dashboard_slices, backref="dashboards") + + +def create_new_markdown_component(chart_position, url): + return { + "type": "MARKDOWN", + "id": "MARKDOWN-{}".format(uuid.uuid4().hex[:8]), + "children": [], + "parents": chart_position["parents"], + "meta": { + "width": chart_position["meta"]["width"], + "height": chart_position["meta"]["height"], + "code": f'', + }, + } + + +def upgrade(): + bind = op.get_bind() + session = db.Session(bind=bind) + + dash_to_migrate = defaultdict(list) + iframe_urls = defaultdict(list) + + try: + # find iframe viz_type and its url + iframes = session.query(Slice).filter_by(viz_type="iframe").all() + iframe_ids = [slc.id for slc in iframes] + + for iframe in iframes: + iframe_params = json.loads(iframe.params or "{}") + url = iframe_params.get("url") + iframe_urls[iframe.id] = url + + # find iframe viz_type that used in dashboard + dash_slc = ( + session.query(dashboard_slices) + .filter(dashboard_slices.c.slice_id.in_(iframe_ids)) + .all() + ) + for entry in dash_slc: + dash_to_migrate[entry.dashboard_id].append(entry.slice_id) + dashboard_ids = list(dash_to_migrate.keys()) + + # replace iframe in dashboard metadata + dashboards = ( + session.query(Dashboard).filter(Dashboard.id.in_(dashboard_ids)).all() + ) + for i, dashboard in enumerate(dashboards): + print( + f"scanning dashboard ({i + 1}/{len(dashboards)}) dashboard: {dashboard.id} >>>>" + ) + + # remove iframe slices from dashboard + dashboard.slices = [ + slc for slc in dashboard.slices if slc.id not in iframe_ids + ] + + # find iframe chart position in metadata + # and replace it with markdown component + position_dict = json.loads(dashboard.position_json or "{}") + for key, chart_position in position_dict.items(): + if ( + chart_position + and isinstance(chart_position, dict) + and chart_position["type"] == "CHART" + and chart_position["meta"] + and chart_position["meta"]["chartId"] in iframe_ids + ): + iframe_id = chart_position["meta"]["chartId"] + # make new markdown component + markdown = create_new_markdown_component( + chart_position, iframe_urls[iframe_id] + ) + position_dict.pop(key) + position_dict[markdown["id"]] = markdown + + # add markdown to layout tree + parent_id = markdown["parents"][-1] + children = position_dict[parent_id]["children"] + children.remove(key) + children.append(markdown["id"]) + + dashboard.position_json = json.dumps( + position_dict, + indent=None, + separators=(",", ":"), + sort_keys=True, + ) + session.merge(dashboard) + + # remove iframe, separator and markup charts + slices_to_remove = ( + session.query(Slice) + .filter(Slice.viz_type.in_(["iframe", "separator", "markup"])) + .all() + ) + slices_ids = [slc.id for slc in slices_to_remove] + + # remove dependencies first + session.query(dashboard_slices).filter( + dashboard_slices.c.slice_id.in_(slices_ids) + ).delete(synchronize_session=False) + + session.query(slice_user).filter(slice_user.c.slice_id.in_(slices_ids)).delete( + synchronize_session=False + ) + + # remove slices + session.query(Slice).filter(Slice.id.in_(slices_ids)).delete( + synchronize_session=False + ) + + except Exception as ex: + logging.exception(f"dashboard {dashboard.id} has error: {ex}") + + session.commit() + session.close() + + +def downgrade(): + # note: this upgrade is irreversible. + # this migration removed all iframe, separator, and markup type slices, + # and Superset will not support these 3 viz_type anymore. + pass diff --git a/superset/migrations/versions/f80a3b88324b_.py b/superset/migrations/versions/f80a3b88324b_.py new file mode 100644 index 0000000000000..7defdc5e2be45 --- /dev/null +++ b/superset/migrations/versions/f80a3b88324b_.py @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""empty message + +Revision ID: f80a3b88324b +Revises: ('978245563a02', 'f120347acb39') +Create Date: 2020-08-12 15:47:56.580191 + +""" + +# revision identifiers, used by Alembic. +revision = "f80a3b88324b" +down_revision = ("978245563a02", "f120347acb39") + +import sqlalchemy as sa +from alembic import op + + +def upgrade(): + pass + + +def downgrade(): + pass diff --git a/superset/tasks/slack_util.py b/superset/tasks/slack_util.py index 865aa59782455..64e992390da3d 100644 --- a/superset/tasks/slack_util.py +++ b/superset/tasks/slack_util.py @@ -26,7 +26,7 @@ from superset import app # Globals -config = app.config # type: ignore +config = app.config logger = logging.getLogger("tasks.slack_util") diff --git a/superset/viz.py b/superset/viz.py index 14eedf0ba1737..5dc05edfc98a2 100644 --- a/superset/viz.py +++ b/superset/viz.py @@ -2066,25 +2066,6 @@ def get_data(self, df: pd.DataFrame) -> VizData: return d -class IFrameViz(BaseViz): - - """You can squeeze just about anything in this iFrame component""" - - viz_type = "iframe" - verbose_name = _("iFrame") - credits = 'a Superset original' - is_timeseries = False - - def query_obj(self) -> QueryObjectDict: - return {} - - def get_df(self, query_obj: Optional[QueryObjectDict] = None) -> pd.DataFrame: - return pd.DataFrame() - - def get_data(self, df: pd.DataFrame) -> VizData: - return {"iframe": True} - - class ParallelCoordinatesViz(BaseViz): """Interactive parallel coordinate implementation