Schema Viewer Drawer (getredash#3291)

* Process extra column metadata for a few sql-based data sources. * Add Table and Column metadata tables. * Periodically update table and column schema tables in a celery task. * Fetching schema returns data from table and column metadata tables. * Add tests for backend changes. * Front-end shows extra table metadata and uses new schema response. * Delete datasource schema data when deleting a data source. * Process and store data source schema when a data source is first created or after a migration. * Tables should have a unique name per datasource. * Addressing review comments. * Update migration file for mixins. * Appease PEP8 * Upgrade migration file for rebase. * Cascade delete. * Adding org_id * Remove redundant column and table prefixes. * Non-existing tables and columns should be filtered out on the server side not client side. * Fetching table samples should be optional and should happen in a separate task per table. * Allow users to force a schema refresh. * Use updated_at to help prune old schema metadata periodically. * Using settings.SCHEMAS_REFRESH_QUEUE * fix for getredash#2426 test * more stable test_interactive_new * Closes #927, #928: Schema refresh improvements. * Closes #934, #935: Remove type from schema browser and don't show empty example column in schema drawer (#936) * Speed up schema fetch requests with fewer postgres queries. * Add column metadata to Athena glue processing. * Fix bug assuming 'metadata' exists for every table. * Closes #939: Persisted, existing table metadata should be updated. * Sample processing should be rate-limited. * Add cli command for refreshing data samples. * Schema refreshes should not overwrite column 'example' field. * refresh_samples() should filter tables_to_sample on the datasource's id being sampled * Correctly wrap long text in schema drawer. Schema Improvements Part 2: Add data source config options. Adding BigQuery schema drawer with data types and samples. Add empty migration to replace the removed schedule_until migration Add merge migration. Fix spacing issue with data scanned value in query execution metadata. Increase schema refresh timeout. Remove old migrations. Co-authored-by: Alison <github@bankofknowledge.net> Co-authored-by: Jannis Leidel <jannis@leidel.info>
mozilla · Oct 15, 2020 · e4365e5 · e4365e5
1 parent 15b4c16
commit e4365e5
Show file tree

Hide file tree

Showing 54 changed files with 2,741 additions and 222 deletions.
diff --git a/client/app/assets/less/inc/base.less b/client/app/assets/less/inc/base.less
@@ -114,6 +114,10 @@ strong {
   transition: height 0s, width 0s !important;
 }
 
+.admin-schema-editor {
+  padding: 50px 0;
+}
+
 .bg-ace {
   background-color: fade(@redash-gray, 12%) !important;
 }

diff --git a/client/app/assets/less/inc/popover.less b/client/app/assets/less/inc/popover.less
@@ -1,5 +1,7 @@
 .popover {
     box-shadow: fade(@redash-gray, 25%) 0px 0px 15px 0px;
+    color: #000000;
+    z-index: 1000000001; // So that it can popover a dropdown menu
 }
 
 .popover-title {
@@ -19,4 +21,4 @@
     p {
         margin-bottom: 0; 
     }
-} 
+} 
diff --git a/client/app/assets/less/inc/schema-browser.less b/client/app/assets/less/inc/schema-browser.less
@@ -8,14 +8,14 @@ div.table-name {
   position: relative;
   height: 22px;
 
-  .copy-to-editor {
+  .copy-to-editor, .info {
     display: none;
   }
 
   &:hover {
     background: fade(@redash-gray, 10%);
 
-    .copy-to-editor {
+    .copy-to-editor, .info {
       display: flex;
     }
   }
@@ -45,7 +45,7 @@ div.table-name {
     background: transparent;
   }
 
-  .copy-to-editor {
+  .copy-to-editor, .info {
     color: fade(@redash-gray, 90%);
     cursor: pointer;
     position: absolute;
@@ -58,6 +58,10 @@ div.table-name {
     justify-content: center;
   }
 
+  .info {
+    right: 20px
+  }
+
   .table-open {
     padding: 0 22px 0 26px;
     overflow: hidden;
@@ -73,14 +77,14 @@ div.table-name {
       text-transform: uppercase;
     }
 
-    .copy-to-editor {
+    .copy-to-editor, .info {
       display: none;
     }
 
     &:hover {
       background: fade(@redash-gray, 10%);
 
-      .copy-to-editor {
+      .copy-to-editor, .info {
         display: flex;
       }
     }

diff --git a/client/app/assets/less/redash/query.less b/client/app/assets/less/redash/query.less
@@ -493,3 +493,17 @@ nav .rg-bottom {
     padding-right: 0;
   }
 }
+
+.ui-select-choices-row .info {
+  display: none;
+}
+
+.ui-select-choices-row {
+    &:hover {
+        .info {
+            cursor: pointer;
+            width: 20px;
+            display: inline;
+        }
+    }
+}
diff --git a/client/app/components/dynamic-form/dynamicFormHelper.js b/client/app/components/dynamic-form/dynamicFormHelper.js
@@ -100,6 +100,13 @@ function getFields(type = {}, target = { options: {} }) {
       placeholder: `My ${type.name}`,
       autoFocus: isNewTarget,
     },
+    {
+      name: "description",
+      title: "Description",
+      type: "text",
+      required: false,
+      initialValue: target.description,
+    },
     ...orderedInputs(configurationSchema.properties, configurationSchema.order, target.options),
   ];
 
@@ -108,6 +115,7 @@ function getFields(type = {}, target = { options: {} }) {
 
 function updateTargetWithValues(target, values) {
   target.name = values.name;
+  target.description = values.description;
   Object.keys(values).forEach(key => {
     if (key !== "name") {
       target.options[key] = values[key];

diff --git a/client/app/components/proptypes.js b/client/app/components/proptypes.js
@@ -11,8 +11,16 @@ export const DataSource = PropTypes.shape({
   type_name: PropTypes.string,
 });
 
+export const DataSourceMetadata = PropTypes.shape({
+  key: PropTypes.number,
+  name: PropTypes.string,
+  type: PropTypes.string,
+  example: PropTypes.string,
+  description: PropTypes.string,
+});
+
 export const Table = PropTypes.shape({
-  columns: PropTypes.arrayOf(PropTypes.string).isRequired,
+  columns: PropTypes.arrayOf(PropTypes.object).isRequired,
 });
 
 export const Schema = PropTypes.arrayOf(Table);
@@ -31,6 +39,60 @@ export const RefreshScheduleDefault = {
   until: null,
 };
 
+export const TableMetadata = PropTypes.shape({
+  key: PropTypes.number.isRequired,
+  name: PropTypes.string.isRequired,
+  description: PropTypes.string,
+  visible: PropTypes.bool.isRequired,
+});
+
+export const Field = PropTypes.shape({
+  name: PropTypes.string.isRequired,
+  title: PropTypes.string,
+  type: PropTypes.oneOf([
+    "ace",
+    "text",
+    "textarea",
+    "email",
+    "password",
+    "number",
+    "checkbox",
+    "file",
+    "select",
+    "content",
+  ]).isRequired,
+  initialValue: PropTypes.oneOfType([
+    PropTypes.string,
+    PropTypes.number,
+    PropTypes.bool,
+    PropTypes.arrayOf(PropTypes.string),
+    PropTypes.arrayOf(PropTypes.number),
+  ]),
+  content: PropTypes.node,
+  mode: PropTypes.string,
+  required: PropTypes.bool,
+  extra: PropTypes.bool,
+  readOnly: PropTypes.bool,
+  autoFocus: PropTypes.bool,
+  minLength: PropTypes.number,
+  placeholder: PropTypes.string,
+  contentAfter: PropTypes.oneOfType([PropTypes.node, PropTypes.func]),
+  loading: PropTypes.bool,
+  props: PropTypes.object, // eslint-disable-line react/forbid-prop-types
+});
+
+export const Action = PropTypes.shape({
+  name: PropTypes.string.isRequired,
+  callback: PropTypes.func.isRequired,
+  type: PropTypes.string,
+  pullRight: PropTypes.bool,
+  disabledWhenDirty: PropTypes.bool,
+});
+
+export const AntdForm = PropTypes.shape({
+  validateFieldsAndScroll: PropTypes.func,
+});
+
 export const UserProfile = PropTypes.shape({
   id: PropTypes.number.isRequired,
   name: PropTypes.string.isRequired,

diff --git a/client/app/components/queries/QueryEditor/ace.js b/client/app/components/queries/QueryEditor/ace.js
@@ -32,6 +32,7 @@ function buildTableColumnKeywords(table) {
   table.columns.forEach(column => {
     const columnName = get(column, "name");
     keywords.push({
+      caption: columnName,
       name: `${table.name}.${columnName}`,
       value: `${table.name}.${columnName}`,
       score: 100,

diff --git a/client/app/components/queries/SchemaBrowser.jsx b/client/app/components/queries/SchemaBrowser.jsx
@@ -15,6 +15,7 @@ import useImmutableCallback from "@/lib/hooks/useImmutableCallback";
 import LoadingState from "../items-list/components/LoadingState";
 import { clientConfig } from "@/services/auth";
 import notification from "@/services/notification";
+import SchemaData from "@/components/queries/SchemaData";
 
 const SchemaItemColumnType = PropTypes.shape({
   name: PropTypes.string.isRequired,
@@ -31,7 +32,7 @@ export const SchemaItemType = PropTypes.shape({
 const schemaTableHeight = 22;
 const schemaColumnHeight = 18;
 
-function SchemaItem({ item, expanded, onToggle, onSelect, ...props }) {
+function SchemaItem({ item, expanded, onToggle, onSelect, onShowSchema, ...props }) {
   const handleSelect = useCallback(
     (event, ...args) => {
       event.preventDefault();
@@ -41,6 +42,15 @@ function SchemaItem({ item, expanded, onToggle, onSelect, ...props }) {
     [onSelect]
   );
 
+  const handleShowSchema = useCallback(
+    (event, ...args) => {
+      event.preventDefault();
+      event.stopPropagation();
+      onShowSchema(...args);
+    },
+    [onShowSchema]
+  );
+
   if (!item) {
     return null;
   }
@@ -53,6 +63,12 @@ function SchemaItem({ item, expanded, onToggle, onSelect, ...props }) {
           <span title={item.name}>{item.name}</span>
           {!isNil(item.size) && <span> ({item.size})</span>}
         </strong>
+        <i
+            className="fa fa-question-circle info"
+            title="More Info"
+            aria-hidden="true"
+            onClick={e => handleShowSchema(e, item)}
+        />
         <i
           className="fa fa-angle-double-right copy-to-editor"
           aria-hidden="true"
@@ -107,7 +123,7 @@ function SchemaLoadingState() {
   );
 }
 
-export function SchemaList({ loading, schema, expandedFlags, onTableExpand, onItemSelect }) {
+export function SchemaList({ loading, schema, expandedFlags, onTableExpand, onItemSelect, openSchemaInfo, closeSchemaInfo }) {
   const [listRef, setListRef] = useState(null);
 
   useEffect(() => {
@@ -143,6 +159,7 @@ export function SchemaList({ loading, schema, expandedFlags, onTableExpand, onIt
                     expanded={expandedFlags[item.name]}
                     onToggle={() => onTableExpand(item.name)}
                     onSelect={onItemSelect}
+                    onShowSchema={openSchemaInfo}
                   />
                 );
               }}
@@ -154,6 +171,14 @@ export function SchemaList({ loading, schema, expandedFlags, onTableExpand, onIt
   );
 }
 
+function itemExists(item) {
+  if ("visible" in item) {
+    return item.visible;
+  } else {
+    return false;
+  }
+};
+
 export function applyFilterOnSchema(schema, filterString, showHidden, toggleString) {
   const filters = filter(filterString.toLowerCase().split(/\s+/), s => s.length > 0);
 
@@ -170,6 +195,9 @@ export function applyFilterOnSchema(schema, filterString, showHidden, toggleStri
       }
   }
 
+  // Filter out all columns set to invisible
+  schema = filter(schema, itemExists);
+
   // Empty string: return original schema
   if (filters.length === 0) {
     return schema;
@@ -221,8 +249,17 @@ export default function SchemaBrowser({
   const [handleToggleChange] = useDebouncedCallback(setShowHidden, 100);
   const [expandedFlags, setExpandedFlags] = useState({});
 
+  const [showSchemaInfo, setShowSchemaInfo] = useState(false);
+  const [tableName, setTableName] = useState("");
+  const [tableDescription, setTableDescription] = useState("");
+  const [tableMetadata, setTableMetadata] = useState([]);
+  const [sampleQueries, setSampleQueries] = useState([]);
+
   const handleSchemaUpdate = useImmutableCallback(onSchemaUpdate);
 
+  useEffect(() => {
+    setExpandedFlags({});
+  }, [schema]);
 
   useEffect(() => {
     setExpandedFlags({});
@@ -255,6 +292,18 @@ export default function SchemaBrowser({
     return toggleString;
   }
 
+  function openSchemaInfo(table) {
+    setTableName(table.name);
+    setTableDescription(table.description);
+    setTableMetadata(table.columns);
+    setSampleQueries(Object.values(table.sample_queries));
+    setShowSchemaInfo(true);
+  }
+
+  function closeSchemaInfo() {
+    setShowSchemaInfo(false);
+  };
+
   return (
     <div className="schema-container" {...props}>
       <div className="schema-control">
@@ -285,6 +334,17 @@ export default function SchemaBrowser({
         expandedFlags={expandedFlags}
         onTableExpand={toggleTable}
         onItemSelect={onItemSelect}
+        toggleString={toggleString}
+        openSchemaInfo={openSchemaInfo}
+        closeSchemaInfo={closeSchemaInfo}
+      />
+      <SchemaData
+        show={showSchemaInfo}
+        tableName={tableName}
+        tableDescription={tableDescription}
+        tableMetadata={tableMetadata}
+        sampleQueries={sampleQueries}
+        onClose={closeSchemaInfo}
       />
     </div>
   );