Merge pull request #1582 from merico-dev/1579-query-from-merico-metri…

…c-system

1579 query from merico metric system

api/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devtable/api",
-  "version": "13.44.2",
+  "version": "14.1.1",
   "description": "",
   "main": "index.js",
   "scripts": {

api/src/api_models/dashboard_content.ts

@@ -29,12 +29,12 @@ export class Query {
   })
   key: string;
 
-  @IsString()
+  @IsObject()
   @ApiModelProperty({
-    description: 'Query SQL',
+    description: 'Query config, vary by type',
     required: true,
   })
-  sql: string;
+  config: any;
 
   @IsString()
   @ApiModelProperty({

api/src/api_models/query.ts

@@ -30,13 +30,13 @@ export class QueryParams {
   name: 'QueryRequest',
 })
 export class QueryRequest {
-  @IsIn(['postgresql', 'mysql', 'http'])
+  @IsIn(['postgresql', 'mysql', 'http', 'merico_metric_system'])
   @ApiModelProperty({
     description: 'datasource type of query',
     required: true,
-    enum: ['postgresql', 'mysql', 'http'],
+    enum: ['postgresql', 'mysql', 'http', 'merico_metric_system'],
   })
-  type: 'postgresql' | 'mysql' | 'http';
+  type: 'postgresql' | 'mysql' | 'http' | 'merico_metric_system';
 
   @IsString()
   @ApiModelProperty({
@@ -219,3 +219,68 @@ export class QueryStructureRequest {
   })
   offset?: number;
 }
+
+@ApiModel({
+  description: 'Query Merico Metrics Info',
+  name: 'QueryMericoMetricInfoRequest',
+})
+export class QueryMericoMetricInfoRequest {
+  @IsString()
+  @ApiModelProperty({
+    description: 'datasource key',
+    required: true,
+  })
+  key: string;
+
+  @IsString()
+  @ApiModelProperty({
+    description:
+      'query to be executed against selected datasource. For http data sources query must be a json parsable object string',
+    required: true,
+  })
+  query: string;
+
+  @IsString()
+  @ApiModelProperty({
+    description: 'id of the dashboard content',
+    required: true,
+  })
+  content_id: string;
+
+  @IsObject()
+  @Type(() => QueryParams)
+  @ValidateNested({ each: true })
+  @ApiModelProperty({
+    description: 'Query params',
+    required: true,
+    model: 'QueryParams',
+  })
+  params: QueryParams;
+
+  @IsOptional()
+  @IsObject()
+  @ApiModelProperty({
+    description: 'Query env config',
+    required: false,
+    type: SwaggerDefinitionConstant.JSON,
+  })
+  env?: Record<string, any>;
+
+  @IsOptional()
+  @IsBoolean()
+  @ApiModelProperty({
+    description: 'refresh cached data',
+    required: false,
+  })
+  refresh_cache?: boolean;
+
+  @IsOptional()
+  @Type(() => Authentication)
+  @ValidateNested({ each: true })
+  @ApiModelProperty({
+    description: 'authentication object for use with app_id / app_secret',
+    required: false,
+    model: 'Authentication',
+  })
+  authentication?: Authentication;
+}

api/src/controller/query.controller.ts

@@ -4,7 +4,7 @@ import { controller, httpPost, interfaces } from 'inversify-express-utils';
 import { ApiOperationPost, ApiPath } from 'swagger-express-ts';
 import { QueryService } from '../services/query.service';
 import { validate } from '../middleware/validation';
-import { QueryRequest, QueryStructureRequest } from '../api_models/query';
+import { QueryRequest, QueryStructureRequest, QueryMericoMetricInfoRequest } from '../api_models/query';
 import permission from '../middleware/permission';
 import { PERMISSIONS } from '../services/role.service';
 import { ApiKey } from '../api_models/api';
@@ -38,10 +38,11 @@ export class QueryController implements interfaces.Controller {
     try {
       const auth: Account | ApiKey | undefined = req.body.auth;
       const { type, key, query, content_id, query_id, params, env, refresh_cache } = req.body as QueryRequest;
+      const needToDecodeQuery = type !== 'http' && type !== 'merico_metric_system';
       const result = await this.queryService.query(
         type,
         key,
-        type !== 'http' ? decode(query) : query,
+        needToDecodeQuery ? decode(query) : query,
         content_id,
         query_id,
         params,
@@ -89,4 +90,44 @@ export class QueryController implements interfaces.Controller {
       next(error);
     }
   }
+
+  @ApiOperationPost({
+    path: '/merico_metric_info',
+    description: 'query merico_metric_info',
+    parameters: {
+      body: { description: 'Query Metric Info', required: true, model: 'QueryMericoMetricInfoRequest' },
+    },
+    responses: {
+      200: { description: 'Query result' },
+      500: { description: 'ApiError', model: 'ApiError' },
+    },
+  })
+  @httpPost(
+    '/merico_metric_info',
+    permission({ match: 'all', permissions: [PERMISSIONS.DASHBOARD_MANAGE] }),
+    validate(QueryMericoMetricInfoRequest),
+  )
+  public async queryMericoMetricInfo(
+    req: express.Request,
+    res: express.Response,
+    next: express.NextFunction,
+  ): Promise<void> {
+    try {
+      const auth: Account | ApiKey | undefined = req.body.auth;
+      const { key, query, content_id, params, env, refresh_cache } = req.body as QueryMericoMetricInfoRequest;
+      const result = await this.queryService.queryMericoMetricInfo(
+        key,
+        query,
+        content_id,
+        params,
+        env || {},
+        refresh_cache,
+        req.locale,
+        auth,
+      );
+      res.json(result);
+    } catch (error) {
+      next(error);
+    }
+  }
 }

api/src/dashboard_migration/handlers/13.45.0.ts

@@ -0,0 +1,171 @@
+type LegacyQuery = {
+  id: string;
+  name: string;
+  key: string;
+  pre_process: string;
+  post_process: string;
+  run_by: string[];
+  react_to: string[];
+  dep_query_ids: string[];
+  type: 'postgresql' | 'mysql' | 'http' | 'transform' | 'merico_metric_system';
+  sql: string;
+};
+type BaseQuery = {
+  id: string;
+  name: string;
+  key: string;
+  pre_process: string;
+  post_process: string;
+  run_by: string[];
+};
+type DBQuery = BaseQuery & {
+  type: 'postgresql' | 'mysql';
+  config: {
+    _type: 'postgresql' | 'mysql';
+    sql: string;
+  };
+};
+type HTTPQuery = BaseQuery & {
+  type: 'http';
+  config: {
+    _type: 'http';
+    react_to: string[];
+  };
+};
+type TransformQuery = BaseQuery & {
+  type: 'transform';
+  config: {
+    _type: 'transform';
+    dep_query_ids: string[];
+  };
+};
+type DimensionToVariable = {
+  dimension: string;
+  variable: string;
+};
+type MericoMetricQuery = BaseQuery & {
+  type: 'merico_metric_system';
+  config: {
+    _type: 'merico_metric_system';
+    id: string;
+    type: string;
+    filters: DimensionToVariable[];
+    groupBys: string[];
+    timeQuery: {
+      enabled: boolean;
+      range_variable: string;
+      unit_variable: string;
+      timezone: string;
+      stepKeyFormat: string;
+    };
+  };
+};
+type NewQuery = DBQuery | HTTPQuery | TransformQuery | MericoMetricQuery;
+
+function getCommonProperties(q: LegacyQuery) {
+  const { id, name, key, pre_process, post_process, run_by } = q;
+  return { id, name, key, pre_process, post_process, run_by };
+}
+
+function toDBQuery(q: LegacyQuery): DBQuery {
+  const { sql, type } = q;
+  if (type !== 'postgresql' && type !== 'mysql') {
+    throw new Error(`wrong type[${type}]`);
+  }
+
+  return {
+    ...getCommonProperties(q),
+    type,
+    config: { _type: type, sql },
+  };
+}
+
+function toHTTPQuery(q: LegacyQuery): HTTPQuery {
+  const { react_to, type } = q;
+  if (type !== 'http') {
+    throw new Error(`wrong type[${type}]`);
+  }
+
+  return {
+    ...getCommonProperties(q),
+    type,
+    config: { react_to, _type: type },
+  };
+}
+
+function toTransformQuery(q: LegacyQuery): TransformQuery {
+  const { dep_query_ids, type } = q;
+  if (type !== 'transform') {
+    throw new Error(`wrong type[${type}]`);
+  }
+
+  return {
+    ...getCommonProperties(q),
+    type,
+    config: { _type: type, dep_query_ids },
+  };
+}
+
+function toMMQuery(q: LegacyQuery): MericoMetricQuery {
+  const { type } = q;
+  if (type !== 'merico_metric_system') {
+    throw new Error(`wrong type[${type}]`);
+  }
+
+  return {
+    ...getCommonProperties(q),
+    type,
+    config: {
+      _type: type,
+      id: '',
+      type: 'derived',
+      filters: [],
+      groupBys: [],
+      timeQuery: {
+        enabled: false,
+        range_variable: '',
+        unit_variable: '',
+        timezone: 'PRC',
+        stepKeyFormat: 'YYYY-MM-DD',
+      },
+    },
+  };
+}
+
+function upgradeQueries(queries: LegacyQuery[]): NewQuery[] {
+  return queries.map((q) => {
+    if ('config' in q) {
+      console.warn("query is already upgraded, but it's unexpected");
+      console.log(JSON.stringify(q, null, 2));
+      return q as NewQuery;
+    }
+    switch (q.type) {
+      case 'postgresql':
+      case 'mysql':
+        return toDBQuery(q);
+      case 'http':
+        return toHTTPQuery(q);
+      case 'transform':
+        return toTransformQuery(q);
+      case 'merico_metric_system':
+        return toMMQuery(q);
+      default:
+        throw new Error(`wrong type[${q.type}]`);
+    }
+  });
+}
+
+/**
+ * https://github.com/merico-dev/table/issues/1579
+ */
+export function main({ definition, ...rest }: Record<string, any>) {
+  const finalQueries = upgradeQueries(definition.queries);
+  return {
+    ...rest,
+    definition: {
+      ...definition,
+      queries: finalQueries,
+    },
+    version: '13.45.0',
+  };
+}

api/src/dashboard_migration/index.ts

@@ -26,6 +26,7 @@ export const versions = [
   '10.41.0',
   '11.0.0',
   '11.10.0',
+  '13.45.0',
   // ... future versions
 ];
 

api/src/services/datasource.service.ts

@@ -66,7 +66,7 @@ export class DataSourceService {
       offset,
       data: datasources.map((d) => ({
         ...d,
-        config: d.type !== 'http' ? {} : d.config,
+        config: d.type !== 'http' && d.type !== 'merico_metric_system' ? {} : d.config,
       })),
     };
   }
@@ -120,7 +120,7 @@ export class DataSourceService {
     maybeEncryptPassword(config);
     const dataSourceRepo = dashboardDataSource.getRepository(DataSource);
     const dataSource = await dataSourceRepo.findOneByOrFail({ id });
-    if (dataSource.type !== 'http') {
+    if (dataSource.type !== 'http' && dataSource.type !== 'merico_metric_system') {
       throw new ApiError(BAD_REQUEST, { message: translate('DATASOURCE_ONLY_HTTP_IS_EDITABLE', locale) });
     }
     dataSource.config = config;