diff --git a/.github/workflows/cloud.yml b/.github/workflows/cloud.yml index 81ea469639f83..04e2ff0087425 100644 --- a/.github/workflows/cloud.yml +++ b/.github/workflows/cloud.yml @@ -25,6 +25,9 @@ on: - 'package.json' - 'yarn.lock' +env: + CUBEJS_TESSERACT_ORCHESTRATOR: true + jobs: latest-tag-sha: runs-on: ubuntu-20.04 @@ -59,15 +62,31 @@ jobs: matrix: node-version: [ 20.x ] db: [ 'athena', 'bigquery', 'snowflake' ] + target: [ "x86_64-unknown-linux-gnu" ] fail-fast: false steps: - name: Checkout uses: actions/checkout@v4 + - name: Install Rust + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + toolchain: nightly-2024-07-15 + # override: true # this is by default on + rustflags: "" + components: rustfmt + target: ${{ matrix.target }} - name: Install Node.js ${{ matrix.node-version }} uses: actions/setup-node@v4 with: node-version: ${{ matrix.node-version }} + - name: Install cargo-cp-artifact + run: npm install -g cargo-cp-artifact@0.1 + - uses: Swatinem/rust-cache@v2 + with: + workspaces: ./packages/cubejs-backend-native + key: native-${{ runner.OS }}-${{ matrix.target }} + shared-key: native-${{ runner.OS }}-${{ matrix.target }} - name: Get yarn cache directory path id: yarn-cache-dir-path run: echo "dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT" @@ -87,6 +106,8 @@ jobs: run: yarn build - name: Lerna tsc run: yarn tsc + - name: Build native (no python) + run: cd packages/cubejs-backend-native && npm run native:build-release - name: Run Integration tests for ${{ matrix.db }} matrix timeout-minutes: 30 env: diff --git a/.github/workflows/drivers-tests.yml b/.github/workflows/drivers-tests.yml index 1ab0154df375b..3cec502954c68 100644 --- a/.github/workflows/drivers-tests.yml +++ b/.github/workflows/drivers-tests.yml @@ -56,6 +56,9 @@ on: - 'packages/cubejs-backend-native/**' - 'rust/cubesql/**' +env: + CUBEJS_TESSERACT_ORCHESTRATOR: true + jobs: latest-tag-sha: runs-on: ubuntu-20.04 @@ -117,8 +120,8 @@ jobs: - uses: Swatinem/rust-cache@v2 with: workspaces: ./packages/cubejs-backend-native - key: native-${{ runner.OS }}-x86_64-unknown-linux-gnu - shared-key: native-${{ runner.OS }}-x86_64-unknown-linux-gnu + key: native-${{ runner.OS }}-${{ matrix.target }} + shared-key: native-${{ runner.OS }}-${{ matrix.target }} - name: Build native (fallback) if: (matrix.python-version == 'fallback') env: @@ -165,7 +168,12 @@ jobs: uses: actions/download-artifact@v4 with: name: backend-native - path: packages/cubejs-backend-native/ + path: packages/cubejs-backend-native + # current .dockerignore prevents use of native build + - name: Unignore native from .dockerignore + run: | + grep -v -E "packages/cubejs-backend-native/((native)|(index.node))" .dockerignore > .dockerignore.tmp + mv .dockerignore.tmp .dockerignore - name: Build and push uses: docker/build-push-action@v6 with: diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index d1fbc157e5b47..9b0c92d7269d1 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -16,6 +16,8 @@ on: - 'rust/cubesql/**' branches: - master +env: + CUBEJS_TESSERACT_ORCHESTRATOR: true jobs: latest-tag-sha: runs-on: ubuntu-20.04 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 92d896a6421bd..3c3d0e70a329f 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -552,7 +552,7 @@ jobs: - name: Push to Docker Hub uses: docker/build-push-action@v6 with: - context: ./rust/cubestore/ + context: ./rust/ file: ./rust/cubestore/Dockerfile platforms: ${{ matrix.platforms }} build-args: ${{ matrix.build-args }} diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 057532433495d..25d30e269e617 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -9,6 +9,10 @@ on: - 'packages/**' - 'rust/cubestore/**' - 'rust/cubesql/**' + - 'rust/cubenativeutils/**' + - 'rust/cubeorchestrator/**' + - 'rust/cubeshared/**' + - 'rust/cubesqlplanner/**' - '.eslintrc.js' - '.prettierrc' - 'package.json' @@ -24,6 +28,10 @@ on: - 'packages/**' - 'rust/cubestore/**' - 'rust/cubesql/**' + - 'rust/cubenativeutils/**' + - 'rust/cubeorchestrator/**' + - 'rust/cubeshared/**' + - 'rust/cubesqlplanner/**' - '.eslintrc.js' - '.prettierrc' - 'package.json' @@ -31,6 +39,9 @@ on: - 'rollup.config.js' - 'yarn.lock' +env: + CUBEJS_TESSERACT_ORCHESTRATOR: true + jobs: unit: runs-on: ubuntu-20.04 @@ -95,6 +106,8 @@ jobs: command: yarn install --frozen-lockfile - name: Lerna tsc run: yarn tsc + - name: Build native (no python) + run: cd packages/cubejs-backend-native && npm run native:build-release - name: Build client run: yarn build - name: Build cubejs-backend-native (with Python) @@ -102,7 +115,6 @@ jobs: working-directory: ./packages/cubejs-backend-native env: PYO3_PYTHON: python${{ matrix.python-version }} - - name: Lerna test run: yarn lerna run --concurrency 1 --stream --no-prefix unit # - uses: codecov/codecov-action@v1 @@ -111,6 +123,21 @@ jobs: # files: ./packages/*/coverage/clover.xml # flags: cube-backend # verbose: true # optional (default = false) + - name: Cargo test cubeorchestrator + run: | + cargo test --manifest-path rust/cubeorchestrator/Cargo.toml -j 1 + - name: Cargo test cubenativeutils + run: | + cargo test --manifest-path rust/cubenativeutils/Cargo.toml -j 1 + - name: Cargo test cubeshared + run: | + cargo test --manifest-path rust/cubeshared/Cargo.toml -j 1 +# - name: Cargo test cubesql +# run: | +# cargo test --manifest-path rust/cubesql/Cargo.toml -j 1 +# - name: Cargo test cubesqlplanner +# run: | +# cargo test --manifest-path rust/cubesqlplanner/cubesqlplanner/Cargo.toml -j 1 lint: runs-on: ubuntu-20.04 @@ -159,6 +186,21 @@ jobs: run: yarn lint:npm - name: Lerna lint run: yarn lerna run --concurrency 1 lint + - name: Cargo fmt cubeorchestrator + run: | + cargo fmt --manifest-path rust/cubeorchestrator/Cargo.toml -- --check + - name: Cargo fmt cubenativeutils + run: | + cargo fmt --manifest-path rust/cubenativeutils/Cargo.toml -- --check + - name: Cargo fmt cubeshared + run: | + cargo fmt --manifest-path rust/cubeshared/Cargo.toml -- --check +# - name: Cargo fmt cubesql +# run: | +# cargo fmt --manifest-path rust/cubesql/Cargo.toml -- --check +# - name: Cargo fmt cubesqlplanner +# run: | +# cargo fmt --manifest-path rust/cubesqlplanner/cubesqlplanner/Cargo.toml -- --check build: runs-on: ubuntu-20.04 @@ -211,6 +253,21 @@ jobs: run: yarn lerna run --concurrency 1 build env: NODE_OPTIONS: --max_old_space_size=4096 + - name: Cargo build cubeorchestrator + run: | + cargo build --manifest-path rust/cubeorchestrator/Cargo.toml -j 4 + - name: Cargo build cubenativeutils + run: | + cargo build --manifest-path rust/cubenativeutils/Cargo.toml -j 4 + - name: Cargo build cubeshared + run: | + cargo build --manifest-path rust/cubeshared/Cargo.toml -j 4 +# - name: Cargo build cubesql +# run: | +# cargo build --manifest-path rust/cubesql/Cargo.toml -j 4 +# - name: Cargo build cubesqlplanner +# run: | +# cargo build --manifest-path rust/cubesqlplanner/cubesqlplanner/Cargo.toml -j 4 build-cubestore: needs: [latest-tag-sha] @@ -276,6 +333,13 @@ jobs: df -h - name: Checkout uses: actions/checkout@v4 + - name: Install Rust + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + toolchain: nightly-2024-07-15 + # override: true # this is by default on + rustflags: "" + components: rustfmt - name: Install Node.js ${{ matrix.node-version }} uses: actions/setup-node@v4 with: @@ -305,6 +369,9 @@ jobs: command: yarn install --frozen-lockfile - name: Lerna tsc run: yarn tsc + - name: Build cubejs-backend-native (without Python) + run: yarn run native:build-release + working-directory: ./packages/cubejs-backend-native - name: Download cubestored-x86_64-unknown-linux-gnu-release artifact uses: actions/download-artifact@v4 with: @@ -539,6 +606,8 @@ jobs: - 5000:5000 strategy: matrix: + node-version: [ 20 ] + target: [ "x86_64-unknown-linux-gnu" ] dockerfile: - dev.Dockerfile include: @@ -565,21 +634,29 @@ jobs: df -h - name: Checkout uses: actions/checkout@v4 - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Build image - uses: docker/build-push-action@v6 - timeout-minutes: 30 + - name: Install Rust + uses: actions-rust-lang/setup-rust-toolchain@v1 with: - context: . - file: ./packages/cubejs-docker/${{ matrix.dockerfile }} - platforms: linux/amd64 - push: true - tags: localhost:5000/cubejs/cube:${{ matrix.tag }} - - name: Use Node.js 20.x + toolchain: nightly-2024-07-15 + # override: true # this is by default on + rustflags: "" + components: rustfmt + target: ${{ matrix.target }} + - name: Install Node.js ${{ matrix.node-version }} uses: actions/setup-node@v4 with: - node-version: 20.x + node-version: ${{ matrix.node-version }} + - name: Install Yarn + run: npm install -g yarn + - name: Set Yarn version + run: yarn policies set-version v1.22.22 + - name: Install cargo-cp-artifact + run: npm install -g cargo-cp-artifact@0.1 + - uses: Swatinem/rust-cache@v2 + with: + workspaces: ./packages/cubejs-backend-native + key: native-${{ runner.OS }}-${{ matrix.target }} + shared-key: native-${{ runner.OS }}-${{ matrix.target }} - name: Get yarn cache directory path id: yarn-cache-dir-path run: echo "dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT" @@ -591,8 +668,6 @@ jobs: key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} restore-keys: | ${{ runner.os }}-yarn- - - name: Set Yarn version - run: yarn policies set-version v1.22.22 - name: Yarn install uses: nick-fields/retry@v3 env: @@ -607,6 +682,24 @@ jobs: run: yarn build - name: Lerna tsc run: yarn tsc + - name: Build native (no python) + run: cd packages/cubejs-backend-native && npm run native:build-release + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + # current .dockerignore prevents use of native build + - name: Unignore native from .dockerignore + run: | + grep -v -E "packages/cubejs-backend-native/((native)|(index.node))" .dockerignore > .dockerignore.tmp + mv .dockerignore.tmp .dockerignore + - name: Build image + uses: docker/build-push-action@v6 + timeout-minutes: 30 + with: + context: . + file: ./packages/cubejs-docker/${{ matrix.dockerfile }} + platforms: linux/amd64 + push: true + tags: localhost:5000/cubejs/cube:${{ matrix.tag }} - name: Testing CubeJS (container mode) via BirdBox run: | cd packages/cubejs-testing/ diff --git a/.github/workflows/rust-cubestore-master.yml b/.github/workflows/rust-cubestore-master.yml index 29211f7c24a83..eb72e456f30ea 100644 --- a/.github/workflows/rust-cubestore-master.yml +++ b/.github/workflows/rust-cubestore-master.yml @@ -151,7 +151,7 @@ jobs: - name: Push to Docker Hub uses: docker/build-push-action@v6 with: - context: ./rust/cubestore + context: ./rust file: ./rust/cubestore/Dockerfile platforms: ${{ matrix.platforms }} build-args: ${{ matrix.build-args }} diff --git a/.github/workflows/rust-cubestore.yml b/.github/workflows/rust-cubestore.yml index 3b4a2eb6182c3..05f443a705cc5 100644 --- a/.github/workflows/rust-cubestore.yml +++ b/.github/workflows/rust-cubestore.yml @@ -99,7 +99,7 @@ jobs: - name: Build only uses: docker/build-push-action@v6 with: - context: ./rust/cubestore/ + context: ./rust/ file: ./rust/cubestore/Dockerfile platforms: ${{ matrix.platforms }} build-args: ${{ matrix.build-args }} diff --git a/packages/cubejs-api-gateway/src/SubscriptionServer.ts b/packages/cubejs-api-gateway/src/SubscriptionServer.ts index 9cf7a17b32886..c45c302915d96 100644 --- a/packages/cubejs-api-gateway/src/SubscriptionServer.ts +++ b/packages/cubejs-api-gateway/src/SubscriptionServer.ts @@ -19,7 +19,7 @@ const calcMessageLength = (message: unknown) => Buffer.byteLength( typeof message === 'string' ? message : JSON.stringify(message) ); -export type WebSocketSendMessageFn = (connectionId: string, message: any) => void; +export type WebSocketSendMessageFn = (connectionId: string, message: any) => Promise; export class SubscriptionServer { public constructor( @@ -31,7 +31,7 @@ export class SubscriptionServer { } public resultFn(connectionId: string, messageId: string, requestId: string | undefined) { - return (message, { status } = { status: 200 }) => { + return async (message, { status } = { status: 200 }) => { this.apiGateway.log({ type: 'Outgoing network usage', service: 'api-ws', diff --git a/packages/cubejs-api-gateway/src/gateway.ts b/packages/cubejs-api-gateway/src/gateway.ts index 006db77b91d3b..af90986b4117b 100644 --- a/packages/cubejs-api-gateway/src/gateway.ts +++ b/packages/cubejs-api-gateway/src/gateway.ts @@ -10,6 +10,11 @@ import { getRealType, QueryAlias, } from '@cubejs-backend/shared'; +import { + ResultArrayWrapper, + ResultMultiWrapper, + ResultWrapper, +} from '@cubejs-backend/native'; import type { Application as ExpressApplication, ErrorRequestHandler, @@ -82,7 +87,6 @@ import { createJWKsFetcher } from './jwk'; import { SQLServer, SQLServerConstructorOptions } from './sql-server'; import { getJsonQueryFromGraphQLQuery, makeSchema } from './graphql'; import { ConfigItem, prepareAnnotation } from './helpers/prepareAnnotation'; -import transformData from './helpers/transformData'; import { transformCube, transformMeasure, @@ -1541,7 +1545,7 @@ class ApiGateway { context: RequestContext, normalizedQuery: NormalizedQuery, sqlQuery: any, - ) { + ): Promise { const queries = [{ ...sqlQuery, query: sqlQuery.sql[0], @@ -1586,15 +1590,28 @@ class ApiGateway { response.total = normalizedQuery.total ? Number(total.data[0][QueryAlias.TOTAL_COUNT]) : undefined; - return response; + + return this.wrapAdapterQueryResultIfNeeded(response); + } + + /** + * Wraps the adapter's response in unified ResultWrapper if it comes from + * a common driver (not a Cubestore's one, cause Cubestore Driver internally creates ResultWrapper) + * @param res Adapter's response + * @private + */ + private wrapAdapterQueryResultIfNeeded(res: any): ResultWrapper { + res.data = new ResultWrapper(res.data); + + return res; } /** - * Convert adapter's result and other request paramters to a final + * Prepare adapter's result and other transform parameters for a final * result object. * @internal */ - private getResultInternal( + private prepareResultTransformData( context: RequestContext, queryType: QueryType, normalizedQuery: NormalizedQuery, @@ -1615,21 +1632,23 @@ class ApiGateway { }, response: any, responseType?: ResultType, - ) { - return { + ): ResultWrapper { + const resultWrapper = response.data; + + const transformDataParams = { + aliasToMemberNameMap: sqlQuery.aliasNameToMember, + annotation: { + ...annotation.measures, + ...annotation.dimensions, + ...annotation.timeDimensions + } as { [member: string]: ConfigItem }, + query: normalizedQuery, + queryType, + resType: responseType, + }; + + const resObj = { query: normalizedQuery, - data: transformData( - sqlQuery.aliasNameToMember, - { - ...annotation.measures, - ...annotation.dimensions, - ...annotation.timeDimensions - } as { [member: string]: ConfigItem }, - response.data, - normalizedQuery, - queryType, - responseType, - ), lastRefreshTime: response.lastRefreshTime?.toISOString(), ...( getEnv('devMode') || @@ -1650,6 +1669,11 @@ class ApiGateway { slowQuery: Boolean(response.slowQuery), total: normalizedQuery.total ? response.total : null, }; + + resultWrapper.setTransformData(transformDataParams); + resultWrapper.setRootResultObject(resObj); + + return resultWrapper; } /** @@ -1733,6 +1757,17 @@ class ApiGateway { const [queryType, normalizedQueries] = await this.getNormalizedQueries(query, context); + if ( + queryType !== QueryTypeEnum.REGULAR_QUERY && + props.queryType == null + ) { + throw new UserError( + `'${queryType + }' query type is not supported by the client.` + + 'Please update the client.' + ); + } + let metaConfigResult = await (await this .getCompilerApi(context)).metaConfig(request.context, { requestId: context.requestId @@ -1749,17 +1784,17 @@ class ApiGateway { slowQuery = slowQuery || Boolean(sqlQueries[index].slowQuery); - const annotation = prepareAnnotation( - metaConfigResult, normalizedQuery - ); - const response = await this.getSqlResponseInternal( context, normalizedQuery, sqlQueries[index], ); - return this.getResultInternal( + const annotation = prepareAnnotation( + metaConfigResult, normalizedQuery + ); + + return this.prepareResultTransformData( context, queryType, normalizedQuery, @@ -1783,37 +1818,24 @@ class ApiGateway { queries: results.length, queriesWithPreAggregations: results.filter( - (r: any) => Object.keys( - r.usedPreAggregations || {} - ).length + (r: any) => Object.keys(r.getRootResultObject()[0].usedPreAggregations || {}).length ).length, - queriesWithData: - results.filter((r: any) => r.data?.length).length, - dbType: results.map(r => r.dbType), + // Have to omit because data could be processed natively + // so it is not known at this point + // queriesWithData: + // results.filter((r: any) => r.data?.length).length, + dbType: results.map(r => r.getRootResultObject()[0].dbType), }, context, ); - if ( - queryType !== QueryTypeEnum.REGULAR_QUERY && - props.queryType == null - ) { - throw new UserError( - `'${queryType - }' query type is not supported by the client.` + - 'Please update the client.' - ); - } - if (props.queryType === 'multi') { - res({ - queryType, - results, - pivotQuery: getPivotQuery(queryType, normalizedQueries), - slowQuery - }); + // We prepare the final json result on native side + const resultMulti = new ResultMultiWrapper(results, { queryType, slowQuery }); + await res(resultMulti); } else { - res(results[0]); + // We prepare the full final json result on native side + await res(results[0]); } } catch (e: any) { this.handleError({ @@ -1909,6 +1931,8 @@ class ApiGateway { annotation }]; } + + await res(request.streaming ? results[0] : { results }); } else { results = await Promise.all( normalizedQueries.map(async (normalizedQuery, index) => { @@ -1929,7 +1953,7 @@ class ApiGateway { sqlQueries[index], ); - return this.getResultInternal( + return this.prepareResultTransformData( context, queryType, normalizedQuery, @@ -1940,11 +1964,15 @@ class ApiGateway { ); }) ); - } - res(request.streaming ? results[0] : { - results, - }); + if (request.streaming) { + await res(results[0]); + } else { + // We prepare the final json result on native side + const resultArray = new ResultArrayWrapper(results); + await res(resultArray); + } + } } catch (e: any) { this.handleError({ e, context, query, res, requestStarted @@ -1990,7 +2018,7 @@ class ApiGateway { query, context, res: (message, opts) => { - if (!Array.isArray(message) && message.error) { + if (!Array.isArray(message) && 'error' in message && message.error) { error = { message, opts }; } else { result = { message, opts }; @@ -2014,7 +2042,18 @@ class ApiGateway { } protected resToResultFn(res: ExpressResponse) { - return (message, { status }: { status?: number } = {}) => (status ? res.status(status).json(message) : res.json(message)); + return async (message, { status }: { status?: number } = {}) => { + if (status) { + res.status(status); + } + + if (message.isWrapper) { + res.set('Content-Type', 'application/json'); + res.send(Buffer.from(await message.getFinalResult())); + } else { + res.json(message); + } + }; } protected parseQueryParam(query): Query | Query[] { diff --git a/packages/cubejs-api-gateway/src/helpers/prepareAnnotation.ts b/packages/cubejs-api-gateway/src/helpers/prepareAnnotation.ts index 268e354279e0e..db0385e7248d4 100644 --- a/packages/cubejs-api-gateway/src/helpers/prepareAnnotation.ts +++ b/packages/cubejs-api-gateway/src/helpers/prepareAnnotation.ts @@ -154,6 +154,7 @@ function prepareAnnotation(metaConfig: MetaConfig[], query: any) { export default prepareAnnotation; export { ConfigItem, + GranularityMeta, annotation, prepareAnnotation, }; diff --git a/packages/cubejs-api-gateway/src/helpers/toConfigMap.ts b/packages/cubejs-api-gateway/src/helpers/toConfigMap.ts index e95cb8aba0786..5766d0763a847 100644 --- a/packages/cubejs-api-gateway/src/helpers/toConfigMap.ts +++ b/packages/cubejs-api-gateway/src/helpers/toConfigMap.ts @@ -28,7 +28,7 @@ type MetaConfigMap = { }; /** - * Convert cpecified array of MetaConfig objects to the + * Convert specified array of MetaConfig objects to the * MetaConfigMap. */ function toConfigMap(metaConfig: MetaConfig[]): MetaConfigMap { diff --git a/packages/cubejs-api-gateway/src/helpers/transformData.ts b/packages/cubejs-api-gateway/src/helpers/transformData.ts deleted file mode 100644 index a9c4f4b9a294e..0000000000000 --- a/packages/cubejs-api-gateway/src/helpers/transformData.ts +++ /dev/null @@ -1,373 +0,0 @@ -/** - * @license Apache-2.0 - * @copyright Cube Dev, Inc. - * @fileoverview - * transformData function and related types definition. - */ - -import R from 'ramda'; -import { UserError } from '../UserError'; -import { ConfigItem } from './prepareAnnotation'; -import { - DBResponsePrimitive, - DBResponseValue, - transformValue, -} from './transformValue'; -import { - NormalizedQuery, - QueryTimeDimension -} from '../types/query'; -import { - ResultType, - QueryType, -} from '../types/strings'; -import { - ResultType as ResultTypeEnum, - QueryType as QueryTypeEnum, -} from '../types/enums'; - -const COMPARE_DATE_RANGE_FIELD = 'compareDateRange'; -const COMPARE_DATE_RANGE_SEPARATOR = ' - '; -const BLENDING_QUERY_KEY_PREFIX = 'time.'; -const BLENDING_QUERY_RES_SEPARATOR = '.'; -const MEMBER_SEPARATOR = '.'; - -/** - * SQL aliases to cube properties hash map. - */ -type AliasToMemberMap = { [alias: string]: string }; - -/** - * Parse date range value from time dimension. - * @internal - */ -function getDateRangeValue( - timeDimensions?: QueryTimeDimension[] -): string { - if (!timeDimensions) { - throw new UserError( - 'QueryTimeDimension should be specified ' + - 'for the compare date range query.' - ); - } else { - const [dim] = timeDimensions; - if (!dim.dateRange) { - throw new UserError( - `${'Inconsistent QueryTimeDimension configuration ' + - 'for the compare date range query, dateRange required: '}${ - dim}` - ); - } else if (typeof dim.dateRange === 'string') { - throw new UserError( - 'Inconsistent dateRange configuration for the ' + - `compare date range query: ${dim.dateRange}` - ); - } else { - return dim.dateRange.join(COMPARE_DATE_RANGE_SEPARATOR); - } - } -} - -/** - * Parse blending query key from time time dimension. - * @internal - */ -function getBlendingQueryKey( - timeDimensions?: QueryTimeDimension[] -): string { - if (!timeDimensions) { - throw new UserError( - 'QueryTimeDimension should be specified ' + - 'for the blending query.' - ); - } else { - const [dim] = timeDimensions; - if (!dim.granularity) { - throw new UserError( - 'Inconsistent QueryTimeDimension configuration ' + - `for the blending query, granularity required: ${dim}` - ); - } else { - return BLENDING_QUERY_KEY_PREFIX + dim.granularity; - } - } -} - -/** - * Parse blending response key from time time dimension. - * @internal - */ -function getBlendingResponseKey( - timeDimensions?: QueryTimeDimension[] -): string { - if (!timeDimensions) { - throw new UserError( - 'QueryTimeDimension should be specified ' + - 'for the blending query.' - ); - } else { - const [dim] = timeDimensions; - if (!dim.granularity) { - throw new UserError( - 'Inconsistent QueryTimeDimension configuration ' + - `for the blending query, granularity required: ${dim}` - ); - } else if (!dim.dimension) { - throw new UserError( - 'Inconsistent QueryTimeDimension configuration ' + - `for the blending query, dimension required: ${dim}` - ); - } else { - return dim.dimension + - BLENDING_QUERY_RES_SEPARATOR + - dim.granularity; - } - } -} - -/** - * Parse members names from request/response. - * @internal - */ -function getMembers( - queryType: QueryType, - query: NormalizedQuery, - dbData: { [sqlAlias: string]: DBResponseValue }[], - aliasToMemberNameMap: AliasToMemberMap, - annotation: { [member: string]: ConfigItem }, -): { [member: string]: string } { - const members: { [member: string]: string } = {}; - if (!dbData.length) { - return members; - } - const columns = Object.keys(dbData[0]); - columns.forEach((column) => { - if (!aliasToMemberNameMap[column] || !annotation[aliasToMemberNameMap[column]]) { - throw new UserError( - `You requested hidden member: '${ - column - }'. Please make it visible using \`shown: true\`. ` + - 'Please note primaryKey fields are `shown: false` by ' + - 'default: https://cube.dev/docs/schema/reference/joins#' + - 'setting-a-primary-key.' - ); - } - members[aliasToMemberNameMap[column]] = column; - const path = aliasToMemberNameMap[column] - .split(MEMBER_SEPARATOR); - const calcMember = - [path[0], path[1]].join(MEMBER_SEPARATOR); - if ( - path.length === 3 && - query.dimensions?.indexOf(calcMember) === -1 - ) { - members[calcMember] = column; - } - }); - if (queryType === QueryTypeEnum.COMPARE_DATE_RANGE_QUERY) { - members[COMPARE_DATE_RANGE_FIELD] = - QueryTypeEnum.COMPARE_DATE_RANGE_QUERY; - } else if (queryType === QueryTypeEnum.BLENDING_QUERY) { - members[getBlendingQueryKey(query.timeDimensions)] = - // @ts-ignore - members[query.timeDimensions[0].dimension]; - } - return members; -} - -/** - * Convert DB response object to the compact output format. - * @internal - * @todo should we use transformValue for blending query? - */ -function getCompactRow( - membersToAliasMap: { [member: string]: string }, - annotation: { [member: string]: ConfigItem }, - queryType: QueryType, - members: string[], - timeDimensions: QueryTimeDimension[] | undefined, - dbRow: { [sqlAlias: string]: DBResponseValue }, -): DBResponsePrimitive[] { - const row: DBResponsePrimitive[] = []; - members.forEach((m: string) => { - if (annotation[m]) { - row.push( - transformValue( - dbRow[membersToAliasMap[m]], - annotation[m].type - ), - ); - } - }); - if (queryType === QueryTypeEnum.COMPARE_DATE_RANGE_QUERY) { - row.push( - getDateRangeValue(timeDimensions) - ); - } else if (queryType === QueryTypeEnum.BLENDING_QUERY) { - row.push( - dbRow[ - membersToAliasMap[ - getBlendingResponseKey(timeDimensions) - ] - ] as DBResponsePrimitive - ); - } - return row; -} - -/** - * Convert DB response object to the vanila output format. - * @todo rewrite me please! - * @internal - */ -function getVanilaRow( - aliasToMemberNameMap: AliasToMemberMap, - annotation: { [member: string]: ConfigItem }, - queryType: QueryType, - query: NormalizedQuery, - dbRow: { [sqlAlias: string]: DBResponseValue }, -): { [member: string]: DBResponsePrimitive } { - const row = R.pipe( - R.toPairs, - R.map(p => { - const memberName = aliasToMemberNameMap[p[0]]; - const annotationForMember = annotation[memberName]; - if (!annotationForMember) { - throw new UserError( - `You requested hidden member: '${ - p[0] - }'. Please make it visible using \`shown: true\`. ` + - 'Please note primaryKey fields are `shown: false` by ' + - 'default: https://cube.dev/docs/schema/reference/joins#' + - 'setting-a-primary-key.' - ); - } - const transformResult = [ - memberName, - transformValue( - p[1] as DBResponseValue, - annotationForMember.type - ) - ]; - const path = memberName.split(MEMBER_SEPARATOR); - - /** - * Time dimensions without granularity. - * @deprecated - * @todo backward compatibility for referencing - */ - const memberNameWithoutGranularity = - [path[0], path[1]].join(MEMBER_SEPARATOR); - if ( - path.length === 3 && - (query.dimensions || []) - .indexOf(memberNameWithoutGranularity) === -1 - ) { - return [ - transformResult, - [ - memberNameWithoutGranularity, - transformResult[1] - ] - ]; - } - - return [transformResult]; - }), - // @ts-ignore - R.unnest, - R.fromPairs - // @ts-ignore - )(dbRow); - if (queryType === QueryTypeEnum.COMPARE_DATE_RANGE_QUERY) { - return { - ...row, - compareDateRange: getDateRangeValue(query.timeDimensions) - }; - } else if (queryType === QueryTypeEnum.BLENDING_QUERY) { - return { - ...row, - [getBlendingQueryKey(query.timeDimensions)]: - row[getBlendingResponseKey(query.timeDimensions)] - }; - } - return row as { [member: string]: DBResponsePrimitive; }; -} - -/** - * Transforms queried data array to the output format. - */ -function transformData( - aliasToMemberNameMap: AliasToMemberMap, - annotation: { [member: string]: ConfigItem }, - data: { [sqlAlias: string]: unknown }[], - query: NormalizedQuery, - queryType: QueryType, - resType?: ResultType -): { - members: string[], - dataset: DBResponsePrimitive[][] -} | { - [member: string]: DBResponsePrimitive -}[] { - const d = data as { [sqlAlias: string]: DBResponseValue }[]; - const membersToAliasMap = getMembers( - queryType, - query, - d, - aliasToMemberNameMap, - annotation, - ); - const members: string[] = Object.keys(membersToAliasMap); - const dataset: DBResponsePrimitive[][] | { - [member: string]: DBResponsePrimitive - }[] = d.map((r) => { - const row: DBResponsePrimitive[] | { - [member: string]: DBResponsePrimitive - } = resType === ResultTypeEnum.COMPACT - ? getCompactRow( - membersToAliasMap, - annotation, - queryType, - members, - query.timeDimensions, - r, - ) - : getVanilaRow( - aliasToMemberNameMap, - annotation, - queryType, - query, - r, - ); - return row; - }) as DBResponsePrimitive[][] | { - [member: string]: DBResponsePrimitive - }[]; - return (resType === ResultTypeEnum.COMPACT - ? { members, dataset } - : dataset - ) as { - members: string[], - dataset: DBResponsePrimitive[][] - } | { - [member: string]: DBResponsePrimitive - }[]; -} - -export default transformData; -export { - AliasToMemberMap, - COMPARE_DATE_RANGE_FIELD, - COMPARE_DATE_RANGE_SEPARATOR, - BLENDING_QUERY_KEY_PREFIX, - BLENDING_QUERY_RES_SEPARATOR, - MEMBER_SEPARATOR, - getDateRangeValue, - getBlendingQueryKey, - getBlendingResponseKey, - getMembers, - getCompactRow, - getVanilaRow, - transformData, -}; diff --git a/packages/cubejs-api-gateway/src/helpers/transformValue.ts b/packages/cubejs-api-gateway/src/helpers/transformValue.ts deleted file mode 100644 index 78d80f16f15bc..0000000000000 --- a/packages/cubejs-api-gateway/src/helpers/transformValue.ts +++ /dev/null @@ -1,48 +0,0 @@ -/** - * @license Apache-2.0 - * @copyright Cube Dev, Inc. - * @fileoverview - * transformValue function and related types definition. - */ - -import moment, { MomentInput } from 'moment'; - -/** - * Query 'or'-filters type definition. - */ -type DBResponsePrimitive = - null | - boolean | - number | - string; - -type DBResponseValue = - Date | - DBResponsePrimitive | - { value: DBResponsePrimitive }; - -/** - * Transform cpecified `value` with specified `type` to the network - * protocol type. - */ -function transformValue( - value: DBResponseValue, - type: string -): DBResponsePrimitive { - // TODO: support for max time - if (value && (type === 'time' || value instanceof Date)) { - return ( - value instanceof Date - ? moment(value) - : moment.utc(value as MomentInput) - ).format(moment.HTML5_FMT.DATETIME_LOCAL_MS); - } - return value as DBResponsePrimitive; -} - -export default transformValue; -export { - DBResponsePrimitive, - DBResponseValue, - transformValue, -}; diff --git a/packages/cubejs-api-gateway/src/index.ts b/packages/cubejs-api-gateway/src/index.ts index ced6087014c9c..a990e663bf6e8 100644 --- a/packages/cubejs-api-gateway/src/index.ts +++ b/packages/cubejs-api-gateway/src/index.ts @@ -4,3 +4,4 @@ export * from './interfaces'; export * from './CubejsHandlerError'; export * from './UserError'; export { getRequestIdFromRequest } from './requestParser'; +export { TransformDataRequest } from './types/responses'; diff --git a/packages/cubejs-api-gateway/src/interfaces.ts b/packages/cubejs-api-gateway/src/interfaces.ts index aad8f106e3e85..681fa4507228c 100644 --- a/packages/cubejs-api-gateway/src/interfaces.ts +++ b/packages/cubejs-api-gateway/src/interfaces.ts @@ -10,6 +10,7 @@ import { import { QueryType, + ResultType, } from './types/enums'; import { @@ -43,9 +44,24 @@ import { QueryRequest } from './types/request'; +import { + AliasToMemberMap, + TransformDataResponse +} from './types/responses'; + +import { + ConfigItem, + GranularityMeta +} from './helpers/prepareAnnotation'; + export { + AliasToMemberMap, + TransformDataResponse, + ConfigItem, + GranularityMeta, QueryTimeDimensionGranularity, QueryType, + ResultType, QueryFilter, LogicalAndFilter, LogicalOrFilter, diff --git a/packages/cubejs-api-gateway/src/types/request.ts b/packages/cubejs-api-gateway/src/types/request.ts index ed6c5bbd80174..3393554fd8d65 100644 --- a/packages/cubejs-api-gateway/src/types/request.ts +++ b/packages/cubejs-api-gateway/src/types/request.ts @@ -6,6 +6,7 @@ */ import type { Request as ExpressRequest } from 'express'; +import type { DataResult } from '@cubejs-backend/native'; import { RequestType, ApiType, ResultType } from './strings'; import { Query } from './query'; @@ -105,7 +106,7 @@ type MetaResponseResultFn = (message: MetaResponse | ErrorResponse) => void; */ type ResponseResultFn = ( - message: (Record | Record[]) | ErrorResponse, + message: (Record | Record[]) | DataResult | ErrorResponse, extra?: { status: number } ) => void; diff --git a/packages/cubejs-api-gateway/src/types/responses.ts b/packages/cubejs-api-gateway/src/types/responses.ts new file mode 100644 index 0000000000000..d5e1d8ea1e0aa --- /dev/null +++ b/packages/cubejs-api-gateway/src/types/responses.ts @@ -0,0 +1,35 @@ +import type { ConfigItem } from '../helpers/prepareAnnotation'; +import type { NormalizedQuery } from './query'; +import type { QueryType, ResultType } from './strings'; + +export type DBResponsePrimitive = + null | + boolean | + number | + string; + +export type DBResponseValue = + Date | + DBResponsePrimitive | + { value: DBResponsePrimitive }; + +export type TransformDataResponse = { + members: string[], + dataset: DBResponsePrimitive[][] +} | { + [member: string]: DBResponsePrimitive +}[]; + +/** + * SQL aliases to cube properties hash map. + */ +export type AliasToMemberMap = { [alias: string]: string }; + +export type TransformDataRequest = { + aliasToMemberNameMap: { [alias: string]: string }, + annotation: { [member: string]: ConfigItem }, + data: { [sqlAlias: string]: unknown }[], + query: NormalizedQuery, + queryType: QueryType, + resType?: ResultType +}; diff --git a/packages/cubejs-api-gateway/test/helpers/transformData.test.ts b/packages/cubejs-api-gateway/test/helpers/transformData.test.ts deleted file mode 100644 index 8936bd3d44418..0000000000000 --- a/packages/cubejs-api-gateway/test/helpers/transformData.test.ts +++ /dev/null @@ -1,1339 +0,0 @@ -/** - * @license Apache-2.0 - * @copyright Cube Dev, Inc. - * @fileoverview transformData related helpers unit tests. - */ - -/* globals describe,test,expect */ -/* eslint-disable import/no-duplicates */ -/* eslint-disable @typescript-eslint/no-duplicate-imports */ - -import { - QueryTimeDimension, - NormalizedQuery, -} from '../../src/types/query'; -import { - QueryType as QueryTypeEnum, - ResultType as ResultTypeEnum, -} from '../../src/types/enums'; -import { - DBResponseValue, -} from '../../src/helpers/transformValue'; -import { - ConfigItem, -} from '../../src/helpers/prepareAnnotation'; -import transformDataDefault - from '../../src/helpers/transformData'; -import { - COMPARE_DATE_RANGE_FIELD, - COMPARE_DATE_RANGE_SEPARATOR, - BLENDING_QUERY_KEY_PREFIX, - BLENDING_QUERY_RES_SEPARATOR, - MEMBER_SEPARATOR, - getDateRangeValue, - getBlendingQueryKey, - getBlendingResponseKey, - getMembers, - getCompactRow, - getVanilaRow, - transformData, -} from '../../src/helpers/transformData'; -import { - QueryType, -} from '../../src/types/strings'; - -const mockData = { - regular_discount_by_city: { - query: { - dimensions: [ - 'ECommerceRecordsUs2021.city' - ], - measures: [ - 'ECommerceRecordsUs2021.avg_discount' - ], - limit: 2 - }, - data: { - aliasToMemberNameMap: { - e_commerce_records_us2021__avg_discount: 'ECommerceRecordsUs2021.avg_discount', - e_commerce_records_us2021__city: 'ECommerceRecordsUs2021.city' - }, - annotation: { - 'ECommerceRecordsUs2021.avg_discount': { - title: 'E Commerce Records Us2021 Avg Discount', - shortTitle: 'Avg Discount', - type: 'number', - drillMembers: [], - drillMembersGrouped: { - measures: [], - dimensions: [] - } - }, - 'ECommerceRecordsUs2021.city': { - title: 'E Commerce Records Us2021 City', - shortTitle: 'City', - type: 'string' - } - }, - data: [ - { - e_commerce_records_us2021__city: 'Missouri City', - e_commerce_records_us2021__avg_discount: '0.80000000000000000000' - }, - { - e_commerce_records_us2021__city: 'Abilene', - e_commerce_records_us2021__avg_discount: '0.80000000000000000000' - } - ], - query: { - dimensions: [ - 'ECommerceRecordsUs2021.city' - ], - measures: [ - 'ECommerceRecordsUs2021.avg_discount' - ], - limit: 2, - rowLimit: 2, - timezone: 'UTC', - order: [], - filters: [], - timeDimensions: [] - }, - queryType: 'regularQuery', - result_default: [ - { - 'ECommerceRecordsUs2021.city': 'Missouri City', - 'ECommerceRecordsUs2021.avg_discount': '0.80000000000000000000' - }, - { - 'ECommerceRecordsUs2021.city': 'Abilene', - 'ECommerceRecordsUs2021.avg_discount': '0.80000000000000000000' - } - ], - result_compact: { - members: ['ECommerceRecordsUs2021.city', 'ECommerceRecordsUs2021.avg_discount'], - dataset: [['Missouri City', '0.80000000000000000000'], ['Abilene', '0.80000000000000000000']], - } - } - }, - regular_profit_by_postal_code: { - query: { - dimensions: [ - 'ECommerceRecordsUs2021.postalCode' - ], - measures: [ - 'ECommerceRecordsUs2021.avg_profit' - ], - limit: 2 - }, - data: { - aliasToMemberNameMap: { - e_commerce_records_us2021__avg_profit: 'ECommerceRecordsUs2021.avg_profit', - e_commerce_records_us2021__postal_code: 'ECommerceRecordsUs2021.postalCode' - }, - annotation: { - 'ECommerceRecordsUs2021.avg_profit': { - title: 'E Commerce Records Us2021 Avg Profit', - shortTitle: 'Avg Profit', - type: 'number', - drillMembers: [], - drillMembersGrouped: { - measures: [], - dimensions: [] - } - }, - 'ECommerceRecordsUs2021.postalCode': { - title: 'E Commerce Records Us2021 Postal Code', - shortTitle: 'Postal Code', - type: 'string' - } - }, - data: [ - { - e_commerce_records_us2021__postal_code: '95823', - e_commerce_records_us2021__avg_profit: '646.1258666666666667' - }, - { - e_commerce_records_us2021__postal_code: '64055', - e_commerce_records_us2021__avg_profit: '487.8315000000000000' - } - ], - query: { - dimensions: [ - 'ECommerceRecordsUs2021.postalCode' - ], - measures: [ - 'ECommerceRecordsUs2021.avg_profit' - ], - limit: 2, - rowLimit: 2, - timezone: 'UTC', - order: [], - filters: [], - timeDimensions: [] - }, - queryType: 'regularQuery', - result_default: [ - { - 'ECommerceRecordsUs2021.postalCode': '95823', - 'ECommerceRecordsUs2021.avg_profit': '646.1258666666666667' - }, - { - 'ECommerceRecordsUs2021.postalCode': '64055', - 'ECommerceRecordsUs2021.avg_profit': '487.8315000000000000' - } - ], - result_compact: { - members: [ - 'ECommerceRecordsUs2021.postalCode', - 'ECommerceRecordsUs2021.avg_profit', - ], - dataset: [ - ['95823', '646.1258666666666667'], - ['64055', '487.8315000000000000'] - ], - } - } - }, - compare_date_range_count_by_order_date: { - http_params: { - queryType: 'whatever value or nothing, \'multi\' to apply pivot transformation' - }, - query: { - measures: ['ECommerceRecordsUs2021.count'], - timeDimensions: [ - { - dimension: 'ECommerceRecordsUs2021.orderDate', - granularity: 'day', - compareDateRange: [ - ['2020-01-01', '2020-01-31'], - ['2020-03-01', '2020-03-31'] - ] - } - ], - limit: 2 - }, - data: [{ - aliasToMemberNameMap: { - e_commerce_records_us2021__count: 'ECommerceRecordsUs2021.count', - e_commerce_records_us2021__order_date_day: 'ECommerceRecordsUs2021.orderDate.day' - }, - annotation: { - 'ECommerceRecordsUs2021.count': { - title: 'E Commerce Records Us2021 Count', - shortTitle: 'Count', - type: 'number', - drillMembers: [ - 'ECommerceRecordsUs2021.city', - 'ECommerceRecordsUs2021.country', - 'ECommerceRecordsUs2021.customerId', - 'ECommerceRecordsUs2021.orderId', - 'ECommerceRecordsUs2021.productId', - 'ECommerceRecordsUs2021.productName', - 'ECommerceRecordsUs2021.orderDate' - ], - drillMembersGrouped: { - measures: [], - dimensions: [ - 'ECommerceRecordsUs2021.city', - 'ECommerceRecordsUs2021.country', - 'ECommerceRecordsUs2021.customerId', - 'ECommerceRecordsUs2021.orderId', - 'ECommerceRecordsUs2021.productId', - 'ECommerceRecordsUs2021.productName', - 'ECommerceRecordsUs2021.orderDate' - ] - } - }, - 'ECommerceRecordsUs2021.orderDate.day': { - title: 'E Commerce Records Us2021 Order Date', - shortTitle: 'Order Date', - type: 'time' - }, - 'ECommerceRecordsUs2021.orderDate': { - title: 'E Commerce Records Us2021 Order Date', - shortTitle: 'Order Date', - type: 'time' - } - }, - data: [ - { - e_commerce_records_us2021__order_date_day: '2020-01-01T00:00:00.000', - e_commerce_records_us2021__count: '10' - }, - { - e_commerce_records_us2021__order_date_day: '2020-01-02T00:00:00.000', - e_commerce_records_us2021__count: '8' - } - ], - query: { - measures: [ - 'ECommerceRecordsUs2021.count' - ], - timeDimensions: [ - { - dimension: 'ECommerceRecordsUs2021.orderDate', - granularity: 'day', - dateRange: [ - '2020-01-01T00:00:00.000', - '2020-01-31T23:59:59.999' - ] - } - ], - limit: 2, - rowLimit: 2, - timezone: 'UTC', - order: [], - filters: [], - dimensions: [] - }, - queryType: 'compareDateRangeQuery', - result_default: [ - { - 'ECommerceRecordsUs2021.orderDate.day': '2020-01-01T00:00:00.000', - 'ECommerceRecordsUs2021.orderDate': '2020-01-01T00:00:00.000', - 'ECommerceRecordsUs2021.count': '10', - compareDateRange: '2020-01-01T00:00:00.000 - 2020-01-31T23:59:59.999' - }, - { - 'ECommerceRecordsUs2021.orderDate.day': '2020-01-02T00:00:00.000', - 'ECommerceRecordsUs2021.orderDate': '2020-01-02T00:00:00.000', - 'ECommerceRecordsUs2021.count': '8', - compareDateRange: '2020-01-01T00:00:00.000 - 2020-01-31T23:59:59.999' - } - ], - result_compact: { - members: [ - 'ECommerceRecordsUs2021.orderDate.day', - 'ECommerceRecordsUs2021.orderDate', - 'ECommerceRecordsUs2021.count', - 'compareDateRange', - ], - dataset: [ - [ - '2020-01-01T00:00:00.000', - '2020-01-01T00:00:00.000', - '10', - '2020-01-01T00:00:00.000 - 2020-01-31T23:59:59.999', - ], - [ - '2020-01-02T00:00:00.000', - '2020-01-02T00:00:00.000', - '8', - '2020-01-01T00:00:00.000 - 2020-01-31T23:59:59.999' - ], - ], - }, - }, { - aliasToMemberNameMap: { - e_commerce_records_us2021__count: 'ECommerceRecordsUs2021.count', - e_commerce_records_us2021__order_date_day: 'ECommerceRecordsUs2021.orderDate.day' - }, - annotation: { - 'ECommerceRecordsUs2021.count': { - title: 'E Commerce Records Us2021 Count', - shortTitle: 'Count', - type: 'number', - drillMembers: [ - 'ECommerceRecordsUs2021.city', - 'ECommerceRecordsUs2021.country', - 'ECommerceRecordsUs2021.customerId', - 'ECommerceRecordsUs2021.orderId', - 'ECommerceRecordsUs2021.productId', - 'ECommerceRecordsUs2021.productName', - 'ECommerceRecordsUs2021.orderDate' - ], - drillMembersGrouped: { - measures: [], - dimensions: [ - 'ECommerceRecordsUs2021.city', - 'ECommerceRecordsUs2021.country', - 'ECommerceRecordsUs2021.customerId', - 'ECommerceRecordsUs2021.orderId', - 'ECommerceRecordsUs2021.productId', - 'ECommerceRecordsUs2021.productName', - 'ECommerceRecordsUs2021.orderDate' - ] - } - }, - 'ECommerceRecordsUs2021.orderDate.day': { - title: 'E Commerce Records Us2021 Order Date', - shortTitle: 'Order Date', - type: 'time' - }, - 'ECommerceRecordsUs2021.orderDate': { - title: 'E Commerce Records Us2021 Order Date', - shortTitle: 'Order Date', - type: 'time' - } - }, - data: [ - { - e_commerce_records_us2021__order_date_day: '2020-03-02T00:00:00.000', - e_commerce_records_us2021__count: '11' - }, - { - e_commerce_records_us2021__order_date_day: '2020-03-03T00:00:00.000', - e_commerce_records_us2021__count: '7' - } - ], - query: { - measures: [ - 'ECommerceRecordsUs2021.count' - ], - timeDimensions: [ - { - dimension: 'ECommerceRecordsUs2021.orderDate', - granularity: 'day', - dateRange: [ - '2020-03-01T00:00:00.000', - '2020-03-31T23:59:59.999' - ] - } - ], - limit: 2, - rowLimit: 2, - timezone: 'UTC', - order: [], - filters: [], - dimensions: [] - }, - queryType: 'compareDateRangeQuery', - result_default: [ - { - 'ECommerceRecordsUs2021.orderDate.day': '2020-03-02T00:00:00.000', - 'ECommerceRecordsUs2021.orderDate': '2020-03-02T00:00:00.000', - 'ECommerceRecordsUs2021.count': '11', - compareDateRange: '2020-03-01T00:00:00.000 - 2020-03-31T23:59:59.999' - }, - { - 'ECommerceRecordsUs2021.orderDate.day': '2020-03-03T00:00:00.000', - 'ECommerceRecordsUs2021.orderDate': '2020-03-03T00:00:00.000', - 'ECommerceRecordsUs2021.count': '7', - compareDateRange: '2020-03-01T00:00:00.000 - 2020-03-31T23:59:59.999' - } - ], - result_compact: { - members: [ - 'ECommerceRecordsUs2021.orderDate.day', - 'ECommerceRecordsUs2021.orderDate', - 'ECommerceRecordsUs2021.count', - 'compareDateRange', - ], - dataset: [ - [ - '2020-03-02T00:00:00.000', - '2020-03-02T00:00:00.000', - '11', - '2020-03-01T00:00:00.000 - 2020-03-31T23:59:59.999', - ], - [ - '2020-03-03T00:00:00.000', - '2020-03-03T00:00:00.000', - '7', - '2020-03-01T00:00:00.000 - 2020-03-31T23:59:59.999' - ], - ], - }, - }] - }, - blending_query_avg_discount_by_date_range_for_the_first_and_standard_ship_mode: { - http_params: { - queryType: 'whatever value or nothing, \'multi\' to apply pivot transformation' - }, - query: [{ - measures: ['ECommerceRecordsUs2021.avg_discount'], - timeDimensions: [ - { - dimension: 'ECommerceRecordsUs2021.orderDate', - granularity: 'month', - dateRange: ['2020-01-01', '2020-12-30'] - } - ], - filters: [{ - member: 'ECommerceRecordsUs2021.shipMode', - operator: 'equals', - values: ['Standard Class'] - }], - limit: 2 - }, { - measures: ['ECommerceRecordsUs2021.avg_discount'], - timeDimensions: [ - { - dimension: 'ECommerceRecordsUs2021.orderDate', - granularity: 'month', - dateRange: ['2020-01-01', '2020-12-30'] - } - ], - filters: [{ - member: 'ECommerceRecordsUs2021.shipMode', - operator: 'equals', - values: ['First Class'] - }], - limit: 2 - }], - data: [{ - aliasToMemberNameMap: { - e_commerce_records_us2021__avg_discount: 'ECommerceRecordsUs2021.avg_discount', - e_commerce_records_us2021__order_date_month: 'ECommerceRecordsUs2021.orderDate.month' - }, - annotation: { - 'ECommerceRecordsUs2021.avg_discount': { - title: 'E Commerce Records Us2021 Avg Discount', - shortTitle: 'Avg Discount', - type: 'number', - drillMembers: [], - drillMembersGrouped: { - measures: [], - dimensions: [] - } - }, - 'ECommerceRecordsUs2021.orderDate.month': { - title: 'E Commerce Records Us2021 Order Date', - shortTitle: 'Order Date', - type: 'time' - }, - 'ECommerceRecordsUs2021.orderDate': { - title: 'E Commerce Records Us2021 Order Date', - shortTitle: 'Order Date', - type: 'time' - } - }, - data: [ - { - e_commerce_records_us2021__order_date_month: '2020-01-01T00:00:00.000', - e_commerce_records_us2021__avg_discount: '0.15638297872340425532' - }, - { - e_commerce_records_us2021__order_date_month: '2020-02-01T00:00:00.000', - e_commerce_records_us2021__avg_discount: '0.17573529411764705882' - } - ], - query: { - measures: [ - 'ECommerceRecordsUs2021.avg_discount' - ], - timeDimensions: [ - { - dimension: 'ECommerceRecordsUs2021.orderDate', - granularity: 'month', - dateRange: [ - '2020-01-01T00:00:00.000', - '2020-12-30T23:59:59.999' - ] - } - ], - filters: [ - { - operator: 'equals', - values: [ - 'Standard Class' - ], - member: 'ECommerceRecordsUs2021.shipMode' - } - ], - limit: 2, - rowLimit: 2, - timezone: 'UTC', - order: [], - dimensions: [] - }, - queryType: 'blendingQuery', - result_default: [ - { - 'ECommerceRecordsUs2021.orderDate.month': '2020-01-01T00:00:00.000', - 'ECommerceRecordsUs2021.orderDate': '2020-01-01T00:00:00.000', - 'ECommerceRecordsUs2021.avg_discount': '0.15638297872340425532', - 'time.month': '2020-01-01T00:00:00.000' - }, - { - 'ECommerceRecordsUs2021.orderDate.month': '2020-02-01T00:00:00.000', - 'ECommerceRecordsUs2021.orderDate': '2020-02-01T00:00:00.000', - 'ECommerceRecordsUs2021.avg_discount': '0.17573529411764705882', - 'time.month': '2020-02-01T00:00:00.000' - } - ], - result_compact: { - members: [ - 'ECommerceRecordsUs2021.orderDate.month', - 'ECommerceRecordsUs2021.orderDate', - 'ECommerceRecordsUs2021.avg_discount', - 'time.month', - ], - dataset: [ - [ - '2020-01-01T00:00:00.000', - '2020-01-01T00:00:00.000', - '0.15638297872340425532', - '2020-01-01T00:00:00.000', - ], - [ - '2020-02-01T00:00:00.000', - '2020-02-01T00:00:00.000', - '0.17573529411764705882', - '2020-02-01T00:00:00.000', - ], - ], - }, - }, { - aliasToMemberNameMap: { - e_commerce_records_us2021__avg_discount: 'ECommerceRecordsUs2021.avg_discount', - e_commerce_records_us2021__order_date_month: 'ECommerceRecordsUs2021.orderDate.month' - }, - annotation: { - 'ECommerceRecordsUs2021.avg_discount': { - title: 'E Commerce Records Us2021 Avg Discount', - shortTitle: 'Avg Discount', - type: 'number', - drillMembers: [], - drillMembersGrouped: { - measures: [], - dimensions: [] - } - }, - 'ECommerceRecordsUs2021.orderDate.month': { - title: 'E Commerce Records Us2021 Order Date', - shortTitle: 'Order Date', - type: 'time' - }, - 'ECommerceRecordsUs2021.orderDate': { - title: 'E Commerce Records Us2021 Order Date', - shortTitle: 'Order Date', - type: 'time' - } - }, - data: [ - { - e_commerce_records_us2021__order_date_month: '2020-01-01T00:00:00.000', - e_commerce_records_us2021__avg_discount: '0.28571428571428571429' - }, - { - e_commerce_records_us2021__order_date_month: '2020-02-01T00:00:00.000', - e_commerce_records_us2021__avg_discount: '0.21777777777777777778' - } - ], - query: { - measures: [ - 'ECommerceRecordsUs2021.avg_discount' - ], - timeDimensions: [ - { - dimension: 'ECommerceRecordsUs2021.orderDate', - granularity: 'month', - dateRange: [ - '2020-01-01T00:00:00.000', - '2020-12-30T23:59:59.999' - ] - } - ], - filters: [ - { - operator: 'equals', - values: [ - 'First Class' - ], - member: 'ECommerceRecordsUs2021.shipMode' - } - ], - limit: 2, - rowLimit: 2, - timezone: 'UTC', - order: [], - dimensions: [] - }, - queryType: 'blendingQuery', - result_default: [{ - 'ECommerceRecordsUs2021.orderDate.month': '2020-01-01T00:00:00.000', - 'ECommerceRecordsUs2021.orderDate': '2020-01-01T00:00:00.000', - 'ECommerceRecordsUs2021.avg_discount': '0.28571428571428571429', - 'time.month': '2020-01-01T00:00:00.000' - }, - { - 'ECommerceRecordsUs2021.orderDate.month': '2020-02-01T00:00:00.000', - 'ECommerceRecordsUs2021.orderDate': '2020-02-01T00:00:00.000', - 'ECommerceRecordsUs2021.avg_discount': '0.21777777777777777778', - 'time.month': '2020-02-01T00:00:00.000' - }], - result_compact: { - members: [ - 'ECommerceRecordsUs2021.orderDate.month', - 'ECommerceRecordsUs2021.orderDate', - 'ECommerceRecordsUs2021.avg_discount', - 'time.month', - ], - dataset: [ - [ - '2020-01-01T00:00:00.000', - '2020-01-01T00:00:00.000', - '0.28571428571428571429', - '2020-01-01T00:00:00.000', - ], - [ - '2020-02-01T00:00:00.000', - '2020-02-01T00:00:00.000', - '0.21777777777777777778', - '2020-02-01T00:00:00.000', - ], - ], - }, - }] - } -}; - -describe('transformData helpers', () => { - test('export looks as expected', () => { - expect(transformDataDefault).toBeDefined(); - expect(COMPARE_DATE_RANGE_FIELD).toBeDefined(); - expect(COMPARE_DATE_RANGE_SEPARATOR).toBeDefined(); - expect(BLENDING_QUERY_KEY_PREFIX).toBeDefined(); - expect(BLENDING_QUERY_RES_SEPARATOR).toBeDefined(); - expect(MEMBER_SEPARATOR).toBeDefined(); - expect(getDateRangeValue).toBeDefined(); - expect(getBlendingQueryKey).toBeDefined(); - expect(getBlendingResponseKey).toBeDefined(); - expect(getMembers).toBeDefined(); - expect(getCompactRow).toBeDefined(); - expect(getVanilaRow).toBeDefined(); - expect(transformData).toBeDefined(); - expect(transformData).toEqual(transformDataDefault); - }); - - test('getDateRangeValue helper', () => { - const timeDimensions = JSON.parse( - JSON.stringify( - mockData - .blending_query_avg_discount_by_date_range_for_the_first_and_standard_ship_mode - .data[0] - .query - .timeDimensions - ) - ) as QueryTimeDimension[]; - - expect(() => { getDateRangeValue(); }).toThrow( - 'QueryTimeDimension should be specified ' + - 'for the compare date range query.' - ); - - expect(() => { - getDateRangeValue([ - { prop: 'val' } as unknown as QueryTimeDimension - ]); - }).toThrow( - `${'Inconsistent QueryTimeDimension configuration ' + - 'for the compare date range query, dateRange required: '}${ - ({ prop: 'val' }).toString()}` - ); - - expect(() => { - getDateRangeValue([ - { dateRange: 'val' } as unknown as QueryTimeDimension - ]); - }).toThrow( - 'Inconsistent dateRange configuration for the ' + - 'compare date range query: val' - ); - - expect(getDateRangeValue(timeDimensions)).toEqual( - `${ - // @ts-ignore - timeDimensions[0].dateRange[0] - }${ - COMPARE_DATE_RANGE_SEPARATOR - }${ - // @ts-ignore - timeDimensions[0].dateRange[1] - }` - ); - }); - - test('getBlendingQueryKey helper', () => { - const timeDimensions = JSON.parse( - JSON.stringify( - mockData - .blending_query_avg_discount_by_date_range_for_the_first_and_standard_ship_mode - .data[0] - .query - .timeDimensions - ) - ) as QueryTimeDimension[]; - - expect(() => { - getBlendingQueryKey(); - }).toThrow( - 'QueryTimeDimension should be specified ' + - 'for the blending query.' - ); - - expect(() => { - getBlendingQueryKey([ - { prop: 'val' } as unknown as QueryTimeDimension - ]); - }).toThrow( - 'Inconsistent QueryTimeDimension configuration ' + - `for the blending query, granularity required: ${ - ({ prop: 'val' }).toString()}` - ); - - expect(getBlendingQueryKey(timeDimensions)) - .toEqual(`${ - BLENDING_QUERY_KEY_PREFIX - }${ - timeDimensions[0].granularity - }`); - }); - - test('getBlendingResponseKey helper', () => { - const timeDimensions = JSON.parse( - JSON.stringify( - mockData - .blending_query_avg_discount_by_date_range_for_the_first_and_standard_ship_mode - .data[0] - .query - .timeDimensions - ) - ) as QueryTimeDimension[]; - - expect(() => { - getBlendingResponseKey(); - }).toThrow( - 'QueryTimeDimension should be specified ' + - 'for the blending query.' - ); - - expect(() => { - getBlendingResponseKey([ - { prop: 'val' } as unknown as QueryTimeDimension - ]); - }).toThrow( - 'Inconsistent QueryTimeDimension configuration ' + - `for the blending query, granularity required: ${ - ({ prop: 'val' }).toString()}` - ); - - expect(() => { - getBlendingResponseKey([ - { granularity: 'day' } as unknown as QueryTimeDimension - ]); - }).toThrow( - 'Inconsistent QueryTimeDimension configuration ' + - `for the blending query, dimension required: ${ - ({ granularity: 'day' }).toString()}` - ); - - expect(getBlendingResponseKey(timeDimensions)) - .toEqual(`${ - timeDimensions[0].dimension - }${ - BLENDING_QUERY_RES_SEPARATOR - }${ - timeDimensions[0].granularity - }`); - }); - - test('getMembers helper', () => { - let data; - - // throw - data = JSON.parse( - JSON.stringify(mockData.regular_profit_by_postal_code.data) - ); - data.aliasToMemberNameMap = {}; - expect(() => { - getMembers( - data.queryType as QueryTypeEnum, - data.query as unknown as NormalizedQuery, - data.data as { [sqlAlias: string]: DBResponseValue }[], - data.aliasToMemberNameMap, - data.annotation, - ); - }).toThrow( - 'You requested hidden member: \'e_commerce_records_us2021__postal_code\'. ' + - 'Please make it visible using `shown: true`. Please note primaryKey fields are ' + - '`shown: false` by default: ' + - 'https://cube.dev/docs/schema/reference/joins#setting-a-primary-key.' - ); - - // regular - data = JSON.parse( - JSON.stringify(mockData.regular_profit_by_postal_code.data) - ); - data.data = []; - expect(getMembers( - data.queryType as QueryTypeEnum, - data.query as unknown as NormalizedQuery, - data.data as { [sqlAlias: string]: DBResponseValue }[], - data.aliasToMemberNameMap, - data.annotation, - )).toEqual({}); - - data = JSON.parse( - JSON.stringify(mockData.regular_profit_by_postal_code.data) - ); - expect(getMembers( - data.queryType as QueryTypeEnum, - data.query as unknown as NormalizedQuery, - data.data as { [sqlAlias: string]: DBResponseValue }[], - data.aliasToMemberNameMap, - data.annotation, - )).toEqual({ - 'ECommerceRecordsUs2021.postalCode': 'e_commerce_records_us2021__postal_code', - 'ECommerceRecordsUs2021.avg_profit': 'e_commerce_records_us2021__avg_profit' - }); - - // compare date range - data = JSON.parse( - JSON.stringify(mockData.compare_date_range_count_by_order_date.data[0]) - ); - data.data = []; - expect(getMembers( - data.queryType as QueryTypeEnum, - data.query as unknown as NormalizedQuery, - data.data as { [sqlAlias: string]: DBResponseValue }[], - data.aliasToMemberNameMap, - data.annotation, - )).toEqual({}); - - data = JSON.parse( - JSON.stringify(mockData.compare_date_range_count_by_order_date.data[0]) - ); - expect(getMembers( - data.queryType as QueryTypeEnum, - data.query as unknown as NormalizedQuery, - data.data as { [sqlAlias: string]: DBResponseValue }[], - data.aliasToMemberNameMap, - data.annotation, - )).toEqual({ - 'ECommerceRecordsUs2021.orderDate.day': 'e_commerce_records_us2021__order_date_day', - 'ECommerceRecordsUs2021.orderDate': 'e_commerce_records_us2021__order_date_day', - 'ECommerceRecordsUs2021.count': 'e_commerce_records_us2021__count', - compareDateRange: 'compareDateRangeQuery', - }); - - data = JSON.parse( - JSON.stringify(mockData.compare_date_range_count_by_order_date.data[1]) - ); - expect(getMembers( - data.queryType as QueryTypeEnum, - data.query as unknown as NormalizedQuery, - data.data as { [sqlAlias: string]: DBResponseValue }[], - data.aliasToMemberNameMap, - data.annotation, - )).toEqual({ - 'ECommerceRecordsUs2021.orderDate.day': 'e_commerce_records_us2021__order_date_day', - 'ECommerceRecordsUs2021.orderDate': 'e_commerce_records_us2021__order_date_day', - 'ECommerceRecordsUs2021.count': 'e_commerce_records_us2021__count', - compareDateRange: 'compareDateRangeQuery', - }); - - // blending - data = JSON.parse( - JSON.stringify( - mockData - .blending_query_avg_discount_by_date_range_for_the_first_and_standard_ship_mode - .data[0] - ) - ); - data.data = []; - expect(getMembers( - data.queryType as QueryTypeEnum, - data.query as unknown as NormalizedQuery, - data.data as { [sqlAlias: string]: DBResponseValue }[], - data.aliasToMemberNameMap, - data.annotation, - )).toEqual({}); - - data = JSON.parse( - JSON.stringify( - mockData - .blending_query_avg_discount_by_date_range_for_the_first_and_standard_ship_mode - .data[0] - ) - ); - expect(getMembers( - data.queryType as QueryTypeEnum, - data.query as unknown as NormalizedQuery, - data.data as { [sqlAlias: string]: DBResponseValue }[], - data.aliasToMemberNameMap, - data.annotation, - )).toEqual({ - 'ECommerceRecordsUs2021.orderDate.month': 'e_commerce_records_us2021__order_date_month', - 'ECommerceRecordsUs2021.orderDate': 'e_commerce_records_us2021__order_date_month', - 'ECommerceRecordsUs2021.avg_discount': 'e_commerce_records_us2021__avg_discount', - 'time.month': 'e_commerce_records_us2021__order_date_month', - }); - - data = JSON.parse( - JSON.stringify( - mockData - .blending_query_avg_discount_by_date_range_for_the_first_and_standard_ship_mode - .data[1] - ) - ); - expect(getMembers( - data.queryType as QueryTypeEnum, - data.query as unknown as NormalizedQuery, - data.data as { [sqlAlias: string]: DBResponseValue }[], - data.aliasToMemberNameMap, - data.annotation, - )).toEqual({ - 'ECommerceRecordsUs2021.orderDate.month': 'e_commerce_records_us2021__order_date_month', - 'ECommerceRecordsUs2021.orderDate': 'e_commerce_records_us2021__order_date_month', - 'ECommerceRecordsUs2021.avg_discount': 'e_commerce_records_us2021__avg_discount', - 'time.month': 'e_commerce_records_us2021__order_date_month', - }); - }); - - test('getCompactRow helper', () => { - let data; - let membersMap; - let members; - - // regular - data = JSON.parse( - JSON.stringify(mockData.regular_profit_by_postal_code.data) - ); - membersMap = getMembers( - data.queryType as QueryTypeEnum, - data.query as unknown as NormalizedQuery, - data.data as { [sqlAlias: string]: DBResponseValue }[], - data.aliasToMemberNameMap, - data.annotation, - ); - members = Object.keys(membersMap); - expect(getCompactRow( - membersMap, - data.annotation as unknown as { [member: string]: ConfigItem }, - data.queryType as QueryType, - members, - data.query.timeDimensions as QueryTimeDimension[], - data.data[0], - )).toEqual(['95823', '646.1258666666666667']); - - data = JSON.parse( - JSON.stringify(mockData.regular_discount_by_city.data) - ); - membersMap = getMembers( - data.queryType as QueryTypeEnum, - data.query as unknown as NormalizedQuery, - data.data as { [sqlAlias: string]: DBResponseValue }[], - data.aliasToMemberNameMap, - data.annotation, - ); - members = Object.keys(membersMap); - expect(getCompactRow( - membersMap, - data.annotation as unknown as { [member: string]: ConfigItem }, - data.queryType as QueryType, - members, - data.query.timeDimensions as QueryTimeDimension[], - data.data[0], - )).toEqual(['Missouri City', '0.80000000000000000000']); - - // compare date range - data = JSON.parse( - JSON.stringify(mockData.compare_date_range_count_by_order_date.data[0]) - ); - membersMap = getMembers( - data.queryType as QueryTypeEnum, - data.query as unknown as NormalizedQuery, - data.data as { [sqlAlias: string]: DBResponseValue }[], - data.aliasToMemberNameMap, - data.annotation, - ); - members = Object.keys(membersMap); - expect(getCompactRow( - membersMap, - data.annotation as unknown as { [member: string]: ConfigItem }, - data.queryType as QueryType, - members, - data.query.timeDimensions as QueryTimeDimension[], - data.data[0], - )).toEqual([ - '2020-01-01T00:00:00.000', - '2020-01-01T00:00:00.000', - '10', - '2020-01-01T00:00:00.000 - 2020-01-31T23:59:59.999' - ]); - - data = JSON.parse( - JSON.stringify(mockData.compare_date_range_count_by_order_date.data[0]) - ); - membersMap = getMembers( - data.queryType as QueryTypeEnum, - data.query as unknown as NormalizedQuery, - data.data as { [sqlAlias: string]: DBResponseValue }[], - data.aliasToMemberNameMap, - data.annotation, - ); - members = Object.keys(membersMap); - expect(getCompactRow( - membersMap, - data.annotation as unknown as { [member: string]: ConfigItem }, - data.queryType as QueryType, - members, - data.query.timeDimensions as QueryTimeDimension[], - data.data[1], - )).toEqual([ - '2020-01-02T00:00:00.000', - '2020-01-02T00:00:00.000', - '8', - '2020-01-01T00:00:00.000 - 2020-01-31T23:59:59.999' - ]); - - // blending - data = JSON.parse( - JSON.stringify(mockData.blending_query_avg_discount_by_date_range_for_the_first_and_standard_ship_mode.data[0]) - ); - membersMap = getMembers( - data.queryType as QueryTypeEnum, - data.query as unknown as NormalizedQuery, - data.data as { [sqlAlias: string]: DBResponseValue }[], - data.aliasToMemberNameMap, - data.annotation, - ); - members = Object.keys(membersMap); - expect(getCompactRow( - membersMap, - data.annotation as unknown as { [member: string]: ConfigItem }, - data.queryType as QueryType, - members, - data.query.timeDimensions as QueryTimeDimension[], - data.data[0], - )).toEqual([ - '2020-01-01T00:00:00.000', - '2020-01-01T00:00:00.000', - '0.15638297872340425532', - '2020-01-01T00:00:00.000', - ]); - }); - - test('getVanilaRow helper', () => { - const data = JSON.parse( - JSON.stringify(mockData.regular_discount_by_city.data) - ); - delete data.aliasToMemberNameMap.e_commerce_records_us2021__avg_discount; - expect(() => getVanilaRow( - data.aliasToMemberNameMap, - data.annotation as unknown as { [member: string]: ConfigItem }, - data.queryType as QueryType, - data.query as unknown as NormalizedQuery, - data.data[0], - )).toThrow( - 'You requested hidden member: \'e_commerce_records_us2021__avg_discount\'. ' + - 'Please make it visible using `shown: true`. Please note ' + - 'primaryKey fields are `shown: false` by default: ' + - 'https://cube.dev/docs/schema/reference/joins#setting-a-primary-key.' - ); - }); -}); - -describe('transformData default mode', () => { - test('regular discount by city', () => { - let data; - - data = JSON.parse( - JSON.stringify(mockData.regular_discount_by_city.data) - ); - delete data.aliasToMemberNameMap.e_commerce_records_us2021__avg_discount; - expect(() => transformData( - data.aliasToMemberNameMap, - data.annotation as unknown as { [member: string]: ConfigItem }, - data.data, - data.query as unknown as NormalizedQuery, - data.queryType as QueryType, - )).toThrow(); - - data = JSON.parse( - JSON.stringify(mockData.regular_discount_by_city.data) - ); - expect( - transformData( - data.aliasToMemberNameMap, - data.annotation as unknown as { [member: string]: ConfigItem }, - data.data, - data.query as unknown as NormalizedQuery, - data.queryType as QueryType, - ) - ).toEqual(data.result_default); - }); - - test('regular profit by postal code', () => { - const data = JSON.parse( - JSON.stringify(mockData.regular_profit_by_postal_code.data) - ); - expect( - transformData( - data.aliasToMemberNameMap, - data.annotation as unknown as { [member: string]: ConfigItem }, - data.data, - data.query as unknown as NormalizedQuery, - data.queryType as QueryType, - ) - ).toEqual(data.result_default); - }); - - test('compare date range count by order date', () => { - const data = JSON.parse( - JSON.stringify(mockData.compare_date_range_count_by_order_date.data) - ); - - expect( - transformData( - data[0].aliasToMemberNameMap, - data[0].annotation as unknown as { [member: string]: ConfigItem }, - data[0].data, - data[0].query as unknown as NormalizedQuery, - data[0].queryType as QueryType, - ) - ).toEqual(data[0].result_default); - - expect( - transformData( - data[1].aliasToMemberNameMap, - data[1].annotation as unknown as { [member: string]: ConfigItem }, - data[1].data, - data[1].query as unknown as NormalizedQuery, - data[1].queryType as QueryType, - ) - ).toEqual(data[1].result_default); - }); - - test('blending query avg discount by date range for the first and standard ship mode', () => { - const data = JSON.parse( - JSON.stringify( - mockData - .blending_query_avg_discount_by_date_range_for_the_first_and_standard_ship_mode - .data - ) - ); - - expect( - transformData( - data[0].aliasToMemberNameMap, - data[0].annotation as unknown as { [member: string]: ConfigItem }, - data[0].data, - data[0].query as unknown as NormalizedQuery, - data[0].queryType as QueryType, - ) - ).toEqual(data[0].result_default); - - expect( - transformData( - data[1].aliasToMemberNameMap, - data[1].annotation as unknown as { [member: string]: ConfigItem }, - data[1].data, - data[1].query as unknown as NormalizedQuery, - data[1].queryType as QueryType, - ) - ).toEqual(data[1].result_default); - }); -}); - -describe('transformData compact mode', () => { - test('regular discount by city', () => { - let data; - - data = JSON.parse( - JSON.stringify(mockData.regular_discount_by_city.data) - ); - delete data.aliasToMemberNameMap.e_commerce_records_us2021__avg_discount; - expect(() => transformData( - data.aliasToMemberNameMap, - data.annotation as unknown as { [member: string]: ConfigItem }, - data.data, - data.query as unknown as NormalizedQuery, - data.queryType as QueryType, - ResultTypeEnum.COMPACT, - )).toThrow(); - - data = JSON.parse( - JSON.stringify(mockData.regular_discount_by_city.data) - ); - expect( - transformData( - data.aliasToMemberNameMap, - data.annotation as unknown as { [member: string]: ConfigItem }, - data.data, - data.query as unknown as NormalizedQuery, - data.queryType as QueryType, - ResultTypeEnum.COMPACT, - ) - ).toEqual(data.result_compact); - }); - - test('regular profit by postal code', () => { - const data = JSON.parse( - JSON.stringify(mockData.regular_profit_by_postal_code.data) - ); - expect( - transformData( - data.aliasToMemberNameMap, - data.annotation as unknown as { [member: string]: ConfigItem }, - data.data, - data.query as unknown as NormalizedQuery, - data.queryType as QueryType, - ResultTypeEnum.COMPACT, - ) - ).toEqual(data.result_compact); - }); - - test('compare date range count by order date', () => { - const data = JSON.parse( - JSON.stringify(mockData.compare_date_range_count_by_order_date.data) - ); - - expect( - transformData( - data[0].aliasToMemberNameMap, - data[0].annotation as unknown as { [member: string]: ConfigItem }, - data[0].data, - data[0].query as unknown as NormalizedQuery, - data[0].queryType as QueryType, - ResultTypeEnum.COMPACT, - ) - ).toEqual(data[0].result_compact); - - expect( - transformData( - data[1].aliasToMemberNameMap, - data[1].annotation as unknown as { [member: string]: ConfigItem }, - data[1].data, - data[1].query as unknown as NormalizedQuery, - data[1].queryType as QueryType, - ResultTypeEnum.COMPACT, - ) - ).toEqual(data[1].result_compact); - }); - - test('blending query avg discount by date range for the first and standard ship mode', () => { - const data = JSON.parse( - JSON.stringify( - mockData - .blending_query_avg_discount_by_date_range_for_the_first_and_standard_ship_mode - .data - ) - ); - - expect( - transformData( - data[0].aliasToMemberNameMap, - data[0].annotation as unknown as { [member: string]: ConfigItem }, - data[0].data, - data[0].query as unknown as NormalizedQuery, - data[0].queryType as QueryType, - ResultTypeEnum.COMPACT, - ) - ).toEqual(data[0].result_compact); - - expect( - transformData( - data[1].aliasToMemberNameMap, - data[1].annotation as unknown as { [member: string]: ConfigItem }, - data[1].data, - data[1].query as unknown as NormalizedQuery, - data[1].queryType as QueryType, - ResultTypeEnum.COMPACT, - ) - ).toEqual(data[1].result_compact); - }); -}); diff --git a/packages/cubejs-api-gateway/test/helpers/transformValue.test.ts b/packages/cubejs-api-gateway/test/helpers/transformValue.test.ts deleted file mode 100644 index 54101c9fcb131..0000000000000 --- a/packages/cubejs-api-gateway/test/helpers/transformValue.test.ts +++ /dev/null @@ -1,33 +0,0 @@ -/** - * @license Apache-2.0 - * @copyright Cube Dev, Inc. - * @fileoverview transformValue function unit tests. - */ - -/* globals describe,test,expect */ -/* eslint-disable import/no-duplicates */ -/* eslint-disable @typescript-eslint/no-duplicate-imports */ - -import moment from 'moment'; -import transformValueDef from '../../src/helpers/transformValue'; -import { transformValue } from '../../src/helpers/transformValue'; - -describe('transformValue helper', () => { - test('export looks as expected', () => { - expect(transformValueDef).toBeDefined(); - expect(transformValue).toBeDefined(); - expect(transformValue).toEqual(transformValueDef); - }); - test('object with the time value', () => { - const date = Date(); - expect(transformValue(date, 'time')).toBe( - moment.utc(date).format(moment.HTML5_FMT.DATETIME_LOCAL_MS) - ); - }); - test('object with the Date value', () => { - const date = new Date(); - expect(transformValue(date, 'date')).toBe( - moment(date).format(moment.HTML5_FMT.DATETIME_LOCAL_MS) - ); - }); -}); diff --git a/packages/cubejs-backend-native/Cargo.lock b/packages/cubejs-backend-native/Cargo.lock index 1ec0fc189e2b5..65113ac0e7061 100644 --- a/packages/cubejs-backend-native/Cargo.lock +++ b/packages/cubejs-backend-native/Cargo.lock @@ -128,7 +128,7 @@ dependencies = [ "chrono", "comfy-table 5.0.1", "csv", - "flatbuffers", + "flatbuffers 2.1.2", "half", "hex", "indexmap 1.9.3", @@ -176,7 +176,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.90", ] [[package]] @@ -187,7 +187,7 @@ checksum = "531b97fb4cd3dfdce92c35dedbfdc1f0b9d8091c8ca943d6dae340ef5012d514" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.90", ] [[package]] @@ -395,7 +395,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.90", "syn_derive", ] @@ -508,6 +508,7 @@ dependencies = [ "iana-time-zone", "js-sys", "num-traits", + "serde", "wasm-bindgen", "windows-targets 0.48.5", ] @@ -732,12 +733,14 @@ dependencies = [ name = "cubejs-native" version = "0.28.0" dependencies = [ + "anyhow", "async-channel", "async-trait", "axum", "bytes", "convert_case", "cubenativeutils", + "cubeorchestrator", "cubesql", "cubesqlplanner", "findshlibs", @@ -779,6 +782,26 @@ dependencies = [ "uuid 0.8.2", ] +[[package]] +name = "cubeorchestrator" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "cubeshared", + "itertools 0.13.0", + "neon", + "serde", + "serde_json", +] + +[[package]] +name = "cubeshared" +version = "0.1.0" +dependencies = [ + "flatbuffers 23.5.26", +] + [[package]] name = "cubesql" version = "0.28.0" @@ -805,7 +828,7 @@ dependencies = [ "futures-util", "hashbrown 0.14.3", "indexmap 1.9.3", - "itertools", + "itertools 0.10.5", "log", "lru", "minijinja", @@ -842,7 +865,7 @@ dependencies = [ "cubeclient", "cubenativeutils", "datafusion", - "itertools", + "itertools 0.10.5", "lazy_static", "minijinja", "nativebridge", @@ -874,7 +897,7 @@ dependencies = [ "datafusion-physical-expr", "futures", "hashbrown 0.12.3", - "itertools", + "itertools 0.10.5", "lazy_static", "log", "num_cpus", @@ -1089,6 +1112,16 @@ dependencies = [ "thiserror", ] +[[package]] +name = "flatbuffers" +version = "23.5.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dac53e22462d78c16d64a1cd22371b54cc3fe94aa15e7886a2fa6e5d1ab8640" +dependencies = [ + "bitflags 1.3.2", + "rustc_version", +] + [[package]] name = "flate2" version = "1.0.28" @@ -1200,7 +1233,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.90", ] [[package]] @@ -1586,6 +1619,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.10" @@ -1903,7 +1945,7 @@ dependencies = [ "Inflector", "async-trait", "byteorder", - "itertools", + "itertools 0.10.5", "proc-macro2", "quote", "syn 1.0.109", @@ -1931,7 +1973,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6813fde79b646e47e7ad75f480aa80ef76a5d9599e2717407961531169ee38b" dependencies = [ "quote", - "syn 2.0.76", + "syn 2.0.90", "syn-mid", ] @@ -2171,7 +2213,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.90", ] [[package]] @@ -2254,7 +2296,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.90", ] [[package]] @@ -2388,9 +2430,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.86" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" dependencies = [ "unicode-ident", ] @@ -2486,7 +2528,7 @@ dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.76", + "syn 2.0.90", ] [[package]] @@ -2498,7 +2540,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.90", ] [[package]] @@ -2775,6 +2817,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rustix" version = "0.38.28" @@ -2888,29 +2939,29 @@ checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" [[package]] name = "serde" -version = "1.0.209" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99fce0ffe7310761ca6bf9faf5115afbc19688edd00171d81b1bb1b116c63e09" +checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.209" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5831b979fd7b5439637af1752d535ff49f4860c0f341d1baeb6faf0f4242170" +checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.90", ] [[package]] name = "serde_json" -version = "1.0.127" +version = "1.0.133" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8043c06d9f82bd7271361ed64f415fe5e12a77fdb52e573e7f06a516dea329ad" +checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" dependencies = [ "indexmap 2.4.0", "itoa", @@ -3100,7 +3151,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.76", + "syn 2.0.90", ] [[package]] @@ -3139,9 +3190,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.76" +version = "2.0.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578e081a14e0cefc3279b0472138c513f37b41a08d5a3cca9b6e4e8ceb6cd525" +checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31" dependencies = [ "proc-macro2", "quote", @@ -3156,7 +3207,7 @@ checksum = "b5dc35bb08dd1ca3dfb09dce91fd2d13294d6711c88897d9a9d60acf39bce049" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.90", ] [[package]] @@ -3168,7 +3219,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.90", ] [[package]] @@ -3247,7 +3298,7 @@ checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.90", ] [[package]] @@ -3325,7 +3376,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.90", ] [[package]] @@ -3468,7 +3519,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.90", ] [[package]] @@ -3727,7 +3778,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.90", "wasm-bindgen-shared", ] @@ -3761,7 +3812,7 @@ checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.90", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -4018,7 +4069,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.90", ] [[package]] diff --git a/packages/cubejs-backend-native/Cargo.toml b/packages/cubejs-backend-native/Cargo.toml index 3c3666761e1e9..37ddb4094c1c1 100644 --- a/packages/cubejs-backend-native/Cargo.toml +++ b/packages/cubejs-backend-native/Cargo.toml @@ -16,13 +16,15 @@ opt-level = 1 crate-type = ["cdylib", "lib"] [dependencies] -cubesqlplanner = { path = "../../rust/cubesqlplanner/cubesqlplanner/" } +cubesqlplanner = { path = "../../rust/cubesqlplanner/cubesqlplanner" } +cubeorchestrator = { path = "../../rust/cubeorchestrator" } cubenativeutils = { path = "../../rust/cubenativeutils" } +cubesql = { path = "../../rust/cubesql/cubesql" } +anyhow = "1.0" async-channel = { version = "2" } async-trait = "0.1.36" convert_case = "0.6.0" pin-project = "1.1.5" -cubesql = { path = "../../rust/cubesql/cubesql" } findshlibs = "0.10.2" futures = "0.3.30" http-body-util = "0.1" diff --git a/packages/cubejs-backend-native/js/ResultWrapper.ts b/packages/cubejs-backend-native/js/ResultWrapper.ts new file mode 100644 index 0000000000000..9068a83ed3a8c --- /dev/null +++ b/packages/cubejs-backend-native/js/ResultWrapper.ts @@ -0,0 +1,237 @@ +import { + getCubestoreResult, + getFinalQueryResult, + getFinalQueryResultMulti, + ResultRow +} from './index'; + +export interface DataResult { + isWrapper: boolean; + getFinalResult(): Promise; + getRawData(): any[]; + getTransformData(): any[]; + getRootResultObject(): any[]; + // eslint-disable-next-line no-use-before-define + getResults(): ResultWrapper[]; +} + +class BaseWrapper { + public readonly isWrapper: boolean = true; +} + +export class ResultWrapper extends BaseWrapper implements DataResult { + private readonly proxy: any; + + private cache: any; + + public cached: Boolean = false; + + private readonly isNative: Boolean = false; + + private readonly nativeReference: any; + + private readonly jsResult: any = null; + + private transformData: any; + + private rootResultObject: any = {}; + + public constructor(input: any) { + super(); + + if (input.isWrapper) { + return input; + } + + if (Array.isArray(input)) { + this.jsResult = input; + } else { + this.isNative = true; + this.nativeReference = input; + } + + this.proxy = new Proxy(this, { + get: (target, prop: string | symbol) => { + // To support iterative access + if (prop === Symbol.iterator) { + const array = this.getArray(); + const l = array.length; + + return function* yieldArrayItem() { + for (let i = 0; i < l; i++) { + yield array[i]; + } + }; + } + + // intercept indexes + if (typeof prop === 'string' && !Number.isNaN(Number(prop))) { + const array = this.getArray(); + return array[Number(prop)]; + } + + // intercept isNative + if (prop === 'isNative') { + return this.isNative; + } + + // intercept array props and methods + if (typeof prop === 'string' && prop in Array.prototype) { + const arrayMethod = (Array.prototype as any)[prop]; + if (typeof arrayMethod === 'function') { + return (...args: any[]) => this.invokeArrayMethod(prop, ...args); + } + + return (this.getArray() as any)[prop]; + } + + // intercept JSON.stringify or toJSON() + if (prop === 'toJSON') { + return () => this.getArray(); + } + + return (target as any)[prop]; + }, + + // intercept array length + getOwnPropertyDescriptor: (target, prop) => { + if (prop === 'length') { + const array = this.getArray(); + return { + configurable: true, + enumerable: true, + value: array.length, + writable: false + }; + } + return Object.getOwnPropertyDescriptor(target, prop); + }, + + ownKeys: (target) => { + const array = this.getArray(); + return [...Object.keys(target), ...Object.keys(array), 'length', 'isNative']; + } + }); + Object.setPrototypeOf(this.proxy, ResultWrapper.prototype); + + return this.proxy; + } + + private getArray(): ResultRow[] { + if (!this.cache) { + if (this.isNative) { + this.cache = getCubestoreResult(this.nativeReference); + } else { + this.cache = this.jsResult; + } + this.cached = true; + } + return this.cache; + } + + private invokeArrayMethod(method: string, ...args: any[]): any { + const array = this.getArray(); + return (array as any)[method](...args); + } + + public getRawData(): any[] { + if (this.isNative) { + return [this.nativeReference]; + } + + return [this.jsResult]; + } + + public setTransformData(td: any) { + this.transformData = td; + } + + public getTransformData(): any[] { + return [this.transformData]; + } + + public setRootResultObject(obj: any) { + this.rootResultObject = obj; + } + + public getRootResultObject(): any[] { + return [this.rootResultObject]; + } + + public async getFinalResult(): Promise { + return getFinalQueryResult(this.transformData, this.getRawData()[0], this.rootResultObject); + } + + public getResults(): ResultWrapper[] { + return [this]; + } +} + +class BaseWrapperArray extends BaseWrapper { + public constructor(protected readonly results: ResultWrapper[]) { + super(); + } + + protected getInternalDataArrays(): any[] { + const [transformDataJson, rawData, resultDataJson] = this.results.reduce<[Object[], any[], Object[]]>( + ([transformList, rawList, resultList], r) => { + transformList.push(r.getTransformData()[0]); + rawList.push(r.getRawData()[0]); + resultList.push(r.getRootResultObject()[0]); + return [transformList, rawList, resultList]; + }, + [[], [], []] + ); + + return [transformDataJson, rawData, resultDataJson]; + } + + // Is invoked from the native side to get + // an array of all raw wrapped results + public getResults(): ResultWrapper[] { + return this.results; + } + + public getTransformData(): any[] { + return this.results.map(r => r.getTransformData()[0]); + } + + public getRawData(): any[] { + return this.results.map(r => r.getRawData()[0]); + } + + public getRootResultObject(): any[] { + return this.results.map(r => r.getRootResultObject()[0]); + } +} + +export class ResultMultiWrapper extends BaseWrapperArray implements DataResult { + public constructor(results: ResultWrapper[], private rootResultObject: any) { + super(results); + } + + public async getFinalResult(): Promise { + const [transformDataJson, rawDataRef, cleanResultList] = this.getInternalDataArrays(); + + const responseDataObj = { + queryType: this.rootResultObject.queryType, + results: cleanResultList, + slowQuery: this.rootResultObject.slowQuery, + }; + + return getFinalQueryResultMulti(transformDataJson, rawDataRef, responseDataObj); + } +} + +// This is consumed by native side via Transport Bridge +export class ResultArrayWrapper extends BaseWrapperArray implements DataResult { + public constructor(results: ResultWrapper[]) { + super(results); + } + + public async getFinalResult(): Promise { + const [transformDataJson, rawDataRef, cleanResultList] = this.getInternalDataArrays(); + + return [transformDataJson, rawDataRef, cleanResultList]; + } +} diff --git a/packages/cubejs-backend-native/js/index.ts b/packages/cubejs-backend-native/js/index.ts index 779195ccfa8f5..f6c54477f463f 100644 --- a/packages/cubejs-backend-native/js/index.ts +++ b/packages/cubejs-backend-native/js/index.ts @@ -3,6 +3,9 @@ import fs from 'fs'; import path from 'path'; import { Writable } from 'stream'; import type { Request as ExpressRequest } from 'express'; +import { ResultWrapper } from './ResultWrapper'; + +export * from './ResultWrapper'; export interface BaseMeta { // postgres or mysql @@ -98,14 +101,28 @@ export type SQLInterfaceOptions = { gatewayPort?: number, }; +export type DBResponsePrimitive = + null | + boolean | + number | + string; + +let loadedNative: any = null; + export function loadNative() { + if (loadedNative) { + return loadedNative; + } + // Development version if (fs.existsSync(path.join(__dirname, '/../../index.node'))) { - return require(path.join(__dirname, '/../../index.node')); + loadedNative = require(path.join(__dirname, '/../../index.node')); + return loadedNative; } if (fs.existsSync(path.join(__dirname, '/../../native/index.node'))) { - return require(path.join(__dirname, '/../../native/index.node')); + loadedNative = require(path.join(__dirname, '/../../native/index.node')); + return loadedNative; } throw new Error( @@ -113,10 +130,6 @@ export function loadNative() { ); } -export function isSupported(): boolean { - return fs.existsSync(path.join(__dirname, '/../../index.node')) || fs.existsSync(path.join(__dirname, '/../../native/index.node')); -} - function wrapNativeFunctionWithChannelCallback( fn: (extra: any) => unknown | Promise, ) { @@ -253,8 +266,9 @@ function wrapNativeFunctionWithStream( }); } else if (response.error) { writerOrChannel.reject(errorString(response)); + } else if (response.isWrapper) { // Native wrapped result + writerOrChannel.resolve(response); } else { - // TODO remove JSON.stringify() writerOrChannel.resolve(JSON.stringify(response)); } } catch (e: any) { @@ -348,6 +362,49 @@ export const buildSqlAndParams = (cubeEvaluator: any): String => { return native.buildSqlAndParams(cubeEvaluator); }; +export type ResultRow = Record; + +export const parseCubestoreResultMessage = async (message: ArrayBuffer): Promise => { + const native = loadNative(); + + const msg = await native.parseCubestoreResultMessage(message); + return new ResultWrapper(msg); +}; + +export const getCubestoreResult = (ref: ResultWrapper): ResultRow[] => { + const native = loadNative(); + + return native.getCubestoreResult(ref); +}; + +/** + * Transform and prepare single query final result data that is sent to the client. + * + * @param transformDataObj Data needed to transform raw query results + * @param rows Raw data received from the source DB via driver or reference to a native CubeStore response result + * @param resultData Final query result structure without actual data + * @return {Promise} ArrayBuffer with json-serialized data which should be directly sent to the client + */ +export const getFinalQueryResult = (transformDataObj: Object, rows: any, resultData: Object): Promise => { + const native = loadNative(); + + return native.getFinalQueryResult(transformDataObj, rows, resultData); +}; + +/** + * Transform and prepare multiple query final results data into a single response structure. + * + * @param transformDataArr Array of data needed to transform raw query results + * @param rows Array of raw data received from the source DB via driver or reference to native CubeStore response results + * @param responseData Final combined query result structure without actual data + * @return {Promise} ArrayBuffer with json-serialized data which should be directly sent to the client + */ +export const getFinalQueryResultMulti = (transformDataArr: Object[], rows: any[], responseData: Object): Promise => { + const native = loadNative(); + + return native.getFinalQueryResultMulti(transformDataArr, rows, responseData); +}; + export interface PyConfiguration { repositoryFactory?: (ctx: unknown) => Promise, logger?: (msg: string, params: Record) => void, diff --git a/packages/cubejs-backend-native/src/channel.rs b/packages/cubejs-backend-native/src/channel.rs index 9394d8e9b27e3..607631c68c661 100644 --- a/packages/cubejs-backend-native/src/channel.rs +++ b/packages/cubejs-backend-native/src/channel.rs @@ -4,7 +4,9 @@ use std::collections::HashMap; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; +use crate::orchestrator::ResultWrapper; use crate::transport::MapCubeErrExt; +use crate::utils::bind_method; use async_trait::async_trait; use cubesql::transport::{SqlGenerator, SqlTemplates}; use cubesql::CubeError; @@ -13,8 +15,6 @@ use log::trace; use neon::prelude::*; use tokio::sync::oneshot; -use crate::utils::bind_method; - type JsAsyncStringChannelCallback = Box) -> Result<(), CubeError> + Send>; type JsAsyncChannelCallback = Box< @@ -194,6 +194,12 @@ where rx.await? } +#[derive(Debug)] +pub enum ValueFromJs { + String(String), + ResultWrapper(Vec), +} + #[allow(clippy::type_complexity)] pub async fn call_raw_js_with_channel_as_callback( channel: Arc, diff --git a/packages/cubejs-backend-native/src/lib.rs b/packages/cubejs-backend-native/src/lib.rs index 07a36aa69481a..68285075541b6 100644 --- a/packages/cubejs-backend-native/src/lib.rs +++ b/packages/cubejs-backend-native/src/lib.rs @@ -11,7 +11,9 @@ pub mod cross; pub mod gateway; pub mod logger; pub mod node_export; +pub mod node_obj_deserializer; pub mod node_obj_serializer; +pub mod orchestrator; #[cfg(feature = "python")] pub mod python; pub mod stream; diff --git a/packages/cubejs-backend-native/src/node_export.rs b/packages/cubejs-backend-native/src/node_export.rs index 53464958d20b2..5d4eb733206c6 100644 --- a/packages/cubejs-backend-native/src/node_export.rs +++ b/packages/cubejs-backend-native/src/node_export.rs @@ -9,11 +9,6 @@ use futures::StreamExt; use serde_json::Map; use tokio::sync::Semaphore; -use std::net::SocketAddr; -use std::rc::Rc; -use std::str::FromStr; -use std::sync::Arc; - use crate::auth::{NativeAuthContext, NodeBridgeAuthService}; use crate::channel::call_js_fn; use crate::config::{NodeConfiguration, NodeConfigurationFactoryOptions, NodeCubeServices}; @@ -31,6 +26,10 @@ use cubenativeutils::wrappers::serializer::NativeDeserialize; use cubenativeutils::wrappers::NativeContextHolder; use cubesqlplanner::cube_bridge::base_query_options::NativeBaseQueryOptions; use cubesqlplanner::planner::base_query::BaseQuery; +use std::net::SocketAddr; +use std::rc::Rc; +use std::str::FromStr; +use std::sync::Arc; use cubesql::{telemetry::ReportingLogger, CubeError}; @@ -513,8 +512,12 @@ pub fn register_module_exports( cx.export_function("isFallbackBuild", is_fallback_build)?; cx.export_function("__js_to_clrepr_to_js", debug_js_to_clrepr_to_js)?; + //============ sql planner exports =================== cx.export_function("buildSqlAndParams", build_sql_and_params)?; + //========= sql orchestrator exports ================= + crate::orchestrator::register_module(&mut cx)?; + crate::template::template_register_module(&mut cx)?; #[cfg(feature = "python")] diff --git a/packages/cubejs-backend-native/src/node_obj_deserializer.rs b/packages/cubejs-backend-native/src/node_obj_deserializer.rs new file mode 100644 index 0000000000000..d9ed815682792 --- /dev/null +++ b/packages/cubejs-backend-native/src/node_obj_deserializer.rs @@ -0,0 +1,436 @@ +use neon::prelude::*; +use neon::result::Throw; +use serde::de::{ + self, Deserializer, EnumAccess, IntoDeserializer, MapAccess, SeqAccess, StdError, + VariantAccess, Visitor, +}; +use serde::forward_to_deserialize_any; +use std::fmt; + +#[derive(Debug)] +pub struct JsDeserializationError(String); + +impl fmt::Display for JsDeserializationError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "JS Deserialization Error: {}", self.0) + } +} + +impl From for JsDeserializationError { + fn from(throw: Throw) -> Self { + JsDeserializationError(throw.to_string()) + } +} + +impl StdError for JsDeserializationError {} + +impl de::Error for JsDeserializationError { + fn custom(msg: T) -> Self { + JsDeserializationError(msg.to_string()) + } +} + +pub struct JsValueDeserializer<'a, 'b> { + pub cx: &'a mut FunctionContext<'b>, + pub value: Handle<'a, JsValue>, +} + +impl<'a, 'b> JsValueDeserializer<'a, 'b> { + pub fn new(cx: &'a mut FunctionContext<'b>, value: Handle<'a, JsValue>) -> Self { + Self { cx, value } + } +} + +impl<'de, 'a, 'b> Deserializer<'de> for JsValueDeserializer<'a, 'b> { + type Error = JsDeserializationError; + + fn deserialize_any(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + if self.value.is_a::(self.cx) { + let value = self + .value + .downcast::(self.cx) + .or_throw(self.cx)? + .value(self.cx); + visitor.visit_string(value) + } else if self.value.is_a::(self.cx) { + let value = self + .value + .downcast::(self.cx) + .or_throw(self.cx)? + .value(self.cx); + + // floats + if value.fract() != 0.0 { + return visitor.visit_f64(value); + } + + // unsigned + if value >= 0.0 { + if value <= u8::MAX as f64 { + return visitor.visit_u8(value as u8); + } + + if value <= u16::MAX as f64 { + return visitor.visit_u16(value as u16); + } + + if value <= u32::MAX as f64 { + return visitor.visit_u32(value as u32); + } + + if value <= u64::MAX as f64 { + return visitor.visit_u64(value as u64); + } + } + + if value >= i8::MIN as f64 && value <= i8::MAX as f64 { + return visitor.visit_i8(value as i8); + } + + if value >= i16::MIN as f64 && value <= i16::MAX as f64 { + return visitor.visit_i16(value as i16); + } + + if value >= i32::MIN as f64 && value <= i32::MAX as f64 { + return visitor.visit_i32(value as i32); + } + + if value >= i64::MIN as f64 && value <= i64::MAX as f64 { + return visitor.visit_i64(value as i64); + } + + Err(JsDeserializationError( + "Unsupported number type for deserialization".to_string(), + )) + } else if self.value.is_a::(self.cx) { + let value = self + .value + .downcast::(self.cx) + .or_throw(self.cx)? + .value(self.cx); + visitor.visit_bool(value) + } else if self.value.is_a::(self.cx) { + let js_array = self + .value + .downcast::(self.cx) + .or_throw(self.cx)?; + let deserializer = JsArrayDeserializer::new(self.cx, js_array); + visitor.visit_seq(deserializer) + } else if self.value.is_a::(self.cx) { + let js_object = self + .value + .downcast::(self.cx) + .or_throw(self.cx)?; + let deserializer = JsObjectDeserializer::new(self.cx, js_object); + visitor.visit_map(deserializer) + } else if self.value.is_a::(self.cx) + || self.value.is_a::(self.cx) + { + visitor.visit_none() + } else if self.value.is_a::(self.cx) { + // We can do nothing with the JS functions in native + visitor.visit_none() + } else { + Err(JsDeserializationError( + "Unsupported type for deserialization".to_string(), + )) + } + } + + fn deserialize_option(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + if self.value.is_a::(self.cx) || self.value.is_a::(self.cx) { + visitor.visit_none() + } else { + visitor.visit_some(self) + } + } + + fn deserialize_unit(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_unit() + } + + fn deserialize_unit_struct( + self, + _name: &'static str, + visitor: V, + ) -> Result + where + V: Visitor<'de>, + { + self.deserialize_unit(visitor) + } + + fn deserialize_newtype_struct( + self, + _name: &'static str, + visitor: V, + ) -> Result + where + V: Visitor<'de>, + { + visitor.visit_newtype_struct(self) + } + + fn deserialize_seq(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + if self.value.is_a::(self.cx) { + let js_array = self + .value + .downcast::(self.cx) + .or_throw(self.cx)?; + let deserializer = JsArrayDeserializer::new(self.cx, js_array); + visitor.visit_seq(deserializer) + } else { + Err(JsDeserializationError("expected an array".to_string())) + } + } + + fn deserialize_tuple(self, _len: usize, visitor: V) -> Result + where + V: Visitor<'de>, + { + self.deserialize_seq(visitor) + } + + fn deserialize_tuple_struct( + self, + _name: &'static str, + _len: usize, + visitor: V, + ) -> Result + where + V: Visitor<'de>, + { + self.deserialize_seq(visitor) + } + + fn deserialize_map(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + if self.value.is_a::(self.cx) { + let js_object = self + .value + .downcast::(self.cx) + .or_throw(self.cx)?; + let deserializer = JsObjectDeserializer::new(self.cx, js_object); + visitor.visit_map(deserializer) + } else { + Err(JsDeserializationError("expected an object".to_string())) + } + } + + fn deserialize_struct( + self, + _name: &'static str, + _fields: &'static [&'static str], + visitor: V, + ) -> Result + where + V: Visitor<'de>, + { + self.deserialize_map(visitor) + } + + fn deserialize_enum( + self, + _name: &'static str, + _variants: &'static [&'static str], + visitor: V, + ) -> Result + where + V: Visitor<'de>, + { + let deserializer = JsEnumDeserializer::new(self.cx, self.value); + visitor.visit_enum(deserializer) + } + + forward_to_deserialize_any! { + bool i8 i16 i32 i64 u8 u16 u32 u64 f32 f64 char str string + bytes byte_buf identifier ignored_any + } +} + +struct JsObjectDeserializer<'a, 'b> { + cx: &'a mut FunctionContext<'b>, + js_object: Handle<'a, JsObject>, + keys: Vec, + index: usize, +} + +impl<'a, 'b> JsObjectDeserializer<'a, 'b> { + fn new(cx: &'a mut FunctionContext<'b>, js_object: Handle<'a, JsObject>) -> Self { + let keys = js_object + .get_own_property_names(cx) + .expect("Failed to get object keys") + .to_vec(cx) + .expect("Failed to convert keys to Vec") + .iter() + .filter_map(|k| { + k.downcast_or_throw::(cx) + .ok() + .map(|js_string| js_string.value(cx)) + }) + .collect::>(); + Self { + cx, + js_object, + keys, + index: 0, + } + } +} + +// `MapAccess` is provided to the `Visitor` to give it the ability to iterate +// through entries of the map. +impl<'de, 'a, 'b> MapAccess<'de> for JsObjectDeserializer<'a, 'b> { + type Error = JsDeserializationError; + + fn next_key_seed(&mut self, seed: K) -> Result, Self::Error> + where + K: de::DeserializeSeed<'de>, + { + if self.index >= self.keys.len() { + return Ok(None); + } + let key = &self.keys[self.index]; + self.index += 1; + seed.deserialize(key.as_str().into_deserializer()).map(Some) + } + + fn next_value_seed(&mut self, seed: V) -> Result + where + V: de::DeserializeSeed<'de>, + { + let key = &self.keys[self.index - 1]; + let value = self + .js_object + .get(self.cx, key.as_str()) + .expect("Failed to get value by key"); + seed.deserialize(JsValueDeserializer::new(self.cx, value)) + } +} + +struct JsArrayDeserializer<'a, 'b> { + cx: &'a mut FunctionContext<'b>, + js_array: Handle<'a, JsArray>, + index: usize, + length: usize, +} + +impl<'a, 'b> JsArrayDeserializer<'a, 'b> { + fn new(cx: &'a mut FunctionContext<'b>, js_array: Handle<'a, JsArray>) -> Self { + let length = js_array.len(cx) as usize; + Self { + cx, + js_array, + index: 0, + length, + } + } +} + +// `SeqAccess` is provided to the `Visitor` to give it the ability to iterate +// through elements of the sequence. +impl<'de, 'a, 'b> SeqAccess<'de> for JsArrayDeserializer<'a, 'b> { + type Error = JsDeserializationError; + + fn next_element_seed(&mut self, seed: T) -> Result, Self::Error> + where + T: de::DeserializeSeed<'de>, + { + if self.index >= self.length { + return Ok(None); + } + let value = self + .js_array + .get(self.cx, self.index as u32) + .map_err(JsDeserializationError::from)?; + self.index += 1; + + seed.deserialize(JsValueDeserializer::new(self.cx, value)) + .map(Some) + } + + fn size_hint(&self) -> Option { + Some(self.length) + } +} + +pub struct JsEnumDeserializer<'a, 'b> { + cx: &'a mut FunctionContext<'b>, + value: Handle<'a, JsValue>, +} + +impl<'a, 'b> JsEnumDeserializer<'a, 'b> { + fn new(cx: &'a mut FunctionContext<'b>, value: Handle<'a, JsValue>) -> Self { + Self { cx, value } + } +} + +// `EnumAccess` is provided to the `Visitor` to give it the ability to determine +// which variant of the enum is supposed to be deserialized. +impl<'de, 'a, 'b> EnumAccess<'de> for JsEnumDeserializer<'a, 'b> { + type Error = JsDeserializationError; + type Variant = Self; + + fn variant_seed(self, seed: T) -> Result<(T::Value, Self::Variant), Self::Error> + where + T: de::DeserializeSeed<'de>, + { + let variant = seed.deserialize(JsValueDeserializer::new(self.cx, self.value))?; + Ok((variant, self)) + } +} + +impl<'de, 'a, 'b> VariantAccess<'de> for JsEnumDeserializer<'a, 'b> { + type Error = JsDeserializationError; + + // If the `Visitor` expected this variant to be a unit variant, the input + // should have been the plain string case handled in `deserialize_enum`. + fn unit_variant(self) -> Result<(), Self::Error> { + Ok(()) + } + + // Newtype variants are represented in JSON as `{ NAME: VALUE }` so + // deserialize the value here. + fn newtype_variant_seed(self, seed: T) -> Result + where + T: de::DeserializeSeed<'de>, + { + seed.deserialize(JsValueDeserializer::new(self.cx, self.value)) + } + + // Tuple variants are represented in JSON as `{ NAME: [DATA...] }` so + // deserialize the sequence of data here. + fn tuple_variant(self, _len: usize, visitor: V) -> Result + where + V: Visitor<'de>, + { + Deserializer::deserialize_seq(JsValueDeserializer::new(self.cx, self.value), visitor) + } + + // Struct variants are represented in JSON as `{ NAME: { K: V, ... } }` so + // deserialize the inner map here. + fn struct_variant( + self, + _fields: &'static [&'static str], + visitor: V, + ) -> Result + where + V: Visitor<'de>, + { + Deserializer::deserialize_map(JsValueDeserializer::new(self.cx, self.value), visitor) + } +} diff --git a/packages/cubejs-backend-native/src/orchestrator.rs b/packages/cubejs-backend-native/src/orchestrator.rs new file mode 100644 index 0000000000000..227e8b9970d76 --- /dev/null +++ b/packages/cubejs-backend-native/src/orchestrator.rs @@ -0,0 +1,346 @@ +use crate::node_obj_deserializer::JsValueDeserializer; +use crate::transport::MapCubeErrExt; +use cubeorchestrator::query_message_parser::QueryResult; +use cubeorchestrator::query_result_transform::{ + DBResponsePrimitive, RequestResultData, RequestResultDataMulti, TransformedData, +}; +use cubeorchestrator::transport::{JsRawData, TransformDataRequest}; +use cubesql::compile::engine::df::scan::{FieldValue, ValueObject}; +use cubesql::CubeError; +use neon::context::{Context, FunctionContext, ModuleContext}; +use neon::handle::Handle; +use neon::object::Object; +use neon::prelude::{ + JsArray, JsArrayBuffer, JsBox, JsBuffer, JsFunction, JsObject, JsPromise, JsResult, JsValue, + NeonResult, +}; +use neon::types::buffer::TypedArray; +use serde::Deserialize; +use std::borrow::Cow; +use std::sync::Arc; + +pub fn register_module(cx: &mut ModuleContext) -> NeonResult<()> { + cx.export_function( + "parseCubestoreResultMessage", + parse_cubestore_result_message, + )?; + cx.export_function("getCubestoreResult", get_cubestore_result)?; + cx.export_function("getFinalQueryResult", final_query_result)?; + cx.export_function("getFinalQueryResultMulti", final_query_result_multi)?; + + Ok(()) +} + +#[derive(Debug, Clone)] +pub struct ResultWrapper { + transform_data: TransformDataRequest, + data: Arc, + transformed_data: Option, +} + +impl ResultWrapper { + pub fn from_js_result_wrapper( + cx: &mut FunctionContext<'_>, + js_result_wrapper_val: Handle, + ) -> Result { + let js_result_wrapper = js_result_wrapper_val + .downcast::(cx) + .map_cube_err("Can't downcast JS ResultWrapper to object")?; + + let get_transform_data_js_method: Handle = js_result_wrapper + .get(cx, "getTransformData") + .map_cube_err("Can't get getTransformData() method from JS ResultWrapper object")?; + + let transform_data_js_arr = get_transform_data_js_method + .call(cx, js_result_wrapper.upcast::(), []) + .map_cube_err("Error calling getTransformData() method of ResultWrapper object")? + .downcast::(cx) + .map_cube_err("Can't downcast JS transformData to array")? + .to_vec(cx) + .map_cube_err("Can't convert JS transformData to array")?; + + let transform_data_js = transform_data_js_arr.first().unwrap(); + + let deserializer = JsValueDeserializer::new(cx, *transform_data_js); + let transform_request: TransformDataRequest = match Deserialize::deserialize(deserializer) { + Ok(data) => data, + Err(_) => { + return Err(CubeError::internal( + "Can't deserialize transformData from JS ResultWrapper object".to_string(), + )) + } + }; + + let get_raw_data_js_method: Handle = js_result_wrapper + .get(cx, "getRawData") + .map_cube_err("Can't get getRawData() method from JS ResultWrapper object")?; + + let raw_data_js_arr = get_raw_data_js_method + .call(cx, js_result_wrapper.upcast::(), []) + .map_cube_err("Error calling getRawData() method of ResultWrapper object")? + .downcast::(cx) + .map_cube_err("Can't downcast JS rawData to array")? + .to_vec(cx) + .map_cube_err("Can't convert JS rawData to array")?; + + let raw_data_js = raw_data_js_arr.first().unwrap(); + + let query_result = + if let Ok(js_box) = raw_data_js.downcast::>, _>(cx) { + Arc::clone(&js_box) + } else if let Ok(js_array) = raw_data_js.downcast::(cx) { + let deserializer = JsValueDeserializer::new(cx, js_array.upcast()); + let js_raw_data: JsRawData = match Deserialize::deserialize(deserializer) { + Ok(data) => data, + Err(_) => { + return Err(CubeError::internal( + "Can't deserialize results raw data from JS ResultWrapper object" + .to_string(), + )); + } + }; + + QueryResult::from_js_raw_data(js_raw_data) + .map(Arc::new) + .map_cube_err("Can't build results data from JS rawData")? + } else { + return Err(CubeError::internal( + "Can't deserialize results raw data from JS ResultWrapper object".to_string(), + )); + }; + + Ok(Self { + transform_data: transform_request, + data: query_result, + transformed_data: None, + }) + } + + pub fn transform_result(&mut self) -> Result<(), CubeError> { + self.transformed_data = Some( + TransformedData::transform(&self.transform_data, &self.data) + .map_cube_err("Can't prepare transformed data")?, + ); + + Ok(()) + } +} + +impl ValueObject for ResultWrapper { + fn len(&mut self) -> Result { + if self.transformed_data.is_none() { + self.transform_result()?; + } + + let data = self.transformed_data.as_ref().unwrap(); + + match data { + TransformedData::Compact { + members: _members, + dataset, + } => Ok(dataset.len()), + TransformedData::Vanilla(dataset) => Ok(dataset.len()), + } + } + + fn get(&mut self, index: usize, field_name: &str) -> Result { + if self.transformed_data.is_none() { + self.transform_result()?; + } + + let data = self.transformed_data.as_ref().unwrap(); + + let value = match data { + TransformedData::Compact { members, dataset } => { + let Some(row) = dataset.get(index) else { + return Err(CubeError::user(format!( + "Unexpected response from Cube, can't get {} row", + index + ))); + }; + + let Some(member_index) = members.iter().position(|m| m == field_name) else { + return Err(CubeError::user(format!( + "Field name '{}' not found in members", + field_name + ))); + }; + + row.get(member_index).unwrap_or(&DBResponsePrimitive::Null) + } + TransformedData::Vanilla(dataset) => { + let Some(row) = dataset.get(index) else { + return Err(CubeError::user(format!( + "Unexpected response from Cube, can't get {} row", + index + ))); + }; + + row.get(field_name).unwrap_or(&DBResponsePrimitive::Null) + } + }; + + Ok(match value { + DBResponsePrimitive::String(s) => FieldValue::String(Cow::Borrowed(s)), + DBResponsePrimitive::Number(n) => FieldValue::Number(*n), + DBResponsePrimitive::Boolean(b) => FieldValue::Bool(*b), + DBResponsePrimitive::Null => FieldValue::Null, + }) + } +} + +fn json_to_array_buffer<'a, C>( + mut cx: C, + json_data: Result, +) -> JsResult<'a, JsArrayBuffer> +where + C: Context<'a>, +{ + match json_data { + Ok(json_data) => { + let json_bytes = json_data.as_bytes(); + let mut js_buffer = cx.array_buffer(json_bytes.len())?; + { + let buffer = js_buffer.as_mut_slice(&mut cx); + buffer.copy_from_slice(json_bytes); + } + Ok(js_buffer) + } + Err(err) => cx.throw_error(err.to_string()), + } +} + +fn extract_query_result( + cx: &mut FunctionContext<'_>, + data_arg: Handle, +) -> Result, anyhow::Error> { + if let Ok(js_box) = data_arg.downcast::>, _>(cx) { + Ok(Arc::clone(&js_box)) + } else if let Ok(js_array) = data_arg.downcast::(cx) { + let deserializer = JsValueDeserializer::new(cx, js_array.upcast()); + let js_raw_data: JsRawData = Deserialize::deserialize(deserializer)?; + + QueryResult::from_js_raw_data(js_raw_data) + .map(Arc::new) + .map_err(anyhow::Error::from) + } else { + Err(anyhow::anyhow!( + "Second argument must be an Array of JsBox> or JsArray" + )) + } +} + +pub fn parse_cubestore_result_message(mut cx: FunctionContext) -> JsResult { + let msg = cx.argument::(0)?; + let msg_data = msg.as_slice(&cx).to_vec(); + + let promise = cx + .task(move || QueryResult::from_cubestore_fb(&msg_data)) + .promise(move |mut cx, res| match res { + Ok(result) => Ok(cx.boxed(Arc::new(result))), + Err(err) => cx.throw_error(err.to_string()), + }); + + Ok(promise) +} + +pub fn get_cubestore_result(mut cx: FunctionContext) -> JsResult { + let result = cx.argument::>>(0)?; + + let js_array = cx.execute_scoped(|mut cx| { + let js_array = JsArray::new(&mut cx, result.rows.len()); + + for (i, row) in result.rows.iter().enumerate() { + let js_row = cx.execute_scoped(|mut cx| { + let js_row = JsObject::new(&mut cx); + for (key, value) in result.columns.iter().zip(row.iter()) { + let js_key = cx.string(key); + let js_value = cx.string(value.to_string()); + js_row.set(&mut cx, js_key, js_value)?; + } + Ok(js_row) + })?; + + js_array.set(&mut cx, i as u32, js_row)?; + } + + Ok(js_array) + })?; + + Ok(js_array.upcast()) +} + +pub fn final_query_result(mut cx: FunctionContext) -> JsResult { + let transform_data_js_object = cx.argument::(0)?; + let deserializer = JsValueDeserializer::new(&mut cx, transform_data_js_object); + let transform_request_data: TransformDataRequest = match Deserialize::deserialize(deserializer) + { + Ok(data) => data, + Err(err) => return cx.throw_error(err.to_string()), + }; + + let data_arg = cx.argument::(1)?; + let cube_store_result: Arc = match extract_query_result(&mut cx, data_arg) { + Ok(query_result) => query_result, + Err(err) => return cx.throw_error(err.to_string()), + }; + + let result_data_js_object = cx.argument::(2)?; + let deserializer = JsValueDeserializer::new(&mut cx, result_data_js_object); + let mut result_data: RequestResultData = match Deserialize::deserialize(deserializer) { + Ok(data) => data, + Err(err) => return cx.throw_error(err.to_string()), + }; + + let promise = cx + .task(move || { + result_data.prepare_results(&transform_request_data, &cube_store_result)?; + + match serde_json::to_string(&result_data) { + Ok(json) => Ok(json), + Err(err) => Err(anyhow::Error::from(err)), + } + }) + .promise(move |cx, json_data| json_to_array_buffer(cx, json_data)); + + Ok(promise) +} + +pub fn final_query_result_multi(mut cx: FunctionContext) -> JsResult { + let transform_data_array = cx.argument::(0)?; + let deserializer = JsValueDeserializer::new(&mut cx, transform_data_array); + let transform_requests: Vec = match Deserialize::deserialize(deserializer) + { + Ok(data) => data, + Err(err) => return cx.throw_error(err.to_string()), + }; + + let data_array = cx.argument::(1)?; + let mut cube_store_results: Vec> = vec![]; + for data_arg in data_array.to_vec(&mut cx)? { + match extract_query_result(&mut cx, data_arg) { + Ok(query_result) => cube_store_results.push(query_result), + Err(err) => return cx.throw_error(err.to_string()), + }; + } + + let result_data_js_object = cx.argument::(2)?; + let deserializer = JsValueDeserializer::new(&mut cx, result_data_js_object); + let mut result_data: RequestResultDataMulti = match Deserialize::deserialize(deserializer) { + Ok(data) => data, + Err(err) => return cx.throw_error(err.to_string()), + }; + + let promise = cx + .task(move || { + result_data.prepare_results(&transform_requests, &cube_store_results)?; + + match serde_json::to_string(&result_data) { + Ok(json) => Ok(json), + Err(err) => Err(anyhow::Error::from(err)), + } + }) + .promise(move |cx, json_data| json_to_array_buffer(cx, json_data)); + + Ok(promise) +} diff --git a/packages/cubejs-backend-native/src/transport.rs b/packages/cubejs-backend-native/src/transport.rs index 30c5fbc1b84e5..a54aa3f46bda3 100644 --- a/packages/cubejs-backend-native/src/transport.rs +++ b/packages/cubejs-backend-native/src/transport.rs @@ -3,8 +3,18 @@ use neon::prelude::*; use std::collections::HashMap; use std::fmt::Display; +use crate::auth::NativeAuthContext; +use crate::channel::{call_raw_js_with_channel_as_callback, NodeSqlGenerator, ValueFromJs}; +use crate::node_obj_serializer::NodeObjSerializer; +use crate::orchestrator::ResultWrapper; +use crate::{ + auth::TransportRequest, channel::call_js_with_channel_as_callback, + stream::call_js_with_stream_as_callback, +}; use async_trait::async_trait; -use cubesql::compile::engine::df::scan::{MemberField, SchemaRef}; +use cubesql::compile::engine::df::scan::{ + convert_transport_response, transform_response, MemberField, RecordBatch, SchemaRef, +}; use cubesql::compile::engine::df::wrapper::SqlQuery; use cubesql::transport::{ SpanId, SqlGenerator, SqlResponse, TransportLoadRequestQuery, TransportLoadResponse, @@ -20,14 +30,6 @@ use serde::Serialize; use std::sync::Arc; use uuid::Uuid; -use crate::auth::NativeAuthContext; -use crate::channel::{call_raw_js_with_channel_as_callback, NodeSqlGenerator}; -use crate::node_obj_serializer::NodeObjSerializer; -use crate::{ - auth::TransportRequest, channel::call_js_with_channel_as_callback, - stream::call_js_with_stream_as_callback, -}; - #[derive(Debug)] pub struct NodeBridgeTransport { channel: Arc, @@ -334,7 +336,9 @@ impl TransportService for NodeBridgeTransport { sql_query: Option, ctx: AuthContextRef, meta: LoadRequestMeta, - ) -> Result { + schema: SchemaRef, + member_fields: Vec, + ) -> Result, CubeError> { trace!("[transport] Request ->"); let native_auth = ctx @@ -369,54 +373,115 @@ impl TransportService for NodeBridgeTransport { streaming: false, })?; - let result = call_js_with_channel_as_callback( + let result = call_raw_js_with_channel_as_callback( self.channel.clone(), self.on_sql_api_load.clone(), - Some(extra), + extra, + Box::new(|cx, v| Ok(cx.string(v).as_value(cx))), + Box::new(move |cx, v| { + if let Ok(js_result_wrapped) = v.downcast::(cx) { + let get_results_js_method: Handle = + js_result_wrapped.get(cx, "getResults").map_cube_err( + "Can't get getResults() method from JS ResultWrapper object", + )?; + + let results = get_results_js_method + .call(cx, js_result_wrapped.upcast::(), []) + .map_cube_err( + "Error calling getResults() method of ResultWrapper object", + )?; + + let js_res_wrapped_vec = results + .downcast::(cx) + .map_cube_err("Can't downcast JS result to array")? + .to_vec(cx) + .map_cube_err("Can't convert JS result to array")?; + + let native_wrapped_results = js_res_wrapped_vec + .iter() + .map(|r| ResultWrapper::from_js_result_wrapper(cx, *r)) + .collect::, _>>() + .map_cube_err( + "Can't construct result wrapper from JS ResultWrapper object", + )?; + + Ok(ValueFromJs::ResultWrapper(native_wrapped_results)) + } else if let Ok(str) = v.downcast::(cx) { + Ok(ValueFromJs::String(str.value(cx))) + } else { + Err(CubeError::internal( + "Can't downcast callback argument to string or resultWrapper object" + .to_string(), + )) + } + }), ) .await; + if let Err(e) = &result { if e.message.to_lowercase().contains("continue wait") { continue; } } - let response: serde_json::Value = result?; - - #[cfg(debug_assertions)] - trace!("[transport] Request <- {:?}", response); - #[cfg(not(debug_assertions))] - trace!("[transport] Request <- "); - - if let Some(error_value) = response.get("error") { - match error_value { - serde_json::Value::String(error) => { - if error.to_lowercase() == *"continue wait" { - debug!( + match result? { + ValueFromJs::String(result) => { + let response: serde_json::Value = match serde_json::from_str(&result) { + Ok(json) => json, + Err(err) => return Err(CubeError::internal(err.to_string())), + }; + + #[cfg(debug_assertions)] + trace!("[transport] Request <- {:?}", response); + #[cfg(not(debug_assertions))] + trace!("[transport] Request <- "); + + if let Some(error_value) = response.get("error") { + match error_value { + serde_json::Value::String(error) => { + if error.to_lowercase() == *"continue wait" { + debug!( "[transport] load - retrying request (continue wait) requestId: {}", request_id ); - continue; - } else { - return Err(CubeError::user(error.clone())); - } - } - other => { - error!( + continue; + } else { + return Err(CubeError::user(error.clone())); + } + } + other => { + error!( "[transport] load - strange response, success which contains error: {:?}", other ); - return Err(CubeError::internal( - "Error response with broken data inside".to_string(), - )); - } - } - }; + return Err(CubeError::internal( + "Error response with broken data inside".to_string(), + )); + } + } + }; - break serde_json::from_value::(response) - .map_err(|err| CubeError::user(err.to_string())); + let response = match serde_json::from_value::(response) { + Ok(v) => v, + Err(err) => { + return Err(CubeError::user(err.to_string())); + } + }; + + break convert_transport_response(response, schema.clone(), member_fields) + .map_err(|err| CubeError::user(err.to_string())); + } + ValueFromJs::ResultWrapper(result_wrappers) => { + break result_wrappers + .into_iter() + .map(|mut wrapper| { + transform_response(&mut wrapper, schema.clone(), &member_fields) + }) + .collect::, _>>(); + } + } } } diff --git a/packages/cubejs-backend-native/test/sql.test.ts b/packages/cubejs-backend-native/test/sql.test.ts index 1d55187483dd7..e417b577bc842 100644 --- a/packages/cubejs-backend-native/test/sql.test.ts +++ b/packages/cubejs-backend-native/test/sql.test.ts @@ -6,7 +6,7 @@ import * as native from '../js'; import metaFixture from './meta'; import { FakeRowStream } from './response-fake'; -const logger = jest.fn(({ event }) => { +const _logger = jest.fn(({ event }) => { if ( !event.error.includes( 'load - strange response, success which contains error' diff --git a/packages/cubejs-backend-shared/package.json b/packages/cubejs-backend-shared/package.json index 691caf3fdee2e..5e31b8527a507 100644 --- a/packages/cubejs-backend-shared/package.json +++ b/packages/cubejs-backend-shared/package.json @@ -27,7 +27,7 @@ "@types/cli-progress": "^3.9.1", "@types/decompress": "^4.2.3", "@types/jest": "^27", - "@types/node": "^12", + "@types/node": "^18", "@types/node-fetch": "^2.5.8", "@types/shelljs": "^0.8.5", "@types/throttle-debounce": "^2.1.0", diff --git a/packages/cubejs-backend-shared/src/env.ts b/packages/cubejs-backend-shared/src/env.ts index 4d4a2e97462a3..33d1854ff7c73 100644 --- a/packages/cubejs-backend-shared/src/env.ts +++ b/packages/cubejs-backend-shared/src/env.ts @@ -193,6 +193,9 @@ const variables: Record any> = { .default('1') .asInt(), nativeSqlPlanner: () => get('CUBEJS_TESSERACT_SQL_PLANNER').asBool(), + nativeOrchestrator: () => get('CUBEJS_TESSERACT_ORCHESTRATOR') + .default('false') + .asBoolStrict(), /** **************************************************************** * Common db options * diff --git a/packages/cubejs-cubestore-driver/package.json b/packages/cubejs-cubestore-driver/package.json index 125e69e55acbf..8a4a7bc8db9c2 100644 --- a/packages/cubejs-cubestore-driver/package.json +++ b/packages/cubejs-cubestore-driver/package.json @@ -29,6 +29,7 @@ "@cubejs-backend/base-driver": "1.1.16", "@cubejs-backend/cubestore": "1.1.12", "@cubejs-backend/shared": "1.1.12", + "@cubejs-backend/native": "1.1.16", "csv-write-stream": "^2.0.0", "flatbuffers": "23.3.3", "fs-extra": "^9.1.0", diff --git a/packages/cubejs-cubestore-driver/src/WebSocketConnection.ts b/packages/cubejs-cubestore-driver/src/WebSocketConnection.ts index 92da79e88ca2d..94170ed8ffd36 100644 --- a/packages/cubejs-cubestore-driver/src/WebSocketConnection.ts +++ b/packages/cubejs-cubestore-driver/src/WebSocketConnection.ts @@ -1,8 +1,9 @@ import WebSocket from 'ws'; import * as flatbuffers from 'flatbuffers'; +import { v4 as uuidv4 } from 'uuid'; import { InlineTable } from '@cubejs-backend/base-driver'; import { getEnv } from '@cubejs-backend/shared'; -import { v4 as uuidv4 } from 'uuid'; +import { parseCubestoreResultMessage } from '@cubejs-backend/native'; import { HttpCommand, HttpError, @@ -108,7 +109,7 @@ export class WebSocketConnection { this.webSocket = undefined; } }); - webSocket.on('message', (msg) => { + webSocket.on('message', async (msg) => { const buf = new flatbuffers.ByteBuffer(msg); const httpMessage = HttpMessage.getRootAsHttpMessage(buf); const resolvers = webSocket.sentMessages[httpMessage.messageId()]; @@ -116,44 +117,59 @@ export class WebSocketConnection { if (!resolvers) { throw new Error(`Cube Store missed message id: ${httpMessage.messageId()}`); // logging } - const commandType = httpMessage.commandType(); - if (commandType === HttpCommand.HttpError) { - resolvers.reject(new Error(`${httpMessage.command(new HttpError())?.error()}`)); - } else if (commandType === HttpCommand.HttpResultSet) { - const resultSet = httpMessage.command(new HttpResultSet()); - if (!resultSet) { - resolvers.reject(new Error('Empty resultSet')); - return; + + if (getEnv('nativeOrchestrator') && msg.length > 1000) { + try { + const nativeResMsg = await parseCubestoreResultMessage(msg); + resolvers.resolve(nativeResMsg); + } catch (e) { + resolvers.reject(e); } - const columnsLen = resultSet.columnsLength(); - const columns: Array = []; - for (let i = 0; i < columnsLen; i++) { - const columnName = resultSet.columns(i); - if (!columnName) { - resolvers.reject(new Error('Column name is not defined')); + } else { + const commandType = httpMessage.commandType(); + + if (commandType === HttpCommand.HttpError) { + resolvers.reject(new Error(`${httpMessage.command(new HttpError())?.error()}`)); + } else if (commandType === HttpCommand.HttpResultSet) { + const resultSet = httpMessage.command(new HttpResultSet()); + + if (!resultSet) { + resolvers.reject(new Error('Empty resultSet')); return; } - columns.push(columnName); - } - const rowLen = resultSet.rowsLength(); - const result: any[] = []; - for (let i = 0; i < rowLen; i++) { - const row = resultSet.rows(i); - if (!row) { - resolvers.reject(new Error('Null row')); - return; + + const columnsLen = resultSet.columnsLength(); + const columns: Array = []; + for (let i = 0; i < columnsLen; i++) { + const columnName = resultSet.columns(i); + if (!columnName) { + resolvers.reject(new Error('Column name is not defined')); + return; + } + columns.push(columnName); } - const valueLen = row.valuesLength(); - const rowObj = {}; - for (let j = 0; j < valueLen; j++) { - const value = row.values(j); - rowObj[columns[j]] = value?.stringValue(); + + const rowLen = resultSet.rowsLength(); + const result: any[] = []; + for (let i = 0; i < rowLen; i++) { + const row = resultSet.rows(i); + if (!row) { + resolvers.reject(new Error('Null row')); + return; + } + const valueLen = row.valuesLength(); + const rowObj = {}; + for (let j = 0; j < valueLen; j++) { + const value = row.values(j); + rowObj[columns[j]] = value?.stringValue(); + } + result.push(rowObj); } - result.push(rowObj); + + resolvers.resolve(result); + } else { + resolvers.reject(new Error('Unsupported command')); } - resolvers.resolve(result); - } else { - resolvers.reject(new Error('Unsupported command')); } }); }); diff --git a/packages/cubejs-mssql-driver/package.json b/packages/cubejs-mssql-driver/package.json index 4bd76fec90535..9ae7ce48d650c 100644 --- a/packages/cubejs-mssql-driver/package.json +++ b/packages/cubejs-mssql-driver/package.json @@ -18,7 +18,7 @@ }, "devDependencies": { "@types/mssql": "^9.1.5", - "@types/node": "^16" + "@types/node": "^18" }, "jest": { "testEnvironment": "node" diff --git a/packages/cubejs-server/src/websocket-server.ts b/packages/cubejs-server/src/websocket-server.ts index 6e4b101ccd23d..cca6c9b44c4c7 100644 --- a/packages/cubejs-server/src/websocket-server.ts +++ b/packages/cubejs-server/src/websocket-server.ts @@ -31,12 +31,26 @@ export class WebSocketServer { const connectionIdToSocket: Record = {}; - this.subscriptionServer = this.serverCore.initSubscriptionServer((connectionId: string, message: any) => { + this.subscriptionServer = this.serverCore.initSubscriptionServer(async (connectionId: string, message: any) => { if (!connectionIdToSocket[connectionId]) { throw new Error(`Socket for ${connectionId} is not found found`); } - connectionIdToSocket[connectionId].send(JSON.stringify(message)); + let messageStr: string; + + if (message.message && message.message.isWrapper) { + // In case we have a wrapped query result, we don't want to parse/stringify + // it again - it's too expensive, instead we serialize the rest of the message and then + // inject query result json into message. + const resMsg = new TextDecoder().decode(await message.message.getFinalResult()); + message.message = '~XXXXX~'; + messageStr = JSON.stringify(message); + messageStr = messageStr.replace('"~XXXXX~"', resMsg); + } else { + messageStr = JSON.stringify(message); + } + + connectionIdToSocket[connectionId].send(messageStr); }); this.wsServer.on('connection', (ws) => { diff --git a/rust/cubenativeutils/Cargo.lock b/rust/cubenativeutils/Cargo.lock index 0959a0f2888e6..59b5872a1fd3c 100644 --- a/rust/cubenativeutils/Cargo.lock +++ b/rust/cubenativeutils/Cargo.lock @@ -165,7 +165,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.95", ] [[package]] @@ -176,7 +176,7 @@ checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.95", ] [[package]] @@ -326,7 +326,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.95", "syn_derive", ] @@ -1048,7 +1048,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.95", ] [[package]] @@ -1726,7 +1726,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6813fde79b646e47e7ad75f480aa80ef76a5d9599e2717407961531169ee38b" dependencies = [ "quote", - "syn 2.0.68", + "syn 2.0.95", "syn-mid", ] @@ -1963,7 +1963,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.95", ] [[package]] @@ -2046,7 +2046,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.95", ] [[package]] @@ -2165,9 +2165,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.86" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" dependencies = [ "unicode-ident", ] @@ -2580,32 +2580,33 @@ checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" [[package]] name = "serde" -version = "1.0.209" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99fce0ffe7310761ca6bf9faf5115afbc19688edd00171d81b1bb1b116c63e09" +checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.209" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5831b979fd7b5439637af1752d535ff49f4860c0f341d1baeb6faf0f4242170" +checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.95", ] [[package]] name = "serde_json" -version = "1.0.120" +version = "1.0.135" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5" +checksum = "2b0d7ba2887406110130a978386c4e1befb98c674b4fba677954e4db976630d9" dependencies = [ "indexmap 2.2.6", "itoa", + "memchr", "ryu", "serde", ] @@ -2770,7 +2771,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.68", + "syn 2.0.95", ] [[package]] @@ -2809,9 +2810,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.68" +version = "2.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "901fa70d88b9d6c98022e23b4136f9f3e54e4662c3bc1bd1d84a42a9a0f0c1e9" +checksum = "46f71c0377baf4ef1cc3e3402ded576dccc315800fbc62dfc7fe04b009773b4a" dependencies = [ "proc-macro2", "quote", @@ -2826,7 +2827,7 @@ checksum = "b5dc35bb08dd1ca3dfb09dce91fd2d13294d6711c88897d9a9d60acf39bce049" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.95", ] [[package]] @@ -2838,7 +2839,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.95", ] [[package]] @@ -2904,7 +2905,7 @@ checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.95", ] [[package]] @@ -2972,7 +2973,7 @@ checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.95", ] [[package]] @@ -3074,7 +3075,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.95", ] [[package]] @@ -3308,7 +3309,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.95", "wasm-bindgen-shared", ] @@ -3342,7 +3343,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.95", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3596,7 +3597,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.95", ] [[package]] diff --git a/rust/cubenativeutils/rustfmt.toml b/rust/cubenativeutils/rustfmt.toml new file mode 100644 index 0000000000000..d9ba5fdb90ba3 --- /dev/null +++ b/rust/cubenativeutils/rustfmt.toml @@ -0,0 +1 @@ +imports_granularity = "Crate" \ No newline at end of file diff --git a/rust/cubenativeutils/src/wrappers/context.rs b/rust/cubenativeutils/src/wrappers/context.rs index ba880a17eff6a..98bb3ba6f7eb9 100644 --- a/rust/cubenativeutils/src/wrappers/context.rs +++ b/rust/cubenativeutils/src/wrappers/context.rs @@ -1,5 +1,4 @@ -use super::inner_types::InnerTypes; -use super::object_handle::NativeObjectHandle; +use super::{inner_types::InnerTypes, object_handle::NativeObjectHandle}; pub trait NativeContext: Clone { fn boolean(&self, v: bool) -> IT::Boolean; diff --git a/rust/cubenativeutils/src/wrappers/inner_types.rs b/rust/cubenativeutils/src/wrappers/inner_types.rs index 24586a56234a5..66bfbf136375c 100644 --- a/rust/cubenativeutils/src/wrappers/inner_types.rs +++ b/rust/cubenativeutils/src/wrappers/inner_types.rs @@ -1,7 +1,9 @@ -use super::context::NativeContext; -use super::object::{ - NativeArray, NativeBoolean, NativeFunction, NativeNumber, NativeObject, NativeString, - NativeStruct, +use super::{ + context::NativeContext, + object::{ + NativeArray, NativeBoolean, NativeFunction, NativeNumber, NativeObject, NativeString, + NativeStruct, + }, }; pub trait InnerTypes: Clone + 'static { type Object: NativeObject; diff --git a/rust/cubenativeutils/src/wrappers/neon/context.rs b/rust/cubenativeutils/src/wrappers/neon/context.rs index e641ab5c20296..96950e5e3aedb 100644 --- a/rust/cubenativeutils/src/wrappers/neon/context.rs +++ b/rust/cubenativeutils/src/wrappers/neon/context.rs @@ -1,18 +1,21 @@ //use super::object::NeonObject; -use super::inner_types::NeonInnerTypes; -use super::object::base_types::*; -use super::object::neon_array::NeonArray; -use super::object::neon_function::NeonFunction; -use super::object::neon_struct::NeonStruct; -use super::object::NeonObject; -use crate::wrappers::context::NativeContext; -use crate::wrappers::object::NativeObject; -use crate::wrappers::object_handle::NativeObjectHandle; +use super::{ + inner_types::NeonInnerTypes, + object::{ + base_types::*, neon_array::NeonArray, neon_function::NeonFunction, neon_struct::NeonStruct, + NeonObject, + }, +}; +use crate::wrappers::{ + context::NativeContext, object::NativeObject, object_handle::NativeObjectHandle, +}; use cubesql::CubeError; use neon::prelude::*; -use std::cell::{RefCell, RefMut}; -use std::marker::PhantomData; -use std::rc::{Rc, Weak}; +use std::{ + cell::{RefCell, RefMut}, + marker::PhantomData, + rc::{Rc, Weak}, +}; pub struct ContextWrapper<'cx, C: Context<'cx>> { cx: C, lifetime: PhantomData<&'cx ()>, diff --git a/rust/cubenativeutils/src/wrappers/neon/inner_types.rs b/rust/cubenativeutils/src/wrappers/neon/inner_types.rs index ebdfdf6f6279f..8bf32d0be843e 100644 --- a/rust/cubenativeutils/src/wrappers/neon/inner_types.rs +++ b/rust/cubenativeutils/src/wrappers/neon/inner_types.rs @@ -1,9 +1,10 @@ -use super::context::ContextHolder; -use super::object::base_types::*; -use super::object::neon_array::NeonArray; -use super::object::neon_function::NeonFunction; -use super::object::neon_struct::NeonStruct; -use super::object::NeonObject; +use super::{ + context::ContextHolder, + object::{ + base_types::*, neon_array::NeonArray, neon_function::NeonFunction, neon_struct::NeonStruct, + NeonObject, + }, +}; use crate::wrappers::inner_types::InnerTypes; use neon::prelude::*; use std::marker::PhantomData; diff --git a/rust/cubenativeutils/src/wrappers/neon/object/mod.rs b/rust/cubenativeutils/src/wrappers/neon/object/mod.rs index cb263bfeb6ea3..40763a228b97d 100644 --- a/rust/cubenativeutils/src/wrappers/neon/object/mod.rs +++ b/rust/cubenativeutils/src/wrappers/neon/object/mod.rs @@ -3,13 +3,14 @@ pub mod neon_array; pub mod neon_function; pub mod neon_struct; -use self::base_types::{NeonBoolean, NeonNumber, NeonString}; -use self::neon_array::NeonArray; -use self::neon_function::NeonFunction; -use self::neon_struct::NeonStruct; +use self::{ + base_types::{NeonBoolean, NeonNumber, NeonString}, + neon_array::NeonArray, + neon_function::NeonFunction, + neon_struct::NeonStruct, +}; use super::inner_types::NeonInnerTypes; -use crate::wrappers::neon::context::ContextHolder; -use crate::wrappers::object::NativeObject; +use crate::wrappers::{neon::context::ContextHolder, object::NativeObject}; use cubesql::CubeError; use neon::prelude::*; diff --git a/rust/cubenativeutils/src/wrappers/neon/object/neon_array.rs b/rust/cubenativeutils/src/wrappers/neon/object/neon_array.rs index 85bf544555b1a..dc06714227c61 100644 --- a/rust/cubenativeutils/src/wrappers/neon/object/neon_array.rs +++ b/rust/cubenativeutils/src/wrappers/neon/object/neon_array.rs @@ -1,7 +1,9 @@ use super::NeonObject; -use crate::wrappers::neon::inner_types::NeonInnerTypes; -use crate::wrappers::object::{NativeArray, NativeObject, NativeType}; -use crate::wrappers::object_handle::NativeObjectHandle; +use crate::wrappers::{ + neon::inner_types::NeonInnerTypes, + object::{NativeArray, NativeObject, NativeType}, + object_handle::NativeObjectHandle, +}; use cubesql::CubeError; use neon::prelude::*; diff --git a/rust/cubenativeutils/src/wrappers/neon/object/neon_function.rs b/rust/cubenativeutils/src/wrappers/neon/object/neon_function.rs index 78f663ff365a9..c8fc68d372d50 100644 --- a/rust/cubenativeutils/src/wrappers/neon/object/neon_function.rs +++ b/rust/cubenativeutils/src/wrappers/neon/object/neon_function.rs @@ -1,7 +1,9 @@ use super::NeonObject; -use crate::wrappers::neon::inner_types::NeonInnerTypes; -use crate::wrappers::object::{NativeFunction, NativeType}; -use crate::wrappers::object_handle::NativeObjectHandle; +use crate::wrappers::{ + neon::inner_types::NeonInnerTypes, + object::{NativeFunction, NativeType}, + object_handle::NativeObjectHandle, +}; use cubesql::CubeError; use lazy_static::lazy_static; use neon::prelude::*; diff --git a/rust/cubenativeutils/src/wrappers/neon/object/neon_struct.rs b/rust/cubenativeutils/src/wrappers/neon/object/neon_struct.rs index 7815a19f306f0..693c19aeb1da3 100644 --- a/rust/cubenativeutils/src/wrappers/neon/object/neon_struct.rs +++ b/rust/cubenativeutils/src/wrappers/neon/object/neon_struct.rs @@ -1,7 +1,9 @@ use super::NeonObject; -use crate::wrappers::neon::inner_types::NeonInnerTypes; -use crate::wrappers::object::{NativeStruct, NativeType}; -use crate::wrappers::object_handle::NativeObjectHandle; +use crate::wrappers::{ + neon::inner_types::NeonInnerTypes, + object::{NativeStruct, NativeType}, + object_handle::NativeObjectHandle, +}; use cubesql::CubeError; use neon::prelude::*; diff --git a/rust/cubenativeutils/src/wrappers/object.rs b/rust/cubenativeutils/src/wrappers/object.rs index f901ededed639..bee3275ea3aac 100644 --- a/rust/cubenativeutils/src/wrappers/object.rs +++ b/rust/cubenativeutils/src/wrappers/object.rs @@ -1,5 +1,4 @@ -use super::inner_types::InnerTypes; -use super::object_handle::NativeObjectHandle; +use super::{inner_types::InnerTypes, object_handle::NativeObjectHandle}; use cubesql::CubeError; pub trait NativeObject: Clone { diff --git a/rust/cubenativeutils/src/wrappers/object_handle.rs b/rust/cubenativeutils/src/wrappers/object_handle.rs index 7dffd313b8b96..501c16e356be8 100644 --- a/rust/cubenativeutils/src/wrappers/object_handle.rs +++ b/rust/cubenativeutils/src/wrappers/object_handle.rs @@ -1,5 +1,4 @@ -use super::inner_types::InnerTypes; -use super::object::NativeObject; +use super::{inner_types::InnerTypes, object::NativeObject}; use cubesql::CubeError; #[derive(Clone)] diff --git a/rust/cubenativeutils/src/wrappers/serializer/deserialize.rs b/rust/cubenativeutils/src/wrappers/serializer/deserialize.rs index 7576e9578e733..3f38afab3a592 100644 --- a/rust/cubenativeutils/src/wrappers/serializer/deserialize.rs +++ b/rust/cubenativeutils/src/wrappers/serializer/deserialize.rs @@ -1,7 +1,8 @@ use super::deserializer::NativeSerdeDeserializer; -use crate::wrappers::inner_types::InnerTypes; -use crate::wrappers::NativeObjectHandle; -use crate::CubeError; +use crate::{ + wrappers::{inner_types::InnerTypes, NativeObjectHandle}, + CubeError, +}; use serde::de::DeserializeOwned; pub trait NativeDeserialize: Sized { diff --git a/rust/cubenativeutils/src/wrappers/serializer/deserializer.rs b/rust/cubenativeutils/src/wrappers/serializer/deserializer.rs index d98984fa2412e..9f3cc78582723 100644 --- a/rust/cubenativeutils/src/wrappers/serializer/deserializer.rs +++ b/rust/cubenativeutils/src/wrappers/serializer/deserializer.rs @@ -1,13 +1,14 @@ use super::error::NativeObjSerializerError; -use crate::wrappers::inner_types::InnerTypes; -use crate::wrappers::object::{ - NativeArray, NativeBoolean, NativeNumber, NativeString, NativeStruct, +use crate::wrappers::{ + inner_types::InnerTypes, + object::{NativeArray, NativeBoolean, NativeNumber, NativeString, NativeStruct}, + object_handle::NativeObjectHandle, +}; +use serde::{ + self, + de::{DeserializeOwned, DeserializeSeed, MapAccess, SeqAccess, Visitor}, + forward_to_deserialize_any, Deserializer, }; -use crate::wrappers::object_handle::NativeObjectHandle; -use serde; -use serde::de::{DeserializeOwned, DeserializeSeed, MapAccess, SeqAccess, Visitor}; -use serde::forward_to_deserialize_any; -use serde::Deserializer; pub struct NativeSerdeDeserializer { input: NativeObjectHandle, diff --git a/rust/cubenativeutils/src/wrappers/serializer/error.rs b/rust/cubenativeutils/src/wrappers/serializer/error.rs index 219b219ca7050..50e55c88da9f4 100644 --- a/rust/cubenativeutils/src/wrappers/serializer/error.rs +++ b/rust/cubenativeutils/src/wrappers/serializer/error.rs @@ -1,6 +1,5 @@ use serde::{de, ser}; -use std::fmt; -use std::fmt::Display; +use std::{fmt, fmt::Display}; #[derive(Debug)] pub enum NativeObjSerializerError { Message(String), diff --git a/rust/cubenativeutils/src/wrappers/serializer/serialize.rs b/rust/cubenativeutils/src/wrappers/serializer/serialize.rs index 1dce7a117d432..e08ef8284053f 100644 --- a/rust/cubenativeutils/src/wrappers/serializer/serialize.rs +++ b/rust/cubenativeutils/src/wrappers/serializer/serialize.rs @@ -1,7 +1,8 @@ use super::serializer::NativeSerdeSerializer; -use crate::wrappers::inner_types::InnerTypes; -use crate::wrappers::{NativeContextHolder, NativeObjectHandle}; -use crate::CubeError; +use crate::{ + wrappers::{inner_types::InnerTypes, NativeContextHolder, NativeObjectHandle}, + CubeError, +}; use serde::Serialize; pub trait NativeSerialize { diff --git a/rust/cubenativeutils/src/wrappers/serializer/serializer.rs b/rust/cubenativeutils/src/wrappers/serializer/serializer.rs index a1e04657f977c..21463875cddd2 100644 --- a/rust/cubenativeutils/src/wrappers/serializer/serializer.rs +++ b/rust/cubenativeutils/src/wrappers/serializer/serializer.rs @@ -1,7 +1,8 @@ use super::error::NativeObjSerializerError; -use crate::wrappers::inner_types::InnerTypes; -use crate::wrappers::NativeContextHolder; -use crate::wrappers::{NativeArray, NativeObjectHandle, NativeString, NativeStruct, NativeType}; +use crate::wrappers::{ + inner_types::InnerTypes, NativeArray, NativeContextHolder, NativeObjectHandle, NativeString, + NativeStruct, NativeType, +}; use serde::{ser, Serialize}; pub struct NativeSerdeSerializer { diff --git a/rust/cubeorchestrator/.gitignore b/rust/cubeorchestrator/.gitignore new file mode 100644 index 0000000000000..2a0a960cb1ec5 --- /dev/null +++ b/rust/cubeorchestrator/.gitignore @@ -0,0 +1,3 @@ +/target +/.idea +.vscode diff --git a/rust/cubeorchestrator/CHANGELOG.md b/rust/cubeorchestrator/CHANGELOG.md new file mode 100644 index 0000000000000..f05ecfa971fe9 --- /dev/null +++ b/rust/cubeorchestrator/CHANGELOG.md @@ -0,0 +1,2 @@ +# ChangeLog + diff --git a/rust/cubeorchestrator/Cargo.lock b/rust/cubeorchestrator/Cargo.lock new file mode 100644 index 0000000000000..f4a1156cfff88 --- /dev/null +++ b/rust/cubeorchestrator/Cargo.lock @@ -0,0 +1,537 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "addr2line" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61f2b7f93d2c7d2b08263acaa4a363b3e276806c68af6134c44f523bf1aacd" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c95c10ba0b00a02636238b814946408b1322d5ac4760326e6fb8ec956d85775" + +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + +[[package]] +name = "backtrace" +version = "0.3.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a905d892734eea339e896738c14b9afce22b5318f64b951e70bf3844419b01" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bumpalo" +version = "3.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" + +[[package]] +name = "cc" +version = "1.0.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17f6e324229dc011159fcc089755d1e2e216a90d43a7dea6853ca740b84f35e7" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-targets", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cubeorchestrator" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "cubeshared", + "itertools", + "neon", + "serde", + "serde_json", +] + +[[package]] +name = "cubeshared" +version = "0.1.0" +dependencies = [ + "flatbuffers", +] + +[[package]] +name = "either" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" + +[[package]] +name = "flatbuffers" +version = "23.5.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dac53e22462d78c16d64a1cd22371b54cc3fe94aa15e7886a2fa6e5d1ab8640" +dependencies = [ + "bitflags", + "rustc_version", +] + +[[package]] +name = "gimli" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0a01e0497841a3b2db4f8afa483cce65f7e96a3498bd6c541734792aeac8fe7" + +[[package]] +name = "iana-time-zone" +version = "0.1.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" + +[[package]] +name = "js-sys" +version = "0.3.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a865e038f7f6ed956f788f0d7d60c541fff74c7bd74272c5d4cf15c63743e705" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.153" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" + +[[package]] +name = "libloading" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c2a198fb6b0eada2a8df47933734e6d35d350665a33a3593d7164fa52c75c19" +dependencies = [ + "cfg-if", + "windows-targets", +] + +[[package]] +name = "log" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" + +[[package]] +name = "memchr" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc" + +[[package]] +name = "miniz_oxide" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" +dependencies = [ + "adler", + "autocfg", +] + +[[package]] +name = "neon" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d75440242411c87dc39847b0e33e961ec1f10326a9d8ecf9c1ea64a3b3c13dc" +dependencies = [ + "libloading", + "neon-macros", + "once_cell", + "semver", + "send_wrapper", + "smallvec", + "tokio", +] + +[[package]] +name = "neon-macros" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6813fde79b646e47e7ad75f480aa80ef76a5d9599e2717407961531169ee38b" +dependencies = [ + "quote", + "syn", + "syn-mid", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "object" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c55827317fb4c08822499848a14237d2874d6f139828893017237e7ab93eb386" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "pin-project-lite" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" + +[[package]] +name = "proc-macro2" +version = "1.0.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dead70b0b5e03e9c814bcb6b01e03e68f7c57a80aa48c72ec92152ab3e818d49" + +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver", +] + +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + +[[package]] +name = "semver" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" + +[[package]] +name = "send_wrapper" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" + +[[package]] +name = "serde" +version = "1.0.215" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.215" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.133" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + +[[package]] +name = "syn" +version = "2.0.90" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn-mid" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5dc35bb08dd1ca3dfb09dce91fd2d13294d6711c88897d9a9d60acf39bce049" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio" +version = "1.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787" +dependencies = [ + "backtrace", + "pin-project-lite", +] + +[[package]] +name = "unicode-ident" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" + +[[package]] +name = "wasm-bindgen" +version = "0.2.97" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d15e63b4482863c109d70a7b8706c1e364eb6ea449b201a76c5b89cedcec2d5c" +dependencies = [ + "cfg-if", + "once_cell", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.97" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d36ef12e3aaca16ddd3f67922bc63e48e953f126de60bd33ccc0101ef9998cd" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.97" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "705440e08b42d3e4b36de7d66c944be628d579796b8090bfa3471478a2260051" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.97" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98c9ae5a76e46f4deecd0f0255cc223cfa18dc9b261213b8aa0c7b36f61b3f1d" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.97" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ee99da9c5ba11bd675621338ef6fa52296b76b83305e9b6e5c77d4c286d6d49" + +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" diff --git a/rust/cubeorchestrator/Cargo.toml b/rust/cubeorchestrator/Cargo.toml new file mode 100644 index 0000000000000..0605d4608ae08 --- /dev/null +++ b/rust/cubeorchestrator/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "cubeorchestrator" +version = "0.1.0" +edition = "2021" + +[dependencies] +chrono = { version = "0.4.31", features = ["serde"] } +cubeshared = { path = "../cubeshared" } +serde = { version = "1.0.215", features = ["derive"] } +serde_json = "1.0.133" +anyhow = "1.0" +itertools = "0.13.0" + +[dependencies.neon] +version = "=1" +default-features = false +features = ["napi-1", "napi-4", "napi-6", "futures"] diff --git a/rust/cubeorchestrator/rust-toolchain.toml b/rust/cubeorchestrator/rust-toolchain.toml new file mode 100644 index 0000000000000..040357e9b1d43 --- /dev/null +++ b/rust/cubeorchestrator/rust-toolchain.toml @@ -0,0 +1,5 @@ +[toolchain] +#channel = "stable" +channel = "nightly-2024-07-15" +components = ["rustfmt", "clippy"] +profile = "minimal" diff --git a/rust/cubeorchestrator/rustfmt.toml b/rust/cubeorchestrator/rustfmt.toml new file mode 100644 index 0000000000000..c3c8c37533810 --- /dev/null +++ b/rust/cubeorchestrator/rustfmt.toml @@ -0,0 +1 @@ +imports_granularity = "Crate" diff --git a/rust/cubeorchestrator/src/lib.rs b/rust/cubeorchestrator/src/lib.rs new file mode 100644 index 0000000000000..03f0453c16db7 --- /dev/null +++ b/rust/cubeorchestrator/src/lib.rs @@ -0,0 +1,3 @@ +pub mod query_message_parser; +pub mod query_result_transform; +pub mod transport; diff --git a/rust/cubeorchestrator/src/query_message_parser.rs b/rust/cubeorchestrator/src/query_message_parser.rs new file mode 100644 index 0000000000000..8d406906ceda1 --- /dev/null +++ b/rust/cubeorchestrator/src/query_message_parser.rs @@ -0,0 +1,146 @@ +use crate::{ + query_result_transform::{DBResponsePrimitive, DBResponseValue}, + transport::JsRawData, +}; +use cubeshared::codegen::{root_as_http_message, HttpCommand}; +use neon::prelude::Finalize; +use std::collections::HashMap; + +#[derive(Debug)] +pub enum ParseError { + UnsupportedCommand, + EmptyResultSet, + NullRow, + ColumnNameNotDefined, + FlatBufferError, + ErrorMessage(String), +} + +impl std::fmt::Display for ParseError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ParseError::UnsupportedCommand => write!(f, "Unsupported command"), + ParseError::EmptyResultSet => write!(f, "Empty resultSet"), + ParseError::NullRow => write!(f, "Null row"), + ParseError::ColumnNameNotDefined => write!(f, "Column name is not defined"), + ParseError::FlatBufferError => write!(f, "FlatBuffer parsing error"), + ParseError::ErrorMessage(msg) => write!(f, "Error: {}", msg), + } + } +} + +impl std::error::Error for ParseError {} + +#[derive(Debug, Clone)] +pub struct QueryResult { + pub columns: Vec, + pub rows: Vec>, + pub columns_pos: HashMap, +} + +impl Finalize for QueryResult {} + +impl QueryResult { + pub fn from_cubestore_fb(msg_data: &[u8]) -> Result { + let mut result = QueryResult { + columns: vec![], + rows: vec![], + columns_pos: HashMap::new(), + }; + + let http_message = + root_as_http_message(msg_data).map_err(|_| ParseError::FlatBufferError)?; + + match http_message.command_type() { + HttpCommand::HttpError => { + let http_error = http_message + .command_as_http_error() + .ok_or(ParseError::FlatBufferError)?; + let error_message = http_error.error().unwrap_or("Unknown error").to_string(); + Err(ParseError::ErrorMessage(error_message)) + } + HttpCommand::HttpResultSet => { + let result_set = http_message + .command_as_http_result_set() + .ok_or(ParseError::EmptyResultSet)?; + + if let Some(result_set_columns) = result_set.columns() { + if result_set_columns.iter().any(|c| c.is_empty()) { + return Err(ParseError::ColumnNameNotDefined); + } + + let (columns, columns_pos): (Vec<_>, HashMap<_, _>) = result_set_columns + .iter() + .enumerate() + .map(|(index, column_name)| { + (column_name.to_owned(), (column_name.to_owned(), index)) + }) + .unzip(); + + result.columns = columns; + result.columns_pos = columns_pos; + } + + if let Some(result_set_rows) = result_set.rows() { + result.rows = Vec::with_capacity(result_set_rows.len()); + + for row in result_set_rows.iter() { + let values = row.values().ok_or(ParseError::NullRow)?; + let row_obj: Vec<_> = values + .iter() + .map(|val| { + DBResponseValue::Primitive(DBResponsePrimitive::String( + val.string_value().unwrap_or("").to_owned(), + )) + }) + .collect(); + + result.rows.push(row_obj); + } + } + + Ok(result) + } + _ => Err(ParseError::UnsupportedCommand), + } + } + + pub fn from_js_raw_data(js_raw_data: JsRawData) -> Result { + if js_raw_data.is_empty() { + return Ok(QueryResult { + columns: vec![], + rows: vec![], + columns_pos: HashMap::new(), + }); + } + + let first_row = &js_raw_data[0]; + let columns: Vec = first_row.keys().cloned().collect(); + let columns_pos: HashMap = columns + .iter() + .enumerate() + .map(|(index, column)| (column.clone(), index)) + .collect(); + + let rows: Vec> = js_raw_data + .into_iter() + .map(|row_map| { + columns + .iter() + .map(|col| { + row_map + .get(col) + .map(|val| DBResponseValue::Primitive(val.clone())) + .unwrap_or(DBResponseValue::Primitive(DBResponsePrimitive::Null)) + }) + .collect() + }) + .collect(); + + Ok(QueryResult { + columns, + rows, + columns_pos, + }) + } +} diff --git a/rust/cubeorchestrator/src/query_result_transform.rs b/rust/cubeorchestrator/src/query_result_transform.rs new file mode 100644 index 0000000000000..22e18a889327b --- /dev/null +++ b/rust/cubeorchestrator/src/query_result_transform.rs @@ -0,0 +1,2500 @@ +use crate::{ + query_message_parser::QueryResult, + transport::{ + AnnotatedConfigItem, ConfigItem, MemberOrMemberExpression, MembersMap, NormalizedQuery, + QueryTimeDimension, QueryType, ResultType, TransformDataRequest, + }, +}; +use anyhow::{bail, Context, Result}; +use chrono::{DateTime, NaiveDateTime, TimeZone, Utc}; +use itertools::multizip; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::{ + collections::{HashMap, HashSet}, + fmt::Display, + sync::Arc, +}; + +pub const COMPARE_DATE_RANGE_FIELD: &str = "compareDateRange"; +pub const COMPARE_DATE_RANGE_SEPARATOR: &str = " - "; +pub const BLENDING_QUERY_KEY_PREFIX: &str = "time."; +pub const BLENDING_QUERY_RES_SEPARATOR: &str = "."; +pub const MEMBER_SEPARATOR: &str = "."; + +/// Transform specified `value` with specified `type` to the network protocol type. +pub fn transform_value(value: DBResponseValue, type_: &str) -> DBResponsePrimitive { + match value { + DBResponseValue::DateTime(dt) if type_ == "time" || type_.is_empty() => { + DBResponsePrimitive::String( + dt.with_timezone(&Utc) + .format("%Y-%m-%dT%H:%M:%S%.3f") + .to_string(), + ) + } + DBResponseValue::Primitive(DBResponsePrimitive::String(ref s)) if type_ == "time" => { + let formatted = DateTime::parse_from_rfc3339(s) + .map(|dt| dt.format("%Y-%m-%dT%H:%M:%S%.3f").to_string()) + .or_else(|_| { + NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S%.3f").map(|dt| { + Utc.from_utc_datetime(&dt) + .format("%Y-%m-%dT%H:%M:%S%.3f") + .to_string() + }) + }) + .or_else(|_| { + NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S").map(|dt| { + Utc.from_utc_datetime(&dt) + .format("%Y-%m-%dT%H:%M:%S%.3f") + .to_string() + }) + }) + .or_else(|_| { + NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S").map(|dt| { + Utc.from_utc_datetime(&dt) + .format("%Y-%m-%dT%H:%M:%S%.3f") + .to_string() + }) + }) + .or_else(|_| { + NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S%.3f %Z").map(|dt| { + Utc.from_utc_datetime(&dt) + .format("%Y-%m-%dT%H:%M:%S%.3f") + .to_string() + }) + }) + .or_else(|_| { + NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S%.3f %:z").map(|dt| { + Utc.from_utc_datetime(&dt) + .format("%Y-%m-%dT%H:%M:%S%.3f") + .to_string() + }) + }) + .unwrap_or_else(|_| s.clone()); + DBResponsePrimitive::String(formatted) + } + DBResponseValue::Primitive(p) => p, + DBResponseValue::Object { value } => value, + _ => DBResponsePrimitive::Null, + } +} + +/// Parse date range value from time dimension. +pub fn get_date_range_value( + time_dimensions: Option<&Vec>, +) -> Result { + let time_dimensions = match time_dimensions { + Some(time_dimensions) => time_dimensions, + None => bail!("QueryTimeDimension should be specified for the compare date range query."), + }; + + let dim = match time_dimensions.first() { + Some(dim) => dim, + None => bail!("No time dimension provided."), + }; + + let date_range: &Vec = match &dim.date_range { + Some(date_range) => date_range, + None => bail!("Inconsistent QueryTimeDimension configuration: dateRange required."), + }; + + if date_range.len() == 1 { + bail!( + "Inconsistent dateRange configuration for the compare date range query: {}", + date_range[0] + ); + } + + Ok(DBResponsePrimitive::String( + date_range.join(COMPARE_DATE_RANGE_SEPARATOR), + )) +} + +/// Parse blending query key from time dimension for query. +pub fn get_blending_query_key(time_dimensions: Option<&Vec>) -> Result { + let dim = time_dimensions + .and_then(|dims| dims.first().cloned()) + .context("QueryTimeDimension should be specified for the blending query.")?; + + let granularity = dim + .granularity.clone() + .context(format!( + "Inconsistent QueryTimeDimension configuration for the blending query, granularity required: {:?}", + dim + ))?; + + Ok(format!("{}{}", BLENDING_QUERY_KEY_PREFIX, granularity)) +} + +/// Parse blending query key from time dimension for response. +pub fn get_blending_response_key( + time_dimensions: Option<&Vec>, +) -> Result { + let dim = time_dimensions + .and_then(|dims| dims.first().cloned()) + .context("QueryTimeDimension should be specified for the blending query.")?; + + let granularity = dim + .granularity.clone() + .context(format!( + "Inconsistent QueryTimeDimension configuration for the blending query, granularity required: {:?}", + dim + ))?; + + let dimension = dim.dimension.clone(); + + Ok(format!( + "{}{}{}", + dimension, BLENDING_QUERY_RES_SEPARATOR, granularity + )) +} + +/// Parse member names from request/response. +pub fn get_members( + query_type: &QueryType, + query: &NormalizedQuery, + db_data: &QueryResult, + alias_to_member_name_map: &HashMap, + annotation: &HashMap, +) -> Result<(MembersMap, Vec)> { + let mut members_map: MembersMap = HashMap::new(); + // Hashmaps don't guarantee the order of the elements while iterating + // this fires in get_compact_row because members map doesn't hold the members for + // date range queries, which are added later and thus columns in final recordset are not + // in sync with the order of members in members list. + let mut members_arr: Vec = vec![]; + + if db_data.columns.is_empty() { + return Ok((members_map, members_arr)); + } + + for column in db_data.columns.iter() { + let member_name = alias_to_member_name_map + .get(column) + .context(format!("Member name not found for alias: '{}'", column))?; + + if !annotation.contains_key(member_name) { + bail!( + concat!( + "You requested hidden member: '{}'. Please make it visible using `shown: true`. ", + "Please note primaryKey fields are `shown: false` by default: ", + "https://cube.dev/docs/schema/reference/joins#setting-a-primary-key." + ), + column + ); + } + + members_map.insert(member_name.clone(), column.clone()); + members_arr.push(member_name.clone()); + + let path = member_name.split(MEMBER_SEPARATOR).collect::>(); + let calc_member = format!("{}{}{}", path[0], MEMBER_SEPARATOR, path[1]); + + if path.len() == 3 + && query.dimensions.as_ref().map_or(true, |dims| { + !dims + .iter() + .any(|dim| *dim == MemberOrMemberExpression::Member(calc_member.clone())) + }) + { + members_map.insert(calc_member.clone(), column.clone()); + members_arr.push(calc_member); + } + } + + match query_type { + QueryType::CompareDateRangeQuery => { + members_map.insert( + COMPARE_DATE_RANGE_FIELD.to_string(), + QueryType::CompareDateRangeQuery.to_string(), + ); + members_arr.push(COMPARE_DATE_RANGE_FIELD.to_string()); + } + QueryType::BlendingQuery => { + let blending_key = get_blending_query_key(query.time_dimensions.as_ref()) + .context("Failed to generate blending query key")?; + if let Some(dim) = query + .time_dimensions + .as_ref() + .and_then(|dims| dims.first().cloned()) + { + let val = members_map.get(&dim.dimension).unwrap(); + members_map.insert(blending_key.clone(), val.clone()); + members_arr.push(blending_key); + } + } + _ => {} + } + + Ok((members_map, members_arr)) +} + +/// Convert DB response object to the compact output format. +pub fn get_compact_row( + members_to_alias_map: &HashMap, + annotation: &HashMap, + query_type: &QueryType, + members: &[String], + time_dimensions: Option<&Vec>, + db_row: &[DBResponseValue], + columns_pos: &HashMap, +) -> Result> { + let mut row: Vec = Vec::with_capacity(members.len()); + + for m in members { + if let Some(annotation_item) = annotation.get(m) { + if let Some(alias) = members_to_alias_map.get(m) { + if let Some(key) = columns_pos.get(alias) { + if let Some(value) = db_row.get(*key) { + let mtype = annotation_item.member_type.as_deref().unwrap_or(""); + row.push(transform_value(value.clone(), mtype)); + } + } + } + } + } + + match query_type { + QueryType::CompareDateRangeQuery => { + row.push(get_date_range_value(time_dimensions)?); + } + QueryType::BlendingQuery => { + let blending_key = get_blending_response_key(time_dimensions)?; + + if let Some(alias) = members_to_alias_map.get(&blending_key) { + if let Some(key) = columns_pos.get(alias) { + if let Some(value) = db_row.get(*key) { + let member_type = annotation.get(alias).map_or("", |annotation_item| { + annotation_item.member_type.as_deref().unwrap_or("") + }); + + row.push(transform_value(value.clone(), member_type)); + } + } + } + } + _ => {} + } + + Ok(row) +} + +/// Convert DB response object to the vanilla output format. +pub fn get_vanilla_row( + alias_to_member_name_map: &HashMap, + annotation: &HashMap, + query_type: &QueryType, + query: &NormalizedQuery, + db_row: &[DBResponseValue], + columns_pos: &HashMap, +) -> Result> { + let mut row = HashMap::new(); + + for (alias, &index) in columns_pos { + if let Some(value) = db_row.get(index) { + let member_name = match alias_to_member_name_map.get(alias) { + Some(m) => m, + None => { + bail!("Missing member name for alias: {}", alias); + } + }; + + let annotation_for_member = match annotation.get(member_name) { + Some(am) => am, + None => { + bail!( + concat!( + "You requested hidden member: '{}'. Please make it visible using `shown: true`. ", + "Please note primaryKey fields are `shown: false` by default: ", + "https://cube.dev/docs/schema/reference/joins#setting-a-primary-key." + ), + alias + ) + } + }; + + let transformed_value = transform_value( + value.clone(), + annotation_for_member + .member_type + .as_ref() + .unwrap_or(&"".to_string()), + ); + + row.insert(member_name.clone(), transformed_value.clone()); + + // Handle deprecated time dimensions without granularity + let path: Vec<&str> = member_name.split(MEMBER_SEPARATOR).collect(); + let member_name_without_granularity = + format!("{}{}{}", path[0], MEMBER_SEPARATOR, path[1]); + if path.len() == 3 + && query.dimensions.as_ref().map_or(true, |dims| { + !dims.iter().any(|dim| { + *dim == MemberOrMemberExpression::Member( + member_name_without_granularity.clone(), + ) + }) + }) + { + row.insert(member_name_without_granularity, transformed_value); + } + } + } + + match query_type { + QueryType::CompareDateRangeQuery => { + let date_range_value = get_date_range_value(query.time_dimensions.as_ref())?; + row.insert("compareDateRange".to_string(), date_range_value); + } + QueryType::BlendingQuery => { + let blending_key = get_blending_query_key(query.time_dimensions.as_ref())?; + let response_key = get_blending_response_key(query.time_dimensions.as_ref())?; + + if let Some(value) = row.get(&response_key) { + row.insert(blending_key, value.clone()); + } + } + _ => {} + } + + Ok(row) +} + +/// Helper to get a list if unique granularities from normalized queries +pub fn get_query_granularities(queries: &[&NormalizedQuery]) -> Vec { + queries + .iter() + .filter_map(|query| { + query + .time_dimensions + .as_ref() + .and_then(|tds| tds.first()) + .and_then(|td| td.granularity.clone()) + }) + .collect::>() + .into_iter() + .collect() +} + +/// Get Pivot Query for a list of queries +pub fn get_pivot_query( + query_type: &QueryType, + queries: &Vec<&NormalizedQuery>, +) -> Result { + let mut pivot_query = queries + .first() + .copied() + .cloned() + .ok_or_else(|| anyhow::anyhow!("Queries list cannot be empty"))?; + + match query_type { + QueryType::BlendingQuery => { + // Merge and deduplicate measures and dimensions across all queries + let mut merged_measures = HashSet::new(); + let mut merged_dimensions = HashSet::new(); + + for query in queries { + if let Some(measures) = &query.measures { + merged_measures.extend(measures.iter().cloned()); + } + if let Some(dimensions) = &query.dimensions { + merged_dimensions.extend(dimensions.iter().cloned()); + } + } + + pivot_query.measures = if !merged_measures.is_empty() { + Some(merged_measures.into_iter().collect()) + } else { + None + }; + pivot_query.dimensions = if !merged_dimensions.is_empty() { + Some(merged_dimensions.into_iter().collect()) + } else { + None + }; + + // Add time dimensions + let granularities = get_query_granularities(queries); + if !granularities.is_empty() { + pivot_query.time_dimensions = Some(vec![QueryTimeDimension { + dimension: "time".to_string(), + date_range: None, + compare_date_range: None, + granularity: granularities.first().cloned(), + }]); + } + } + QueryType::CompareDateRangeQuery => { + let mut dimensions = vec![MemberOrMemberExpression::Member( + "compareDateRange".to_string(), + )]; + if let Some(dims) = pivot_query.dimensions { + dimensions.extend(dims.clone()); + } + pivot_query.dimensions = Option::from(dimensions); + } + _ => {} + } + + pivot_query.query_type = Option::from(query_type.clone()); + + Ok(pivot_query) +} + +pub fn get_final_cubestore_result_array( + transform_requests: &[TransformDataRequest], + cube_store_results: &[Arc], + result_data: &mut [RequestResultData], +) -> Result<()> { + for (transform_data, cube_store_result, result) in multizip(( + transform_requests.iter(), + cube_store_results.iter(), + result_data.iter_mut(), + )) { + result.prepare_results(transform_data, cube_store_result)?; + } + + Ok(()) +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(untagged)] +pub enum TransformedData { + Compact { + members: Vec, + dataset: Vec>, + }, + Vanilla(Vec>), +} + +impl TransformedData { + /// Transforms queried data array to the output format. + pub fn transform( + request_data: &TransformDataRequest, + cube_store_result: &QueryResult, + ) -> Result { + let alias_to_member_name_map = &request_data.alias_to_member_name_map; + let annotation = &request_data.annotation; + let query = &request_data.query; + let query_type = &request_data.query_type.clone().unwrap_or_default(); + let res_type = request_data.res_type.clone(); + + let (members_to_alias_map, members) = get_members( + query_type, + query, + cube_store_result, + alias_to_member_name_map, + annotation, + )?; + + match res_type { + Some(ResultType::Compact) => { + let dataset: Vec<_> = cube_store_result + .rows + .iter() + .map(|row| { + get_compact_row( + &members_to_alias_map, + annotation, + query_type, + &members, + query.time_dimensions.as_ref(), + row, + &cube_store_result.columns_pos, + ) + }) + .collect::>>()?; + Ok(TransformedData::Compact { members, dataset }) + } + _ => { + let dataset: Vec<_> = cube_store_result + .rows + .iter() + .map(|row| { + get_vanilla_row( + alias_to_member_name_map, + annotation, + query_type, + query, + row, + &cube_store_result.columns_pos, + ) + }) + .collect::>>()?; + Ok(TransformedData::Vanilla(dataset)) + } + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RequestResultDataMulti { + pub query_type: QueryType, + pub results: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub pivot_query: Option, + pub slow_query: bool, +} + +impl RequestResultDataMulti { + /// Processes multiple results and populates the final `RequestResultDataMulti` structure + /// which is sent to the client. + pub fn prepare_results( + &mut self, + request_data: &[TransformDataRequest], + cube_store_result: &[Arc], + ) -> Result<()> { + for (transform_data, cube_store_result, result) in multizip(( + request_data.iter(), + cube_store_result.iter(), + self.results.iter_mut(), + )) { + result.prepare_results(transform_data, cube_store_result)?; + } + + let normalized_queries = self + .results + .iter() + .map(|result| &result.query) + .collect::>(); + + self.pivot_query = Some(get_pivot_query(&self.query_type, &normalized_queries)?); + + Ok(()) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RequestResultData { + pub query: NormalizedQuery, + #[serde(skip_serializing_if = "Option::is_none")] + pub last_refresh_time: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub refresh_key_values: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub used_pre_aggregations: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub transformed_query: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub request_id: Option, + pub annotation: HashMap>, + #[serde(skip_serializing_if = "Option::is_none")] + pub data_source: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub db_type: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub ext_db_type: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub external: Option, + pub slow_query: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub total: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub data: Option, +} + +impl RequestResultData { + /// Populates the `RequestResultData` structure with the transformed Query result. + pub fn prepare_results( + &mut self, + request_data: &TransformDataRequest, + cube_store_result: &QueryResult, + ) -> Result<()> { + let transformed = TransformedData::transform(request_data, cube_store_result)?; + self.data = Some(transformed); + + Ok(()) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RequestResultArray { + pub results: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(untagged)] +pub enum DBResponsePrimitive { + Null, + Boolean(bool), + Number(f64), + String(String), +} + +impl Display for DBResponsePrimitive { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let str = match self { + DBResponsePrimitive::Null => "null".to_string(), + DBResponsePrimitive::Boolean(b) => b.to_string(), + DBResponsePrimitive::Number(n) => n.to_string(), + DBResponsePrimitive::String(s) => s.clone(), + }; + write!(f, "{}", str) + } +} + +#[derive(Debug, Clone, Deserialize)] +pub enum DBResponseValue { + DateTime(DateTime), + Primitive(DBResponsePrimitive), + // TODO: Is this variant still used? + Object { value: DBResponsePrimitive }, +} + +impl Display for DBResponseValue { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let str = match self { + DBResponseValue::DateTime(dt) => dt.to_rfc3339(), + DBResponseValue::Primitive(p) => p.to_string(), + DBResponseValue::Object { value } => value.to_string(), + }; + write!(f, "{}", str) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::transport::JsRawData; + use anyhow::Result; + use chrono::{TimeZone, Timelike, Utc}; + use serde_json::from_str; + use std::{fmt, sync::LazyLock}; + + type TestSuiteData = HashMap; + + #[derive(Clone, Deserialize)] + #[serde(rename_all = "camelCase")] + struct TestData { + request: TransformDataRequest, + query_result: JsRawData, + final_result_default: Option, + final_result_compact: Option, + } + + const TEST_SUITE_JSON: &str = r#" +{ + "regular_discount_by_city": { + "request": { + "aliasToMemberNameMap": { + "e_commerce_records_us2021__avg_discount": "ECommerceRecordsUs2021.avg_discount", + "e_commerce_records_us2021__city": "ECommerceRecordsUs2021.city" + }, + "annotation": { + "ECommerceRecordsUs2021.avg_discount": { + "title": "E Commerce Records Us2021 Avg Discount", + "shortTitle": "Avg Discount", + "type": "number", + "drillMembers": [], + "drillMembersGrouped": { + "measures": [], + "dimensions": [] + } + }, + "ECommerceRecordsUs2021.city": { + "title": "E Commerce Records Us2021 City", + "shortTitle": "City", + "type": "string" + } + }, + "query": { + "dimensions": [ + "ECommerceRecordsUs2021.city" + ], + "measures": [ + "ECommerceRecordsUs2021.avg_discount" + ], + "limit": 2, + "rowLimit": 2, + "timezone": "UTC", + "order": [], + "filters": [], + "timeDimensions": [] + }, + "queryType": "regularQuery" + }, + "queryResult": [ + { + "e_commerce_records_us2021__city": "Missouri City", + "e_commerce_records_us2021__avg_discount": "0.80000000000000000000" + }, + { + "e_commerce_records_us2021__city": "Abilene", + "e_commerce_records_us2021__avg_discount": "0.80000000000000000000" + } + ], + "finalResultDefault": [ + { + "ECommerceRecordsUs2021.city": "Missouri City", + "ECommerceRecordsUs2021.avg_discount": "0.80000000000000000000" + }, + { + "ECommerceRecordsUs2021.city": "Abilene", + "ECommerceRecordsUs2021.avg_discount": "0.80000000000000000000" + } + ], + "finalResultCompact": { + "members": [ + "ECommerceRecordsUs2021.city", + "ECommerceRecordsUs2021.avg_discount" + ], + "dataset": [ + [ + "Missouri City", + "0.80000000000000000000" + ], + [ + "Abilene", + "0.80000000000000000000" + ] + ] + } + }, + "regular_profit_by_postal_code": { + "request": { + "aliasToMemberNameMap": { + "e_commerce_records_us2021__avg_profit": "ECommerceRecordsUs2021.avg_profit", + "e_commerce_records_us2021__postal_code": "ECommerceRecordsUs2021.postalCode" + }, + "annotation": { + "ECommerceRecordsUs2021.avg_profit": { + "title": "E Commerce Records Us2021 Avg Profit", + "shortTitle": "Avg Profit", + "type": "number", + "drillMembers": [], + "drillMembersGrouped": { + "measures": [], + "dimensions": [] + } + }, + "ECommerceRecordsUs2021.postalCode": { + "title": "E Commerce Records Us2021 Postal Code", + "shortTitle": "Postal Code", + "type": "string" + } + }, + "query": { + "dimensions": [ + "ECommerceRecordsUs2021.postalCode" + ], + "measures": [ + "ECommerceRecordsUs2021.avg_profit" + ], + "limit": 2, + "rowLimit": 2, + "timezone": "UTC", + "order": [], + "filters": [], + "timeDimensions": [] + }, + "queryType": "regularQuery" + }, + "queryResult": [ + { + "e_commerce_records_us2021__postal_code": "95823", + "e_commerce_records_us2021__avg_profit": "646.1258666666666667" + }, + { + "e_commerce_records_us2021__postal_code": "64055", + "e_commerce_records_us2021__avg_profit": "487.8315000000000000" + } + ], + "finalResultDefault": [ + { + "ECommerceRecordsUs2021.postalCode": "95823", + "ECommerceRecordsUs2021.avg_profit": "646.1258666666666667" + }, + { + "ECommerceRecordsUs2021.postalCode": "64055", + "ECommerceRecordsUs2021.avg_profit": "487.8315000000000000" + } + ], + "finalResultCompact": { + "members": [ + "ECommerceRecordsUs2021.postalCode", + "ECommerceRecordsUs2021.avg_profit" + ], + "dataset": [ + [ + "95823", + "646.1258666666666667" + ], + [ + "64055", + "487.8315000000000000" + ] + ] + } + }, + "compare_date_range_count_by_order_date": { + "request": { + "aliasToMemberNameMap": { + "e_commerce_records_us2021__count": "ECommerceRecordsUs2021.count", + "e_commerce_records_us2021__order_date_day": "ECommerceRecordsUs2021.orderDate.day" + }, + "annotation": { + "ECommerceRecordsUs2021.count": { + "title": "E Commerce Records Us2021 Count", + "shortTitle": "Count", + "type": "number", + "drillMembers": [ + "ECommerceRecordsUs2021.city", + "ECommerceRecordsUs2021.country", + "ECommerceRecordsUs2021.customerId", + "ECommerceRecordsUs2021.orderId", + "ECommerceRecordsUs2021.productId", + "ECommerceRecordsUs2021.productName", + "ECommerceRecordsUs2021.orderDate" + ], + "drillMembersGrouped": { + "measures": [], + "dimensions": [ + "ECommerceRecordsUs2021.city", + "ECommerceRecordsUs2021.country", + "ECommerceRecordsUs2021.customerId", + "ECommerceRecordsUs2021.orderId", + "ECommerceRecordsUs2021.productId", + "ECommerceRecordsUs2021.productName", + "ECommerceRecordsUs2021.orderDate" + ] + } + }, + "ECommerceRecordsUs2021.orderDate.day": { + "title": "E Commerce Records Us2021 Order Date", + "shortTitle": "Order Date", + "type": "time" + }, + "ECommerceRecordsUs2021.orderDate": { + "title": "E Commerce Records Us2021 Order Date", + "shortTitle": "Order Date", + "type": "time" + } + }, + "query": { + "measures": [ + "ECommerceRecordsUs2021.count" + ], + "timeDimensions": [ + { + "dimension": "ECommerceRecordsUs2021.orderDate", + "granularity": "day", + "dateRange": [ + "2020-01-01T00:00:00.000", + "2020-01-31T23:59:59.999" + ] + } + ], + "limit": 2, + "rowLimit": 2, + "timezone": "UTC", + "order": [], + "filters": [], + "dimensions": [] + }, + "queryType": "compareDateRangeQuery" + }, + "queryResult": [ + { + "e_commerce_records_us2021__order_date_day": "2020-01-01T00:00:00.000", + "e_commerce_records_us2021__count": "10" + }, + { + "e_commerce_records_us2021__order_date_day": "2020-01-02T00:00:00.000", + "e_commerce_records_us2021__count": "8" + } + ], + "finalResultDefault": [ + { + "ECommerceRecordsUs2021.orderDate.day": "2020-01-01T00:00:00.000", + "ECommerceRecordsUs2021.orderDate": "2020-01-01T00:00:00.000", + "ECommerceRecordsUs2021.count": "10", + "compareDateRange": "2020-01-01T00:00:00.000 - 2020-01-31T23:59:59.999" + }, + { + "ECommerceRecordsUs2021.orderDate.day": "2020-01-02T00:00:00.000", + "ECommerceRecordsUs2021.orderDate": "2020-01-02T00:00:00.000", + "ECommerceRecordsUs2021.count": "8", + "compareDateRange": "2020-01-01T00:00:00.000 - 2020-01-31T23:59:59.999" + } + ], + "finalResultCompact": { + "members": [ + "ECommerceRecordsUs2021.orderDate.day", + "ECommerceRecordsUs2021.orderDate", + "ECommerceRecordsUs2021.count", + "compareDateRange" + ], + "dataset": [ + [ + "2020-01-01T00:00:00.000", + "2020-01-01T00:00:00.000", + "10", + "2020-01-01T00:00:00.000 - 2020-01-31T23:59:59.999" + ], + [ + "2020-01-02T00:00:00.000", + "2020-01-02T00:00:00.000", + "8", + "2020-01-01T00:00:00.000 - 2020-01-31T23:59:59.999" + ] + ] + } + }, + "compare_date_range_count_by_order_date2": { + "request": { + "aliasToMemberNameMap": { + "e_commerce_records_us2021__count": "ECommerceRecordsUs2021.count", + "e_commerce_records_us2021__order_date_day": "ECommerceRecordsUs2021.orderDate.day" + }, + "annotation": { + "ECommerceRecordsUs2021.count": { + "title": "E Commerce Records Us2021 Count", + "shortTitle": "Count", + "type": "number", + "drillMembers": [ + "ECommerceRecordsUs2021.city", + "ECommerceRecordsUs2021.country", + "ECommerceRecordsUs2021.customerId", + "ECommerceRecordsUs2021.orderId", + "ECommerceRecordsUs2021.productId", + "ECommerceRecordsUs2021.productName", + "ECommerceRecordsUs2021.orderDate" + ], + "drillMembersGrouped": { + "measures": [], + "dimensions": [ + "ECommerceRecordsUs2021.city", + "ECommerceRecordsUs2021.country", + "ECommerceRecordsUs2021.customerId", + "ECommerceRecordsUs2021.orderId", + "ECommerceRecordsUs2021.productId", + "ECommerceRecordsUs2021.productName", + "ECommerceRecordsUs2021.orderDate" + ] + } + }, + "ECommerceRecordsUs2021.orderDate.day": { + "title": "E Commerce Records Us2021 Order Date", + "shortTitle": "Order Date", + "type": "time" + }, + "ECommerceRecordsUs2021.orderDate": { + "title": "E Commerce Records Us2021 Order Date", + "shortTitle": "Order Date", + "type": "time" + } + }, + "query": { + "measures": [ + "ECommerceRecordsUs2021.count" + ], + "timeDimensions": [ + { + "dimension": "ECommerceRecordsUs2021.orderDate", + "granularity": "day", + "dateRange": [ + "2020-03-01T00:00:00.000", + "2020-03-31T23:59:59.999" + ] + } + ], + "limit": 2, + "rowLimit": 2, + "timezone": "UTC", + "order": [], + "filters": [], + "dimensions": [] + }, + "queryType": "compareDateRangeQuery" + }, + "queryResult": [ + { + "e_commerce_records_us2021__order_date_day": "2020-03-02T00:00:00.000", + "e_commerce_records_us2021__count": "11" + }, + { + "e_commerce_records_us2021__order_date_day": "2020-03-03T00:00:00.000", + "e_commerce_records_us2021__count": "7" + } + ], + "finalResultDefault": [ + { + "ECommerceRecordsUs2021.orderDate.day": "2020-03-02T00:00:00.000", + "ECommerceRecordsUs2021.orderDate": "2020-03-02T00:00:00.000", + "ECommerceRecordsUs2021.count": "11", + "compareDateRange": "2020-03-01T00:00:00.000 - 2020-03-31T23:59:59.999" + }, + { + "ECommerceRecordsUs2021.orderDate.day": "2020-03-03T00:00:00.000", + "ECommerceRecordsUs2021.orderDate": "2020-03-03T00:00:00.000", + "ECommerceRecordsUs2021.count": "7", + "compareDateRange": "2020-03-01T00:00:00.000 - 2020-03-31T23:59:59.999" + } + ], + "finalResultCompact": { + "members": [ + "ECommerceRecordsUs2021.orderDate.day", + "ECommerceRecordsUs2021.orderDate", + "ECommerceRecordsUs2021.count", + "compareDateRange" + ], + "dataset": [ + [ + "2020-03-02T00:00:00.000", + "2020-03-02T00:00:00.000", + "11", + "2020-03-01T00:00:00.000 - 2020-03-31T23:59:59.999" + ], + [ + "2020-03-03T00:00:00.000", + "2020-03-03T00:00:00.000", + "7", + "2020-03-01T00:00:00.000 - 2020-03-31T23:59:59.999" + ] + ] + } + }, + "blending_query_avg_discount_by_date_range_for_the_first_and_standard_ship_mode": { + "request": { + "aliasToMemberNameMap": { + "e_commerce_records_us2021__avg_discount": "ECommerceRecordsUs2021.avg_discount", + "e_commerce_records_us2021__order_date_month": "ECommerceRecordsUs2021.orderDate.month" + }, + "annotation": { + "ECommerceRecordsUs2021.avg_discount": { + "title": "E Commerce Records Us2021 Avg Discount", + "shortTitle": "Avg Discount", + "type": "number", + "drillMembers": [], + "drillMembersGrouped": { + "measures": [], + "dimensions": [] + } + }, + "ECommerceRecordsUs2021.orderDate.month": { + "title": "E Commerce Records Us2021 Order Date", + "shortTitle": "Order Date", + "type": "time" + }, + "ECommerceRecordsUs2021.orderDate": { + "title": "E Commerce Records Us2021 Order Date", + "shortTitle": "Order Date", + "type": "time" + } + }, + "query": { + "measures": [ + "ECommerceRecordsUs2021.avg_discount" + ], + "timeDimensions": [ + { + "dimension": "ECommerceRecordsUs2021.orderDate", + "granularity": "month", + "dateRange": [ + "2020-01-01T00:00:00.000", + "2020-12-30T23:59:59.999" + ] + } + ], + "filters": [ + { + "operator": "equals", + "values": [ + "Standard Class" + ], + "member": "ECommerceRecordsUs2021.shipMode" + } + ], + "limit": 2, + "rowLimit": 2, + "timezone": "UTC", + "order": [], + "dimensions": [] + }, + "queryType": "blendingQuery" + }, + "queryResult": [ + { + "e_commerce_records_us2021__order_date_month": "2020-01-01T00:00:00.000", + "e_commerce_records_us2021__avg_discount": "0.15638297872340425532" + }, + { + "e_commerce_records_us2021__order_date_month": "2020-02-01T00:00:00.000", + "e_commerce_records_us2021__avg_discount": "0.17573529411764705882" + } + ], + "finalResultDefault": [ + { + "ECommerceRecordsUs2021.orderDate.month": "2020-01-01T00:00:00.000", + "ECommerceRecordsUs2021.orderDate": "2020-01-01T00:00:00.000", + "ECommerceRecordsUs2021.avg_discount": "0.15638297872340425532", + "time.month": "2020-01-01T00:00:00.000" + }, + { + "ECommerceRecordsUs2021.orderDate.month": "2020-02-01T00:00:00.000", + "ECommerceRecordsUs2021.orderDate": "2020-02-01T00:00:00.000", + "ECommerceRecordsUs2021.avg_discount": "0.17573529411764705882", + "time.month": "2020-02-01T00:00:00.000" + } + ], + "finalResultCompact": { + "members": [ + "ECommerceRecordsUs2021.orderDate.month", + "ECommerceRecordsUs2021.orderDate", + "ECommerceRecordsUs2021.avg_discount", + "time.month" + ], + "dataset": [ + [ + "2020-01-01T00:00:00.000", + "2020-01-01T00:00:00.000", + "0.15638297872340425532", + "2020-01-01T00:00:00.000" + ], + [ + "2020-02-01T00:00:00.000", + "2020-02-01T00:00:00.000", + "0.17573529411764705882", + "2020-02-01T00:00:00.000" + ] + ] + } + }, + "blending_query_avg_discount_by_date_range_for_the_first_and_standard_ship_mode2": { + "request": { + "aliasToMemberNameMap": { + "e_commerce_records_us2021__avg_discount": "ECommerceRecordsUs2021.avg_discount", + "e_commerce_records_us2021__order_date_month": "ECommerceRecordsUs2021.orderDate.month" + }, + "annotation": { + "ECommerceRecordsUs2021.avg_discount": { + "title": "E Commerce Records Us2021 Avg Discount", + "shortTitle": "Avg Discount", + "type": "number", + "drillMembers": [], + "drillMembersGrouped": { + "measures": [], + "dimensions": [] + } + }, + "ECommerceRecordsUs2021.orderDate.month": { + "title": "E Commerce Records Us2021 Order Date", + "shortTitle": "Order Date", + "type": "time" + }, + "ECommerceRecordsUs2021.orderDate": { + "title": "E Commerce Records Us2021 Order Date", + "shortTitle": "Order Date", + "type": "time" + } + }, + "query": { + "measures": [ + "ECommerceRecordsUs2021.avg_discount" + ], + "timeDimensions": [ + { + "dimension": "ECommerceRecordsUs2021.orderDate", + "granularity": "month", + "dateRange": [ + "2020-01-01T00:00:00.000", + "2020-12-30T23:59:59.999" + ] + } + ], + "filters": [ + { + "operator": "equals", + "values": [ + "First Class" + ], + "member": "ECommerceRecordsUs2021.shipMode" + } + ], + "limit": 2, + "rowLimit": 2, + "timezone": "UTC", + "order": [], + "dimensions": [] + }, + "queryType": "blendingQuery" + }, + "queryResult": [ + { + "e_commerce_records_us2021__order_date_month": "2020-01-01T00:00:00.000", + "e_commerce_records_us2021__avg_discount": "0.28571428571428571429" + }, + { + "e_commerce_records_us2021__order_date_month": "2020-02-01T00:00:00.000", + "e_commerce_records_us2021__avg_discount": "0.21777777777777777778" + } + ], + "finalResultDefault": [ + { + "ECommerceRecordsUs2021.orderDate.month": "2020-01-01T00:00:00.000", + "ECommerceRecordsUs2021.orderDate": "2020-01-01T00:00:00.000", + "ECommerceRecordsUs2021.avg_discount": "0.28571428571428571429", + "time.month": "2020-01-01T00:00:00.000" + }, + { + "ECommerceRecordsUs2021.orderDate.month": "2020-02-01T00:00:00.000", + "ECommerceRecordsUs2021.orderDate": "2020-02-01T00:00:00.000", + "ECommerceRecordsUs2021.avg_discount": "0.21777777777777777778", + "time.month": "2020-02-01T00:00:00.000" + } + ], + "finalResultCompact": { + "members": [ + "ECommerceRecordsUs2021.orderDate.month", + "ECommerceRecordsUs2021.orderDate", + "ECommerceRecordsUs2021.avg_discount", + "time.month" + ], + "dataset": [ + [ + "2020-01-01T00:00:00.000", + "2020-01-01T00:00:00.000", + "0.28571428571428571429", + "2020-01-01T00:00:00.000" + ], + [ + "2020-02-01T00:00:00.000", + "2020-02-01T00:00:00.000", + "0.21777777777777777778", + "2020-02-01T00:00:00.000" + ] + ] + } + } +} + "#; + + static TEST_SUITE_DATA: LazyLock = + LazyLock::new(|| from_str(TEST_SUITE_JSON).unwrap()); + + #[derive(Debug)] + pub struct TestError(String); + + impl Display for TestError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Error: {}", self.0) + } + } + + impl std::error::Error for TestError {} + + /// Smart comparator of datasets. + /// Hashmaps don't guarantee the order of the elements while iterating, + /// so it's not possible to simply compare generated one and the one from the json. + fn compare_transformed_data( + left: &TransformedData, + right: &TransformedData, + ) -> Result<(), TestError> { + match (left, right) { + ( + TransformedData::Compact { + members: left_members, + dataset: left_dataset, + }, + TransformedData::Compact { + members: right_members, + dataset: right_dataset, + }, + ) => { + let mut left_sorted_members = left_members.clone(); + let mut right_sorted_members = right_members.clone(); + left_sorted_members.sort(); + right_sorted_members.sort(); + + if left_sorted_members != right_sorted_members { + return Err(TestError("Members do not match after sorting".to_string())); + } + + if left_dataset.len() != right_dataset.len() { + return Err(TestError("Datasets have different lengths".to_string())); + } + + let mut member_index_map = HashMap::new(); + for (i, member) in left_members.iter().enumerate() { + if let Some(right_index) = right_members.iter().position(|x| x == member) { + member_index_map.insert(i, right_index); + } else { + return Err(TestError("Member not found in right object".to_string())); + } + } + + for (i, left_row) in left_dataset.iter().enumerate() { + let right_row = &right_dataset[i]; + + for (j, left_value) in left_row.iter().enumerate() { + let mapped_index = *member_index_map.get(&j).unwrap(); + let right_value = &right_row[mapped_index]; + if left_value != right_value { + return Err(TestError(format!( + "Dataset values at row {} and column {} do not match: {} != {}", + i, j, left_value, right_value + ))); + } + } + } + + Ok(()) + } + (TransformedData::Vanilla(left_dataset), TransformedData::Vanilla(right_dataset)) => { + if left_dataset.len() != right_dataset.len() { + return Err(TestError( + "Vanilla datasets have different lengths".to_string(), + )); + } + + for (i, (left_record, right_record)) in + left_dataset.iter().zip(right_dataset.iter()).enumerate() + { + if left_record.len() != right_record.len() { + return Err(TestError(format!( + "Vanilla dataset records at index {} have different numbers of keys", + i + ))); + } + + for (key, left_value) in left_record { + if let Some(right_value) = right_record.get(key) { + if left_value != right_value { + return Err(TestError(format!( + "Values at index {} for key '{}' do not match: {:?} != {:?}", + i, key, left_value, right_value + ))); + } + } else { + return Err(TestError(format!( + "Key '{}' not found in right record at index {}", + key, i + ))); + } + } + } + + Ok(()) + } + _ => Err(TestError("Mismatched TransformedData types".to_string())), + } + } + + #[test] + fn test_transform_value_datetime_to_time() { + let dt = Utc + .with_ymd_and_hms(2024, 1, 1, 12, 30, 15) + .unwrap() + .with_nanosecond(123_000_000) + .unwrap(); + let value = DBResponseValue::DateTime(dt); + let result = transform_value(value, "time"); + + assert_eq!( + result, + DBResponsePrimitive::String("2024-01-01T12:30:15.123".to_string()) + ); + } + + #[test] + fn test_transform_value_datetime_empty_type() { + let dt = Utc + .with_ymd_and_hms(2024, 1, 1, 12, 30, 15) + .unwrap() + .with_nanosecond(123_000_000) + .unwrap(); + let value = DBResponseValue::DateTime(dt); + let result = transform_value(value, ""); + + assert_eq!( + result, + DBResponsePrimitive::String("2024-01-01T12:30:15.123".to_string()) + ); + } + + #[test] + fn test_transform_value_string_to_time_valid_rfc3339() { + let value = DBResponseValue::Primitive(DBResponsePrimitive::String( + "2024-01-01T12:30:15.123".to_string(), + )); + let result = transform_value(value, "time"); + + assert_eq!( + result, + DBResponsePrimitive::String("2024-01-01T12:30:15.123".to_string()) + ); + } + + #[test] + fn test_transform_value_string_wo_t_to_time_valid_rfc3339() { + let value = DBResponseValue::Primitive(DBResponsePrimitive::String( + "2024-01-01 12:30:15.123".to_string(), + )); + let result = transform_value(value, "time"); + + assert_eq!( + result, + DBResponsePrimitive::String("2024-01-01T12:30:15.123".to_string()) + ); + } + + #[test] + fn test_transform_value_string_wo_mssec_to_time_valid_rfc3339() { + let value = DBResponseValue::Primitive(DBResponsePrimitive::String( + "2024-01-01 12:30:15".to_string(), + )); + let result = transform_value(value, "time"); + + assert_eq!( + result, + DBResponsePrimitive::String("2024-01-01T12:30:15.000".to_string()) + ); + } + + #[test] + fn test_transform_value_string_wo_mssec_w_t_to_time_valid_rfc3339() { + let value = DBResponseValue::Primitive(DBResponsePrimitive::String( + "2024-01-01T12:30:15".to_string(), + )); + let result = transform_value(value, "time"); + + assert_eq!( + result, + DBResponsePrimitive::String("2024-01-01T12:30:15.000".to_string()) + ); + } + + #[test] + fn test_transform_value_string_with_tz_offset_to_time_valid_rfc3339() { + let value = DBResponseValue::Primitive(DBResponsePrimitive::String( + "2024-01-01 12:30:15.123 +00:00".to_string(), + )); + let result = transform_value(value, "time"); + + assert_eq!( + result, + DBResponsePrimitive::String("2024-01-01T12:30:15.123".to_string()) + ); + } + + #[test] + fn test_transform_value_string_with_tz_to_time_valid_rfc3339() { + let value = DBResponseValue::Primitive(DBResponsePrimitive::String( + "2024-01-01 12:30:15.123 UTC".to_string(), + )); + let result = transform_value(value, "time"); + + assert_eq!( + result, + DBResponsePrimitive::String("2024-01-01T12:30:15.123".to_string()) + ); + } + + #[test] + fn test_transform_value_string_to_time_invalid_rfc3339() { + let value = + DBResponseValue::Primitive(DBResponsePrimitive::String("invalid-date".to_string())); + let result = transform_value(value, "time"); + + assert_eq!( + result, + DBResponsePrimitive::String("invalid-date".to_string()) + ); + } + + #[test] + fn test_transform_value_primitive_string_type_not_time() { + let value = + DBResponseValue::Primitive(DBResponsePrimitive::String("some-string".to_string())); + let result = transform_value(value, "other"); + + assert_eq!( + result, + DBResponsePrimitive::String("some-string".to_string()) + ); + } + + #[test] + fn test_transform_value_object() { + let obj_value = DBResponsePrimitive::String("object-value".to_string()); + let value = DBResponseValue::Object { + value: obj_value.clone(), + }; + let result = transform_value(value, "time"); + + assert_eq!(result, obj_value); + } + + #[test] + fn test_transform_value_fallback_to_null() { + let value = DBResponseValue::DateTime(Utc::now()); + let result = transform_value(value, "unknown"); + + assert_eq!(result, DBResponsePrimitive::Null); + } + + #[test] + fn test_get_date_range_value_valid_range() -> Result<()> { + let time_dimensions = vec![QueryTimeDimension { + dimension: "some-dim".to_string(), + date_range: Some(vec![ + "2024-01-01T00:00:00Z".to_string(), + "2024-01-31T23:59:59Z".to_string(), + ]), + compare_date_range: None, + granularity: None, + }]; + + let result = get_date_range_value(Some(&time_dimensions))?; + assert_eq!( + result, + DBResponsePrimitive::String("2024-01-01T00:00:00Z - 2024-01-31T23:59:59Z".to_string()) + ); + Ok(()) + } + + #[test] + fn test_get_date_range_value_no_time_dimensions() { + let result = get_date_range_value(None); + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + "QueryTimeDimension should be specified for the compare date range query." + ); + } + + #[test] + fn test_get_date_range_value_empty_time_dimensions() { + let time_dimensions: Vec = vec![]; + + let result = get_date_range_value(Some(&time_dimensions)); + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + "No time dimension provided." + ); + } + + #[test] + fn test_get_date_range_value_missing_date_range() { + let time_dimensions = vec![QueryTimeDimension { + dimension: "dim".to_string(), + date_range: None, + compare_date_range: None, + granularity: None, + }]; + + let result = get_date_range_value(Some(&time_dimensions)); + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + "Inconsistent QueryTimeDimension configuration: dateRange required." + ); + } + + #[test] + fn test_get_date_range_value_single_date_range() { + let time_dimensions = vec![QueryTimeDimension { + dimension: "dim".to_string(), + date_range: Some(vec!["2024-01-01T00:00:00Z".to_string()]), + compare_date_range: None, + granularity: None, + }]; + + let result = get_date_range_value(Some(&time_dimensions)); + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + "Inconsistent dateRange configuration for the compare date range query: 2024-01-01T00:00:00Z" + ); + } + + #[test] + fn test_get_blending_query_key_valid_granularity() -> Result<()> { + let time_dimensions = vec![QueryTimeDimension { + dimension: "dim".to_string(), + granularity: Some("day".to_string()), + date_range: None, + compare_date_range: None, + }]; + + let result = get_blending_query_key(Some(&time_dimensions))?; + assert_eq!(result, "time.day"); + Ok(()) + } + + #[test] + fn test_get_blending_query_key_no_time_dimensions() { + let result = get_blending_query_key(None); + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + "QueryTimeDimension should be specified for the blending query." + ); + } + + #[test] + fn test_get_blending_query_key_empty_time_dimensions() { + let time_dimensions: Vec = vec![]; + + let result = get_blending_query_key(Some(&time_dimensions)); + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + "QueryTimeDimension should be specified for the blending query." + ); + } + + #[test] + fn test_get_blending_query_key_missing_granularity() { + let time_dimensions = vec![QueryTimeDimension { + dimension: "dim".to_string(), + granularity: None, + date_range: None, + compare_date_range: None, + }]; + + let result = get_blending_query_key(Some(&time_dimensions)); + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + format!( + "Inconsistent QueryTimeDimension configuration for the blending query, granularity required: {:?}", + QueryTimeDimension { + dimension: "dim".to_string(), + granularity: None, + date_range: None, + compare_date_range: None, + } + ) + ); + } + + #[test] + fn test_get_blending_response_key_valid_dimension_and_granularity() -> Result<()> { + let time_dimensions = vec![QueryTimeDimension { + dimension: "orders.created_at".to_string(), + granularity: Some("day".to_string()), + date_range: None, + compare_date_range: None, + }]; + + let result = get_blending_response_key(Some(&time_dimensions))?; + assert_eq!(result, "orders.created_at.day"); + Ok(()) + } + + #[test] + fn test_get_blending_response_key_no_time_dimensions() { + let result = get_blending_response_key(None); + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + "QueryTimeDimension should be specified for the blending query." + ); + } + + #[test] + fn test_get_blending_response_key_empty_time_dimensions() { + let time_dimensions: Vec = vec![]; + + let result = get_blending_response_key(Some(&time_dimensions)); + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + "QueryTimeDimension should be specified for the blending query." + ); + } + + #[test] + fn test_get_blending_response_key_missing_granularity() { + let time_dimensions = vec![QueryTimeDimension { + dimension: "orders.created_at".to_string(), + granularity: None, + date_range: None, + compare_date_range: None, + }]; + + let result = get_blending_response_key(Some(&time_dimensions)); + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + format!( + "Inconsistent QueryTimeDimension configuration for the blending query, granularity required: {:?}", + QueryTimeDimension { + dimension: "orders.created_at".to_string(), + granularity: None, + date_range: None, + compare_date_range: None, + } + ) + ); + } + + #[test] + fn test_regular_profit_by_postal_code_compact() -> Result<()> { + let mut test_data = TEST_SUITE_DATA + .get(&"regular_profit_by_postal_code".to_string()) + .unwrap() + .clone(); + test_data.request.res_type = Some(ResultType::Compact); + let raw_data = QueryResult::from_js_raw_data(test_data.query_result.clone())?; + let transformed = TransformedData::transform(&test_data.request, &raw_data)?; + compare_transformed_data(&transformed, &test_data.final_result_compact.unwrap())?; + Ok(()) + } + + #[test] + fn test_compare_date_range_count_by_order_date() -> Result<()> { + let mut test_data = TEST_SUITE_DATA + .get(&"compare_date_range_count_by_order_date".to_string()) + .unwrap() + .clone(); + test_data.request.res_type = Some(ResultType::Compact); + let raw_data = QueryResult::from_js_raw_data(test_data.query_result.clone())?; + let transformed = TransformedData::transform(&test_data.request, &raw_data)?; + compare_transformed_data(&transformed, &test_data.final_result_compact.unwrap())?; + Ok(()) + } + + #[test] + fn test_compare_date_range_count_by_order_date2() -> Result<()> { + let mut test_data = TEST_SUITE_DATA + .get(&"compare_date_range_count_by_order_date2".to_string()) + .unwrap() + .clone(); + test_data.request.res_type = Some(ResultType::Compact); + let raw_data = QueryResult::from_js_raw_data(test_data.query_result.clone())?; + let transformed = TransformedData::transform(&test_data.request, &raw_data)?; + compare_transformed_data(&transformed, &test_data.final_result_compact.unwrap())?; + Ok(()) + } + + #[test] + fn test_regular_discount_by_city() -> Result<()> { + let mut test_data = TEST_SUITE_DATA + .get(&"regular_discount_by_city".to_string()) + .unwrap() + .clone(); + test_data.request.res_type = Some(ResultType::Compact); + let raw_data = QueryResult::from_js_raw_data(test_data.query_result.clone())?; + let transformed = TransformedData::transform(&test_data.request, &raw_data)?; + compare_transformed_data(&transformed, &test_data.final_result_compact.unwrap())?; + Ok(()) + } + + #[test] + fn test_regular_discount_by_city_to_fail() -> Result<()> { + let mut test_data = TEST_SUITE_DATA + .get(&"regular_discount_by_city".to_string()) + .unwrap() + .clone(); + test_data + .request + .alias_to_member_name_map + .remove(&"e_commerce_records_us2021__avg_discount".to_string()); + test_data.request.res_type = Some(ResultType::Compact); + let raw_data = QueryResult::from_js_raw_data(test_data.query_result.clone())?; + match TransformedData::transform(&test_data.request, &raw_data) { + Ok(_) => Err(TestError("regular_discount_by_city should fail ".to_string()).into()), + Err(_) => Ok(()), // Should throw an error + } + } + + #[test] + fn test_blending_query_avg_discount_by_date_range_for_the_first_and_standard_ship_mode( + ) -> Result<()> { + let mut test_data = TEST_SUITE_DATA + .get( + &"blending_query_avg_discount_by_date_range_for_the_first_and_standard_ship_mode" + .to_string(), + ) + .unwrap() + .clone(); + test_data.request.res_type = Some(ResultType::Compact); + let raw_data = QueryResult::from_js_raw_data(test_data.query_result.clone())?; + let transformed = TransformedData::transform(&test_data.request, &raw_data)?; + compare_transformed_data(&transformed, &test_data.final_result_compact.unwrap())?; + Ok(()) + } + + #[test] + fn test_blending_query_avg_discount_by_date_range_for_the_first_and_standard_ship_mode2( + ) -> Result<()> { + let mut test_data = TEST_SUITE_DATA + .get( + &"blending_query_avg_discount_by_date_range_for_the_first_and_standard_ship_mode" + .to_string(), + ) + .unwrap() + .clone(); + test_data.request.res_type = Some(ResultType::Compact); + let raw_data = QueryResult::from_js_raw_data(test_data.query_result.clone())?; + let transformed = TransformedData::transform(&test_data.request, &raw_data)?; + compare_transformed_data(&transformed, &test_data.final_result_compact.unwrap())?; + Ok(()) + } + + #[test] + fn test_regular_discount_by_city_default_to_fail() -> Result<()> { + let mut test_data = TEST_SUITE_DATA + .get(&"regular_discount_by_city".to_string()) + .unwrap() + .clone(); + test_data + .request + .alias_to_member_name_map + .remove(&"e_commerce_records_us2021__avg_discount".to_string()); + test_data.request.res_type = Some(ResultType::Default); + let raw_data = QueryResult::from_js_raw_data(test_data.query_result.clone())?; + match TransformedData::transform(&test_data.request, &raw_data) { + Ok(_) => Err(TestError("regular_discount_by_city should fail ".to_string()).into()), + Err(_) => Ok(()), // Should throw an error + } + } + + #[test] + fn test_regular_discount_by_city_default() -> Result<()> { + let mut test_data = TEST_SUITE_DATA + .get(&"regular_discount_by_city".to_string()) + .unwrap() + .clone(); + test_data.request.res_type = Some(ResultType::Default); + let raw_data = QueryResult::from_js_raw_data(test_data.query_result.clone())?; + let transformed = TransformedData::transform(&test_data.request, &raw_data)?; + compare_transformed_data(&transformed, &test_data.final_result_default.unwrap())?; + Ok(()) + } + + #[test] + fn test_regular_profit_by_postal_code_default() -> Result<()> { + let mut test_data = TEST_SUITE_DATA + .get(&"regular_discount_by_city".to_string()) + .unwrap() + .clone(); + test_data.request.res_type = Some(ResultType::Default); + let raw_data = QueryResult::from_js_raw_data(test_data.query_result.clone())?; + let transformed = TransformedData::transform(&test_data.request, &raw_data)?; + compare_transformed_data(&transformed, &test_data.final_result_default.unwrap())?; + Ok(()) + } + + #[test] + fn test_blending_query_avg_discount_by_date_range_for_the_first_and_standard_ship_mode_default( + ) -> Result<()> { + let mut test_data = TEST_SUITE_DATA + .get( + &"blending_query_avg_discount_by_date_range_for_the_first_and_standard_ship_mode" + .to_string(), + ) + .unwrap() + .clone(); + test_data.request.res_type = Some(ResultType::Default); + let raw_data = QueryResult::from_js_raw_data(test_data.query_result.clone())?; + let transformed = TransformedData::transform(&test_data.request, &raw_data)?; + compare_transformed_data(&transformed, &test_data.final_result_default.unwrap())?; + Ok(()) + } + + #[test] + fn test_blending_query_avg_discount_by_date_range_for_the_first_and_standard_ship_mode2_default( + ) -> Result<()> { + let mut test_data = TEST_SUITE_DATA + .get( + &"blending_query_avg_discount_by_date_range_for_the_first_and_standard_ship_mode2" + .to_string(), + ) + .unwrap() + .clone(); + test_data.request.res_type = Some(ResultType::Default); + let raw_data = QueryResult::from_js_raw_data(test_data.query_result.clone())?; + let transformed = TransformedData::transform(&test_data.request, &raw_data)?; + compare_transformed_data(&transformed, &test_data.final_result_default.unwrap())?; + Ok(()) + } + + #[test] + fn test_get_members_no_alias_to_member_name_map() -> Result<()> { + let mut test_data = TEST_SUITE_DATA + .get(&"regular_profit_by_postal_code".to_string()) + .unwrap() + .clone(); + let raw_data = QueryResult::from_js_raw_data(test_data.query_result.clone())?; + test_data.request.alias_to_member_name_map = HashMap::new(); + let alias_to_member_name_map = &test_data.request.alias_to_member_name_map; + let annotation = &test_data.request.annotation; + let query = &test_data.request.query; + let query_type = &test_data.request.query_type.clone().unwrap_or_default(); + + match get_members( + query_type, + query, + &raw_data, + alias_to_member_name_map, + annotation, + ) { + Ok(_) => Err(TestError("get_members() should fail ".to_string()).into()), + Err(err) => { + assert!(err.to_string().contains("Member name not found for alias")); + Ok(()) + } + } + } + + #[test] + fn test_get_members_empty_dataset() -> Result<()> { + let test_data = TEST_SUITE_DATA + .get(&"regular_profit_by_postal_code".to_string()) + .unwrap() + .clone(); + let alias_to_member_name_map = &test_data.request.alias_to_member_name_map; + let annotation = &test_data.request.annotation; + let query = &test_data.request.query; + let query_type = &test_data.request.query_type.clone().unwrap_or_default(); + + let (members_to_alias_map, members) = get_members( + query_type, + query, + &QueryResult { + columns: vec![], + rows: vec![], + columns_pos: HashMap::new(), + }, + alias_to_member_name_map, + annotation, + )?; + assert_eq!(members_to_alias_map.len(), 0); + assert_eq!(members.len(), 0); + Ok(()) + } + + #[test] + fn test_get_members_filled_dataset() -> Result<()> { + let test_data = TEST_SUITE_DATA + .get(&"regular_profit_by_postal_code".to_string()) + .unwrap() + .clone(); + let raw_data = QueryResult::from_js_raw_data(test_data.query_result.clone())?; + let alias_to_member_name_map = &test_data.request.alias_to_member_name_map; + let annotation = &test_data.request.annotation; + let query = &test_data.request.query; + let query_type = &test_data.request.query_type.clone().unwrap_or_default(); + + let (members_to_alias_map, members) = get_members( + query_type, + query, + &raw_data, + alias_to_member_name_map, + annotation, + )?; + let members_map_expected: MembersMap = HashMap::from([ + ( + "ECommerceRecordsUs2021.postalCode".to_string(), + "e_commerce_records_us2021__postal_code".to_string(), + ), + ( + "ECommerceRecordsUs2021.avg_profit".to_string(), + "e_commerce_records_us2021__avg_profit".to_string(), + ), + ]); + assert_eq!(members_to_alias_map, members_map_expected); + assert_eq!(members.len(), 2); + Ok(()) + } + + #[test] + fn test_get_members_compare_date_range_empty_dataset() -> Result<()> { + let test_data = TEST_SUITE_DATA + .get(&"compare_date_range_count_by_order_date".to_string()) + .unwrap() + .clone(); + let alias_to_member_name_map = &test_data.request.alias_to_member_name_map; + let annotation = &test_data.request.annotation; + let query = &test_data.request.query; + let query_type = &test_data.request.query_type.clone().unwrap_or_default(); + + let (members_to_alias_map, members) = get_members( + query_type, + query, + &QueryResult { + columns: vec![], + rows: vec![], + columns_pos: HashMap::new(), + }, + alias_to_member_name_map, + annotation, + )?; + assert_eq!(members_to_alias_map.len(), 0); + assert_eq!(members.len(), 0); + Ok(()) + } + + #[test] + fn test_get_members_compare_date_range_filled_dataset() -> Result<()> { + let test_data = TEST_SUITE_DATA + .get(&"compare_date_range_count_by_order_date".to_string()) + .unwrap() + .clone(); + let raw_data = QueryResult::from_js_raw_data(test_data.query_result.clone())?; + let alias_to_member_name_map = &test_data.request.alias_to_member_name_map; + let annotation = &test_data.request.annotation; + let query = &test_data.request.query; + let query_type = &test_data.request.query_type.clone().unwrap_or_default(); + + let (members_to_alias_map, members) = get_members( + query_type, + query, + &raw_data, + alias_to_member_name_map, + annotation, + )?; + let members_map_expected: MembersMap = HashMap::from([ + ( + "ECommerceRecordsUs2021.orderDate.day".to_string(), + "e_commerce_records_us2021__order_date_day".to_string(), + ), + ( + "ECommerceRecordsUs2021.orderDate".to_string(), + "e_commerce_records_us2021__order_date_day".to_string(), + ), + ( + "ECommerceRecordsUs2021.count".to_string(), + "e_commerce_records_us2021__count".to_string(), + ), + ( + "compareDateRange".to_string(), + "compareDateRangeQuery".to_string(), + ), + ]); + assert_eq!(members_to_alias_map, members_map_expected); + assert_eq!(members.len(), 4); + Ok(()) + } + + #[test] + fn test_get_members_blending_query_empty_dataset() -> Result<()> { + let test_data = TEST_SUITE_DATA + .get( + &"blending_query_avg_discount_by_date_range_for_the_first_and_standard_ship_mode" + .to_string(), + ) + .unwrap() + .clone(); + let alias_to_member_name_map = &test_data.request.alias_to_member_name_map; + let annotation = &test_data.request.annotation; + let query = &test_data.request.query; + let query_type = &test_data.request.query_type.clone().unwrap_or_default(); + + let (members_to_alias_map, members) = get_members( + query_type, + query, + &QueryResult { + columns: vec![], + rows: vec![], + columns_pos: HashMap::new(), + }, + alias_to_member_name_map, + annotation, + )?; + assert_eq!(members_to_alias_map.len(), 0); + assert_eq!(members.len(), 0); + Ok(()) + } + + #[test] + fn test_get_members_blending_query_filled_dataset() -> Result<()> { + let test_data = TEST_SUITE_DATA + .get( + &"blending_query_avg_discount_by_date_range_for_the_first_and_standard_ship_mode" + .to_string(), + ) + .unwrap() + .clone(); + let raw_data = QueryResult::from_js_raw_data(test_data.query_result.clone())?; + let alias_to_member_name_map = &test_data.request.alias_to_member_name_map; + let annotation = &test_data.request.annotation; + let query = &test_data.request.query; + let query_type = &test_data.request.query_type.clone().unwrap_or_default(); + + let (members_to_alias_map, members) = get_members( + query_type, + query, + &raw_data, + alias_to_member_name_map, + annotation, + )?; + let members_map_expected: HashMap = HashMap::from([ + ( + "ECommerceRecordsUs2021.orderDate.month".to_string(), + "e_commerce_records_us2021__order_date_month".to_string(), + ), + ( + "ECommerceRecordsUs2021.orderDate".to_string(), + "e_commerce_records_us2021__order_date_month".to_string(), + ), + ( + "ECommerceRecordsUs2021.avg_discount".to_string(), + "e_commerce_records_us2021__avg_discount".to_string(), + ), + ( + "time.month".to_string(), + "e_commerce_records_us2021__order_date_month".to_string(), + ), + ]); + assert_eq!(members_to_alias_map, members_map_expected); + assert_eq!(members.len(), 4); + Ok(()) + } + + #[test] + fn test_get_compact_row_regular_profit_by_postal_code() -> Result<()> { + let test_data = TEST_SUITE_DATA + .get(&"regular_profit_by_postal_code".to_string()) + .unwrap() + .clone(); + let raw_data = QueryResult::from_js_raw_data(test_data.query_result.clone())?; + let alias_to_member_name_map = &test_data.request.alias_to_member_name_map; + let annotation = &test_data.request.annotation; + let query = test_data.request.query.clone(); + let query_type = &test_data.request.query_type.clone().unwrap_or_default(); + let time_dimensions = &test_data.request.query.time_dimensions.unwrap(); + + let (members_to_alias_map, members) = get_members( + query_type, + &query, + &raw_data, + alias_to_member_name_map, + annotation, + )?; + let res = get_compact_row( + &members_to_alias_map, + &annotation, + &query_type, + &members, + Some(time_dimensions), + &raw_data.rows[0], + &raw_data.columns_pos, + )?; + + let members_map_expected = HashMap::from([ + ( + "ECommerceRecordsUs2021.postalCode".to_string(), + DBResponsePrimitive::String("95823".to_string()), + ), + ( + "ECommerceRecordsUs2021.avg_profit".to_string(), + DBResponsePrimitive::String("646.1258666666666667".to_string()), + ), + ]); + + assert_eq!(res.len(), members_map_expected.len()); + for (i, val) in members.iter().enumerate() { + assert_eq!(res[i], members_map_expected.get(val).unwrap().clone()); + } + + Ok(()) + } + + #[test] + fn test_get_compact_row_regular_discount_by_city() -> Result<()> { + let test_data = TEST_SUITE_DATA + .get(&"regular_discount_by_city".to_string()) + .unwrap() + .clone(); + let raw_data = QueryResult::from_js_raw_data(test_data.query_result.clone())?; + let alias_to_member_name_map = &test_data.request.alias_to_member_name_map; + let annotation = &test_data.request.annotation; + let query = test_data.request.query.clone(); + let query_type = &test_data.request.query_type.clone().unwrap_or_default(); + let time_dimensions = &test_data.request.query.time_dimensions.unwrap(); + + let (members_to_alias_map, members) = get_members( + query_type, + &query, + &raw_data, + alias_to_member_name_map, + annotation, + )?; + let res = get_compact_row( + &members_to_alias_map, + &annotation, + &query_type, + &members, + Some(time_dimensions), + &raw_data.rows[0], + &raw_data.columns_pos, + )?; + + let members_map_expected = HashMap::from([ + ( + "ECommerceRecordsUs2021.city".to_string(), + DBResponsePrimitive::String("Missouri City".to_string()), + ), + ( + "ECommerceRecordsUs2021.avg_discount".to_string(), + DBResponsePrimitive::String("0.80000000000000000000".to_string()), + ), + ]); + + assert_eq!(res.len(), members_map_expected.len()); + for (i, val) in members.iter().enumerate() { + assert_eq!(res[i], members_map_expected.get(val).unwrap().clone()); + } + + Ok(()) + } + + #[test] + fn test_get_compact_row_compare_date_range_count_by_order_date() -> Result<()> { + let test_data = TEST_SUITE_DATA + .get(&"compare_date_range_count_by_order_date".to_string()) + .unwrap() + .clone(); + let raw_data = QueryResult::from_js_raw_data(test_data.query_result.clone())?; + let alias_to_member_name_map = &test_data.request.alias_to_member_name_map; + let annotation = &test_data.request.annotation; + let query = test_data.request.query.clone(); + let query_type = &test_data.request.query_type.clone().unwrap_or_default(); + let time_dimensions = &test_data.request.query.time_dimensions.unwrap(); + + let (members_to_alias_map, members) = get_members( + query_type, + &query, + &raw_data, + alias_to_member_name_map, + annotation, + )?; + let res = get_compact_row( + &members_to_alias_map, + &annotation, + &query_type, + &members, + Some(time_dimensions), + &raw_data.rows[0], + &raw_data.columns_pos, + )?; + + let members_map_expected = HashMap::from([ + ( + "ECommerceRecordsUs2021.orderDate.day".to_string(), + DBResponsePrimitive::String("2020-01-01T00:00:00.000".to_string()), + ), + ( + "ECommerceRecordsUs2021.orderDate".to_string(), + DBResponsePrimitive::String("2020-01-01T00:00:00.000".to_string()), + ), + ( + "ECommerceRecordsUs2021.count".to_string(), + DBResponsePrimitive::String("10".to_string()), + ), + ( + "compareDateRange".to_string(), + DBResponsePrimitive::String( + "2020-01-01T00:00:00.000 - 2020-01-31T23:59:59.999".to_string(), + ), + ), + ]); + + assert_eq!(res.len(), members_map_expected.len()); + for (i, val) in members.iter().enumerate() { + assert_eq!(res[i], members_map_expected.get(val).unwrap().clone()); + } + + let res = get_compact_row( + &members_to_alias_map, + &annotation, + &query_type, + &members, + Some(time_dimensions), + &raw_data.rows[1], + &raw_data.columns_pos, + )?; + + let members_map_expected = HashMap::from([ + ( + "ECommerceRecordsUs2021.orderDate.day".to_string(), + DBResponsePrimitive::String("2020-01-02T00:00:00.000".to_string()), + ), + ( + "ECommerceRecordsUs2021.orderDate".to_string(), + DBResponsePrimitive::String("2020-01-02T00:00:00.000".to_string()), + ), + ( + "ECommerceRecordsUs2021.count".to_string(), + DBResponsePrimitive::String("8".to_string()), + ), + ( + "compareDateRange".to_string(), + DBResponsePrimitive::String( + "2020-01-01T00:00:00.000 - 2020-01-31T23:59:59.999".to_string(), + ), + ), + ]); + + assert_eq!(res.len(), members_map_expected.len()); + for (i, val) in members.iter().enumerate() { + assert_eq!(res[i], members_map_expected.get(val).unwrap().clone()); + } + + Ok(()) + } + + #[test] + fn test_get_compact_row_blending_query_avg_discount() -> Result<()> { + let test_data = TEST_SUITE_DATA + .get( + &"blending_query_avg_discount_by_date_range_for_the_first_and_standard_ship_mode" + .to_string(), + ) + .unwrap() + .clone(); + let raw_data = QueryResult::from_js_raw_data(test_data.query_result.clone())?; + let alias_to_member_name_map = &test_data.request.alias_to_member_name_map; + let annotation = &test_data.request.annotation; + let query = test_data.request.query.clone(); + let query_type = &test_data.request.query_type.clone().unwrap_or_default(); + let time_dimensions = &test_data.request.query.time_dimensions.unwrap(); + + let (members_to_alias_map, members) = get_members( + query_type, + &query, + &raw_data, + alias_to_member_name_map, + annotation, + )?; + let res = get_compact_row( + &members_to_alias_map, + &annotation, + &query_type, + &members, + Some(time_dimensions), + &raw_data.rows[0], + &raw_data.columns_pos, + )?; + + let members_map_expected = HashMap::from([ + ( + "ECommerceRecordsUs2021.orderDate.month".to_string(), + DBResponsePrimitive::String("2020-01-01T00:00:00.000".to_string()), + ), + ( + "ECommerceRecordsUs2021.orderDate".to_string(), + DBResponsePrimitive::String("2020-01-01T00:00:00.000".to_string()), + ), + ( + "ECommerceRecordsUs2021.avg_discount".to_string(), + DBResponsePrimitive::String("0.15638297872340425532".to_string()), + ), + ( + "time.month".to_string(), + DBResponsePrimitive::String("2020-01-01T00:00:00.000".to_string()), + ), + ]); + + assert_eq!(res.len(), members_map_expected.len()); + for (i, val) in members.iter().enumerate() { + assert_eq!(res[i], members_map_expected.get(val).unwrap().clone()); + } + Ok(()) + } + + #[test] + fn test_get_vanilla_row_regular_discount_by_city() -> Result<()> { + let test_data = TEST_SUITE_DATA + .get(&"regular_discount_by_city".to_string()) + .unwrap() + .clone(); + let raw_data = QueryResult::from_js_raw_data(test_data.query_result.clone())?; + let alias_to_member_name_map = &test_data.request.alias_to_member_name_map; + let annotation = &test_data.request.annotation; + let query = test_data.request.query.clone(); + let query_type = &test_data.request.query_type.clone().unwrap_or_default(); + + let res = get_vanilla_row( + &alias_to_member_name_map, + &annotation, + &query_type, + &query, + &raw_data.rows[0], + &raw_data.columns_pos, + )?; + let expected = HashMap::from([ + ( + "ECommerceRecordsUs2021.city".to_string(), + DBResponsePrimitive::String("Missouri City".to_string()), + ), + ( + "ECommerceRecordsUs2021.avg_discount".to_string(), + DBResponsePrimitive::String("0.80000000000000000000".to_string()), + ), + ]); + assert_eq!(res, expected); + Ok(()) + } + + #[test] + fn test_get_vanilla_row_regular_discount_by_city_to_fail_member() -> Result<()> { + let mut test_data = TEST_SUITE_DATA + .get(&"regular_discount_by_city".to_string()) + .unwrap() + .clone(); + test_data + .request + .alias_to_member_name_map + .remove(&"e_commerce_records_us2021__avg_discount".to_string()); + let raw_data = QueryResult::from_js_raw_data(test_data.query_result.clone())?; + let alias_to_member_name_map = &test_data.request.alias_to_member_name_map; + let annotation = &test_data.request.annotation; + let query = test_data.request.query.clone(); + let query_type = &test_data.request.query_type.clone().unwrap_or_default(); + + match get_vanilla_row( + &alias_to_member_name_map, + &annotation, + &query_type, + &query, + &raw_data.rows[0], + &raw_data.columns_pos, + ) { + Ok(_) => Err(TestError("get_vanilla_row() should fail ".to_string()).into()), + Err(err) => { + assert!(err.to_string().contains("Missing member name for alias")); + Ok(()) + } + } + } + + #[test] + fn test_get_vanilla_row_regular_discount_by_city_to_fail_annotation() -> Result<()> { + let mut test_data = TEST_SUITE_DATA + .get(&"regular_discount_by_city".to_string()) + .unwrap() + .clone(); + test_data + .request + .annotation + .remove(&"ECommerceRecordsUs2021.avg_discount".to_string()); + let raw_data = QueryResult::from_js_raw_data(test_data.query_result.clone())?; + let alias_to_member_name_map = &test_data.request.alias_to_member_name_map; + let annotation = &test_data.request.annotation; + let query = test_data.request.query.clone(); + let query_type = &test_data.request.query_type.clone().unwrap_or_default(); + + match get_vanilla_row( + &alias_to_member_name_map, + &annotation, + &query_type, + &query, + &raw_data.rows[0], + &raw_data.columns_pos, + ) { + Ok(_) => Err(TestError("get_vanilla_row() should fail ".to_string()).into()), + Err(err) => { + assert!(err.to_string().contains("You requested hidden member")); + Ok(()) + } + } + } +} diff --git a/rust/cubeorchestrator/src/transport.rs b/rust/cubeorchestrator/src/transport.rs new file mode 100644 index 0000000000000..5bcd358a28a40 --- /dev/null +++ b/rust/cubeorchestrator/src/transport.rs @@ -0,0 +1,290 @@ +use crate::query_result_transform::DBResponsePrimitive; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::{collections::HashMap, fmt::Display}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum ResultType { + Default, + Compact, +} + +#[derive(Default, Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum QueryType { + #[default] + RegularQuery, + CompareDateRangeQuery, + BlendingQuery, +} + +impl Display for QueryType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let str = serde_json::to_value(self) + .unwrap() + .as_str() + .unwrap() + .to_string(); + write!(f, "{}", str) + } +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum MemberType { + Measures, + Dimensions, + Segments, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum FilterOperator { + Equals, + NotEquals, + Contains, + NotContains, + In, + NotIn, + Gt, + Gte, + Lt, + Lte, + Set, + NotSet, + InDateRange, + NotInDateRange, + OnTheDate, + BeforeDate, + BeforeOrOnDate, + AfterDate, + AfterOrOnDate, + MeasureFilter, + EndsWith, + NotEndsWith, + StartsWith, + NotStartsWith, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct QueryFilter { + pub member: String, + pub operator: FilterOperator, + pub values: Option>, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Hash, Eq, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct GroupingSet { + pub group_type: String, + pub id: u32, + pub sub_id: Option, +} + +// We can do nothing with JS functions here, +// but to keep DTOs in sync with reality, let's keep it. +pub type JsFunction = String; + +#[derive(Debug, Clone, Serialize, Deserialize, Hash, Eq, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct MemberExpression { + // Made as Option and JsValueDeserializer set's it to None. + pub expression: Option, + pub cube_name: String, + pub name: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub expression_name: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub definition: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub grouping_set: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Hash, Eq, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct ParsedMemberExpression { + pub expression: Vec, + pub cube_name: String, + pub name: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub expression_name: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub definition: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub grouping_set: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct QueryTimeDimension { + pub dimension: String, + pub date_range: Option>, + pub compare_date_range: Option>, + pub granularity: Option, +} + +pub type AliasToMemberMap = HashMap; + +pub type MembersMap = HashMap; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GranularityMeta { + pub name: String, + pub title: String, + pub interval: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub offset: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub origin: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ConfigItem { + #[serde(skip_serializing_if = "Option::is_none")] + pub title: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub short_title: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub description: Option, + #[serde(rename = "type")] + #[serde(skip_serializing_if = "Option::is_none")] + pub member_type: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub format: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub meta: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub drill_members: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub drill_members_grouped: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub granularities: Option>, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DrillMembersGrouped { + #[serde(skip_serializing_if = "Option::is_none")] + pub measures: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub dimensions: Option>, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct AnnotatedConfigItem { + #[serde(skip_serializing_if = "Option::is_none")] + pub title: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub short_title: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub description: Option, + #[serde(rename = "type")] + #[serde(skip_serializing_if = "Option::is_none")] + pub member_type: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub format: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub meta: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub drill_members: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub drill_members_grouped: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub granularity: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Order { + pub id: String, + pub desc: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct NormalizedQueryFilter { + pub member: String, + pub operator: FilterOperator, + #[serde(skip_serializing_if = "Option::is_none")] + pub values: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub dimension: Option, +} + +// TODO: Not used, as all members are made as Strings for now +// XXX: Omitted function variant +#[derive(Debug, Clone, Serialize, Deserialize, Hash, Eq, PartialEq)] +#[serde(untagged)] +pub enum MemberOrMemberExpression { + Member(String), + ParsedMemberExpression(ParsedMemberExpression), + MemberExpression(MemberExpression), +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LogicalAndFilter { + pub and: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LogicalOrFilter { + pub or: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(untagged)] +pub enum QueryFilterOrLogicalFilter { + QueryFilter(QueryFilter), + LogicalAndFilter(LogicalAndFilter), + LogicalOrFilter(LogicalOrFilter), + NormalizedQueryFilter(NormalizedQueryFilter), +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct NormalizedQuery { + #[serde(skip_serializing_if = "Option::is_none")] + pub measures: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub dimensions: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub time_dimensions: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub segments: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub limit: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub offset: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub total: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub total_query: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub timezone: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub renew_query: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub ungrouped: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub response_format: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub filters: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub row_limit: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub order: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub query_type: Option, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TransformDataRequest { + pub alias_to_member_name_map: HashMap, + pub annotation: HashMap, + pub query: NormalizedQuery, + pub query_type: Option, + pub res_type: Option, +} + +pub type JsRawData = Vec>; diff --git a/rust/cubeshared/.gitignore b/rust/cubeshared/.gitignore new file mode 100644 index 0000000000000..2a0a960cb1ec5 --- /dev/null +++ b/rust/cubeshared/.gitignore @@ -0,0 +1,3 @@ +/target +/.idea +.vscode diff --git a/rust/cubeshared/CHANGELOG.md b/rust/cubeshared/CHANGELOG.md new file mode 100644 index 0000000000000..f05ecfa971fe9 --- /dev/null +++ b/rust/cubeshared/CHANGELOG.md @@ -0,0 +1,2 @@ +# ChangeLog + diff --git a/rust/cubeshared/Cargo.lock b/rust/cubeshared/Cargo.lock new file mode 100644 index 0000000000000..e7d949fe75ccd --- /dev/null +++ b/rust/cubeshared/Cargo.lock @@ -0,0 +1,41 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "cubeshared" +version = "0.1.0" +dependencies = [ + "flatbuffers", +] + +[[package]] +name = "flatbuffers" +version = "23.5.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dac53e22462d78c16d64a1cd22371b54cc3fe94aa15e7886a2fa6e5d1ab8640" +dependencies = [ + "bitflags", + "rustc_version", +] + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "semver" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" diff --git a/rust/cubeshared/Cargo.toml b/rust/cubeshared/Cargo.toml new file mode 100644 index 0000000000000..b40a0b5739708 --- /dev/null +++ b/rust/cubeshared/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "cubeshared" +version = "0.1.0" +edition = "2021" + +[dependencies] +flatbuffers = "23.1.21" diff --git a/rust/cubeshared/flatbuffers-codegen.sh b/rust/cubeshared/flatbuffers-codegen.sh new file mode 100755 index 0000000000000..e0a62d0d507a5 --- /dev/null +++ b/rust/cubeshared/flatbuffers-codegen.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +cd ./src/codegen || exit 1 +flatc --rust http_message.fbs diff --git a/rust/cubeshared/rust-toolchain.toml b/rust/cubeshared/rust-toolchain.toml new file mode 100644 index 0000000000000..040357e9b1d43 --- /dev/null +++ b/rust/cubeshared/rust-toolchain.toml @@ -0,0 +1,5 @@ +[toolchain] +#channel = "stable" +channel = "nightly-2024-07-15" +components = ["rustfmt", "clippy"] +profile = "minimal" diff --git a/rust/cubeshared/rustfmt.toml b/rust/cubeshared/rustfmt.toml new file mode 100644 index 0000000000000..d9ba5fdb90ba3 --- /dev/null +++ b/rust/cubeshared/rustfmt.toml @@ -0,0 +1 @@ +imports_granularity = "Crate" \ No newline at end of file diff --git a/rust/cubestore/cubestore/src/codegen/http_message.fbs b/rust/cubeshared/src/codegen/http_message.fbs similarity index 100% rename from rust/cubestore/cubestore/src/codegen/http_message.fbs rename to rust/cubeshared/src/codegen/http_message.fbs diff --git a/rust/cubestore/cubestore/src/codegen/http_message_generated.rs b/rust/cubeshared/src/codegen/http_message_generated.rs similarity index 100% rename from rust/cubestore/cubestore/src/codegen/http_message_generated.rs rename to rust/cubeshared/src/codegen/http_message_generated.rs diff --git a/rust/cubestore/cubestore/src/codegen/mod.rs b/rust/cubeshared/src/codegen/mod.rs similarity index 71% rename from rust/cubestore/cubestore/src/codegen/mod.rs rename to rust/cubeshared/src/codegen/mod.rs index 1b691689fb9aa..fa722ae71b1e7 100644 --- a/rust/cubestore/cubestore/src/codegen/mod.rs +++ b/rust/cubeshared/src/codegen/mod.rs @@ -1,3 +1,4 @@ +#[allow(unused_imports)] mod http_message_generated; pub use http_message_generated::*; diff --git a/rust/cubeshared/src/lib.rs b/rust/cubeshared/src/lib.rs new file mode 100644 index 0000000000000..24ccbddd82ed0 --- /dev/null +++ b/rust/cubeshared/src/lib.rs @@ -0,0 +1 @@ +pub mod codegen; diff --git a/rust/cubesql/Cargo.lock b/rust/cubesql/Cargo.lock index 1cb06b84594cd..51c26fde43a59 100644 --- a/rust/cubesql/Cargo.lock +++ b/rust/cubesql/Cargo.lock @@ -188,7 +188,7 @@ checksum = "531b97fb4cd3dfdce92c35dedbfdc1f0b9d8091c8ca943d6dae340ef5012d514" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.87", ] [[package]] @@ -829,7 +829,7 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn 2.0.76", + "syn 2.0.87", ] [[package]] @@ -846,7 +846,7 @@ checksum = "a26acccf6f445af85ea056362561a24ef56cdc15fcc685f03aec50b9c702cb6d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.87", ] [[package]] @@ -1161,7 +1161,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.87", ] [[package]] @@ -2221,7 +2221,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.87", ] [[package]] @@ -2712,29 +2712,29 @@ checksum = "58bf37232d3bb9a2c4e641ca2a11d83b5062066f88df7fed36c28772046d65ba" [[package]] name = "serde" -version = "1.0.209" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99fce0ffe7310761ca6bf9faf5115afbc19688edd00171d81b1bb1b116c63e09" +checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.209" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5831b979fd7b5439637af1752d535ff49f4860c0f341d1baeb6faf0f4242170" +checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.87", ] [[package]] name = "serde_json" -version = "1.0.127" +version = "1.0.135" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8043c06d9f82bd7271361ed64f415fe5e12a77fdb52e573e7f06a516dea329ad" +checksum = "2b0d7ba2887406110130a978386c4e1befb98c674b4fba677954e4db976630d9" dependencies = [ "indexmap 2.4.0", "itoa 1.0.10", @@ -2932,7 +2932,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.76", + "syn 2.0.87", ] [[package]] @@ -2971,9 +2971,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.76" +version = "2.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578e081a14e0cefc3279b0472138c513f37b41a08d5a3cca9b6e4e8ceb6cd525" +checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" dependencies = [ "proc-macro2", "quote", @@ -3064,7 +3064,7 @@ checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.87", ] [[package]] @@ -3151,7 +3151,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.87", ] [[package]] @@ -3260,7 +3260,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.87", ] [[package]] @@ -3842,7 +3842,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.87", ] [[package]] diff --git a/rust/cubesql/cubeclient/src/models/v1_load_result.rs b/rust/cubesql/cubeclient/src/models/v1_load_result.rs index be2a78fbee94b..7b50ab1633e59 100644 --- a/rust/cubesql/cubeclient/src/models/v1_load_result.rs +++ b/rust/cubesql/cubeclient/src/models/v1_load_result.rs @@ -7,7 +7,6 @@ * * Generated by: https://openapi-generator.tech */ - #[derive(Clone, Debug, PartialEq, Default, Serialize, Deserialize)] pub struct V1LoadResult { #[serde(rename = "dataSource", skip_serializing_if = "Option::is_none")] diff --git a/rust/cubesql/cubesql/src/compile/engine/df/scan.rs b/rust/cubesql/cubesql/src/compile/engine/df/scan.rs index 33577e2e6857c..afd410898b25c 100644 --- a/rust/cubesql/cubesql/src/compile/engine/df/scan.rs +++ b/rust/cubesql/cubesql/src/compile/engine/df/scan.rs @@ -1,5 +1,5 @@ use async_trait::async_trait; -use cubeclient::models::{V1LoadRequestQuery, V1LoadResult, V1LoadResultAnnotation}; +use cubeclient::models::{V1LoadRequestQuery, V1LoadResponse}; pub use datafusion::{ arrow::{ array::{ @@ -52,7 +52,7 @@ use datafusion::{ logical_plan::JoinType, scalar::ScalarValue, }; -use serde_json::{json, Value}; +use serde_json::Value; #[derive(Debug, Clone, Eq, PartialEq)] pub enum MemberField { @@ -655,27 +655,22 @@ impl ExecutionPlan for CubeScanExecutionPlan { ))); } - let mut response = JsonValueObject::new( - load_data( - self.span_id.clone(), - request, - self.auth_context.clone(), - self.transport.clone(), - meta.clone(), - self.options.clone(), - self.wrapped_sql.clone(), - ) - .await? - .data, - ); - one_shot_stream.data = Some( - transform_response( - &mut response, - one_shot_stream.schema.clone(), - &one_shot_stream.member_fields, - ) - .map_err(|e| DataFusionError::Execution(e.message.to_string()))?, - ); + let response = load_data( + self.span_id.clone(), + request, + self.auth_context.clone(), + self.transport.clone(), + meta.clone(), + self.schema.clone(), + self.member_fields.clone(), + self.options.clone(), + self.wrapped_sql.clone(), + ) + .await?; + + // For now execute method executes only one query at a time, so we + // take the first result + one_shot_stream.data = Some(response.first().unwrap().clone()); Ok(Box::pin(CubeScanStreamRouter::new( None, @@ -840,9 +835,11 @@ async fn load_data( auth_context: AuthContextRef, transport: Arc, meta: LoadRequestMeta, + schema: SchemaRef, + member_fields: Vec, options: CubeScanOptions, sql_query: Option, -) -> ArrowResult { +) -> ArrowResult> { let no_members_query = request.measures.as_ref().map(|v| v.len()).unwrap_or(0) == 0 && request.dimensions.as_ref().map(|v| v.len()).unwrap_or(0) == 0 && request @@ -860,22 +857,27 @@ async fn load_data( data.push(serde_json::Value::Null) } - V1LoadResult::new( - V1LoadResultAnnotation { - measures: json!(Vec::::new()), - dimensions: json!(Vec::::new()), - segments: json!(Vec::::new()), - time_dimensions: json!(Vec::::new()), - }, - data, - ) + let mut response = JsonValueObject::new(data); + let rec = transform_response(&mut response, schema.clone(), &member_fields) + .map_err(|e| DataFusionError::Execution(e.message.to_string()))?; + + rec } else { let result = transport - .load(span_id, request, sql_query, auth_context, meta) - .await; - let mut response = result.map_err(|err| ArrowError::ComputeError(err.to_string()))?; - if let Some(data) = response.results.pop() { - match (options.max_records, data.data.len()) { + .load( + span_id, + request, + sql_query, + auth_context, + meta, + schema, + member_fields, + ) + .await + .map_err(|err| ArrowError::ComputeError(err.to_string()))?; + let response = result.first(); + if let Some(data) = response.cloned() { + match (options.max_records, data.num_rows()) { (Some(max_records), len) if len >= max_records => { return Err(ArrowError::ComputeError(format!("One of the Cube queries exceeded the maximum row limit ({}). JOIN/UNION is not possible as it will produce incorrect results. Try filtering the results more precisely or moving post-processing functions to an outer query.", max_records))); } @@ -890,7 +892,7 @@ async fn load_data( } }; - Ok(result) + Ok(vec![result]) } fn load_to_stream_sync(one_shot_stream: &mut CubeScanOneShotStream) -> Result<()> { @@ -899,6 +901,8 @@ fn load_to_stream_sync(one_shot_stream: &mut CubeScanOneShotStream) -> Result<() let auth = one_shot_stream.auth_context.clone(); let transport = one_shot_stream.transport.clone(); let meta = one_shot_stream.meta.clone(); + let schema = one_shot_stream.schema.clone(); + let member_fields = one_shot_stream.member_fields.clone(); let options = one_shot_stream.options.clone(); let wrapped_sql = one_shot_stream.wrapped_sql.clone(); @@ -910,22 +914,17 @@ fn load_to_stream_sync(one_shot_stream: &mut CubeScanOneShotStream) -> Result<() auth, transport, meta, + schema, + member_fields, options, wrapped_sql, )) }) .join() - .map_err(|_| DataFusionError::Execution(format!("Can't load to stream")))?; - - let mut response = JsonValueObject::new(res.unwrap().data); - one_shot_stream.data = Some( - transform_response( - &mut response, - one_shot_stream.schema.clone(), - &one_shot_stream.member_fields, - ) - .map_err(|e| DataFusionError::Execution(e.message.to_string()))?, - ); + .map_err(|_| DataFusionError::Execution(format!("Can't load to stream")))??; + + let response = res.first(); + one_shot_stream.data = Some(response.cloned().unwrap()); Ok(()) } @@ -1128,7 +1127,7 @@ pub fn transform_response( )) })?; // TODO switch parsing to microseconds - if timestamp.timestamp_millis() > (((1i64) << 62) / 1_000_000) { + if timestamp.and_utc().timestamp_millis() > (((1i64) << 62) / 1_000_000) { builder.append_null()?; } else if let Some(nanos) = timestamp.timestamp_nanos_opt() { builder.append_value(nanos)?; @@ -1170,10 +1169,10 @@ pub fn transform_response( )) })?; // TODO switch parsing to microseconds - if timestamp.timestamp_millis() > (((1 as i64) << 62) / 1_000_000) { + if timestamp.and_utc().timestamp_millis() > (((1 as i64) << 62) / 1_000_000) { builder.append_null()?; } else { - builder.append_value(timestamp.timestamp_millis())?; + builder.append_value(timestamp.and_utc().timestamp_millis())?; } }, }, @@ -1331,6 +1330,21 @@ pub fn transform_response( Ok(RecordBatch::try_new(schema.clone(), columns)?) } +pub fn convert_transport_response( + response: V1LoadResponse, + schema: SchemaRef, + member_fields: Vec, +) -> std::result::Result, CubeError> { + response + .results + .into_iter() + .map(|r| { + let mut response = JsonValueObject::new(r.data.clone()); + transform_response(&mut response, schema.clone(), &member_fields) + }) + .collect::, CubeError>>() +} + #[cfg(test)] mod tests { use super::*; @@ -1394,9 +1408,12 @@ mod tests { _sql_query: Option, _ctx: AuthContextRef, _meta_fields: LoadRequestMeta, - ) -> Result { + schema: SchemaRef, + member_fields: Vec, + ) -> Result, CubeError> { let response = r#" - { + { + "results": [{ "annotation": { "measures": [], "dimensions": [], @@ -1410,17 +1427,13 @@ mod tests { {"KibanaSampleDataEcommerce.count": null, "KibanaSampleDataEcommerce.maxPrice": null, "KibanaSampleDataEcommerce.isBool": "true", "KibanaSampleDataEcommerce.orderDate": "9999-12-31 00:00:00.000", "KibanaSampleDataEcommerce.city": "City 4"}, {"KibanaSampleDataEcommerce.count": null, "KibanaSampleDataEcommerce.maxPrice": null, "KibanaSampleDataEcommerce.isBool": "false", "KibanaSampleDataEcommerce.orderDate": null, "KibanaSampleDataEcommerce.city": null} ] - } + }] + } "#; - let result: V1LoadResult = serde_json::from_str(response).unwrap(); - - Ok(V1LoadResponse { - pivot_query: None, - slow_query: None, - query_type: None, - results: vec![result], - }) + let result: V1LoadResponse = serde_json::from_str(response).unwrap(); + convert_transport_response(result, schema.clone(), member_fields) + .map_err(|err| CubeError::user(err.to_string())) } async fn load_stream( diff --git a/rust/cubesql/cubesql/src/compile/engine/udf/common.rs b/rust/cubesql/cubesql/src/compile/engine/udf/common.rs index 127fed402cc73..20f57f81379bb 100644 --- a/rust/cubesql/cubesql/src/compile/engine/udf/common.rs +++ b/rust/cubesql/cubesql/src/compile/engine/udf/common.rs @@ -1029,6 +1029,7 @@ pub fn create_date_udf() -> ScalarUDF { builder.append_value( NaiveDateTime::parse_from_str(strings.value(i), "%Y-%m-%d %H:%M:%S%.f") .map_err(|e| DataFusionError::Execution(e.to_string()))? + .and_utc() .timestamp_nanos_opt() .unwrap(), )?; @@ -1233,6 +1234,7 @@ macro_rules! date_math_udf { let interval = intervals.value(i).into(); builder.append_value( $FUN(timestamp, interval, $IS_ADD)? + .and_utc() .timestamp_nanos_opt() .unwrap(), )?; @@ -1569,7 +1571,7 @@ pub fn create_str_to_date_udf() -> ScalarUDF { })?; Ok(ColumnarValue::Scalar(ScalarValue::TimestampNanosecond( - Some(res.timestamp_nanos_opt().unwrap()), + Some(res.and_utc().timestamp_nanos_opt().unwrap()), None, ))) }); @@ -2339,7 +2341,7 @@ macro_rules! generate_series_helper_timestamp { )) })?; let res = date_addsub_month_day_nano(current_dt, $STEP, true)?; - $CURRENT = res.timestamp_nanos_opt().unwrap() as $PRIMITIVE_TYPE; + $CURRENT = res.and_utc().timestamp_nanos_opt().unwrap() as $PRIMITIVE_TYPE; }; } @@ -3433,6 +3435,7 @@ pub fn create_date_to_timestamp_udf() -> ScalarUDF { )?; Ok(Some( NaiveDateTime::new(date, time) + .and_utc() .timestamp_nanos_opt() .unwrap(), )) diff --git a/rust/cubesql/cubesql/src/compile/rewrite/rules/filters.rs b/rust/cubesql/cubesql/src/compile/rewrite/rules/filters.rs index f859ef2e31c9f..bac53c2026a93 100644 --- a/rust/cubesql/cubesql/src/compile/rewrite/rules/filters.rs +++ b/rust/cubesql/cubesql/src/compile/rewrite/rules/filters.rs @@ -4118,8 +4118,8 @@ impl FilterRules { }; let (Some(start_date), Some(end_date)) = ( - start_date.timestamp_nanos_opt(), - end_date.timestamp_nanos_opt(), + start_date.and_utc().timestamp_nanos_opt(), + end_date.and_utc().timestamp_nanos_opt(), ) else { return false; }; diff --git a/rust/cubesql/cubesql/src/compile/test/mod.rs b/rust/cubesql/cubesql/src/compile/test/mod.rs index cc62e88d7b897..8c07ac6b7041a 100644 --- a/rust/cubesql/cubesql/src/compile/test/mod.rs +++ b/rust/cubesql/cubesql/src/compile/test/mod.rs @@ -44,6 +44,9 @@ pub mod test_user_change; #[cfg(test)] pub mod test_wrapper; pub mod utils; +use crate::compile::{ + arrow::record_batch::RecordBatch, engine::df::scan::convert_transport_response, +}; pub use utils::*; pub fn get_test_meta() -> Vec { @@ -804,7 +807,9 @@ impl TransportService for TestConnectionTransport { sql_query: Option, ctx: AuthContextRef, meta: LoadRequestMeta, - ) -> Result { + schema: SchemaRef, + member_fields: Vec, + ) -> Result, CubeError> { { let mut calls = self.load_calls.lock().await; calls.push(TestTransportLoadCall { @@ -822,12 +827,19 @@ impl TransportService for TestConnectionTransport { } let mocks = self.load_mocks.lock().await; - let Some((_req, res)) = mocks.iter().find(|(req, _res)| req == &query) else { + let Some(res) = mocks + .iter() + .find(|(req, _res)| req == &query) + .map(|(_req, res)| { + convert_transport_response(res.clone(), schema.clone(), member_fields) + }) + else { return Err(CubeError::internal(format!( "Unexpected query in test transport: {query:?}" ))); }; - Ok(res.clone()) + + res } async fn load_stream( diff --git a/rust/cubesql/cubesql/src/transport/service.rs b/rust/cubesql/cubesql/src/transport/service.rs index 7812eefebe839..85d9d270910f9 100644 --- a/rust/cubesql/cubesql/src/transport/service.rs +++ b/rust/cubesql/cubesql/src/transport/service.rs @@ -28,15 +28,13 @@ use uuid::Uuid; use crate::{ compile::{ engine::df::{ - scan::MemberField, + scan::{convert_transport_response, MemberField}, wrapper::{GroupingSetDesc, GroupingSetType, SqlQuery}, }, rewrite::LikeType, }, sql::{AuthContextRef, HttpAuthContext}, - transport::{ - MetaContext, TransportLoadRequest, TransportLoadRequestQuery, TransportLoadResponse, - }, + transport::{MetaContext, TransportLoadRequest, TransportLoadRequestQuery}, CubeError, RWLockAsync, }; @@ -142,7 +140,9 @@ pub trait TransportService: Send + Sync + Debug { sql_query: Option, ctx: AuthContextRef, meta_fields: LoadRequestMeta, - ) -> Result; + schema: SchemaRef, + member_fields: Vec, + ) -> Result, CubeError>; async fn load_stream( &self, @@ -280,7 +280,9 @@ impl TransportService for HttpTransport { _sql_query: Option, ctx: AuthContextRef, meta: LoadRequestMeta, - ) -> Result { + schema: SchemaRef, + member_fields: Vec, + ) -> Result, CubeError> { if meta.change_user().is_some() { return Err(CubeError::internal( "Changing security context (__user) is not supported in the standalone mode" @@ -296,7 +298,7 @@ impl TransportService for HttpTransport { let response = cube_api::load_v1(&self.get_client_config_for_ctx(ctx), Some(request)).await?; - Ok(response) + convert_transport_response(response, schema, member_fields) } async fn load_stream( diff --git a/rust/cubesqlplanner/Cargo.lock b/rust/cubesqlplanner/Cargo.lock index 55b1401878539..c2732a97cef95 100644 --- a/rust/cubesqlplanner/Cargo.lock +++ b/rust/cubesqlplanner/Cargo.lock @@ -175,7 +175,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.95", ] [[package]] @@ -186,7 +186,7 @@ checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.95", ] [[package]] @@ -336,7 +336,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.95", "syn_derive", ] @@ -1102,7 +1102,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.95", ] [[package]] @@ -1793,7 +1793,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6813fde79b646e47e7ad75f480aa80ef76a5d9599e2717407961531169ee38b" dependencies = [ "quote", - "syn 2.0.69", + "syn 2.0.95", "syn-mid", ] @@ -2030,7 +2030,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.95", ] [[package]] @@ -2113,7 +2113,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.95", ] [[package]] @@ -2232,9 +2232,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.86" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" dependencies = [ "unicode-ident", ] @@ -2647,32 +2647,33 @@ checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" [[package]] name = "serde" -version = "1.0.209" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99fce0ffe7310761ca6bf9faf5115afbc19688edd00171d81b1bb1b116c63e09" +checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.209" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5831b979fd7b5439637af1752d535ff49f4860c0f341d1baeb6faf0f4242170" +checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.95", ] [[package]] name = "serde_json" -version = "1.0.120" +version = "1.0.135" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5" +checksum = "2b0d7ba2887406110130a978386c4e1befb98c674b4fba677954e4db976630d9" dependencies = [ "indexmap 2.2.6", "itoa", + "memchr", "ryu", "serde", ] @@ -2837,7 +2838,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.69", + "syn 2.0.95", ] [[package]] @@ -2876,9 +2877,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.69" +version = "2.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "201fcda3845c23e8212cd466bfebf0bd20694490fc0356ae8e428e0824a915a6" +checksum = "46f71c0377baf4ef1cc3e3402ded576dccc315800fbc62dfc7fe04b009773b4a" dependencies = [ "proc-macro2", "quote", @@ -2893,7 +2894,7 @@ checksum = "b5dc35bb08dd1ca3dfb09dce91fd2d13294d6711c88897d9a9d60acf39bce049" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.95", ] [[package]] @@ -2905,7 +2906,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.95", ] [[package]] @@ -2971,7 +2972,7 @@ checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.95", ] [[package]] @@ -3039,7 +3040,7 @@ checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.95", ] [[package]] @@ -3141,7 +3142,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.95", ] [[package]] @@ -3375,7 +3376,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.95", "wasm-bindgen-shared", ] @@ -3409,7 +3410,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.95", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3663,7 +3664,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.95", ] [[package]] diff --git a/rust/cubestore/Cargo.lock b/rust/cubestore/Cargo.lock index 89352f63787dc..c6c32d50505b0 100644 --- a/rust/cubestore/Cargo.lock +++ b/rust/cubestore/Cargo.lock @@ -1148,6 +1148,13 @@ dependencies = [ "syn 1.0.107", ] +[[package]] +name = "cubeshared" +version = "0.1.0" +dependencies = [ + "flatbuffers 23.1.21", +] + [[package]] name = "cubestore" version = "0.1.0" @@ -1173,6 +1180,7 @@ dependencies = [ "cubehll", "cuberockstore", "cuberpc", + "cubeshared", "cubezetasketch", "datafusion", "deadqueue", diff --git a/rust/cubestore/Dockerfile b/rust/cubestore/Dockerfile index f522c4ef8e23f..db2ec1ec76cd1 100644 --- a/rust/cubestore/Dockerfile +++ b/rust/cubestore/Dockerfile @@ -1,15 +1,18 @@ FROM cubejs/rust-builder:bookworm-llvm-18 AS builder WORKDIR /build/cubestore -COPY Cargo.toml . -COPY Cargo.lock . -COPY cuberockstore cuberockstore -COPY cubehll cubehll -COPY cubezetasketch cubezetasketch -COPY cubedatasketches cubedatasketches -COPY cuberpc cuberpc -COPY cubestore-sql-tests cubestore-sql-tests -COPY cubestore/Cargo.toml cubestore/Cargo.toml + +COPY cubeshared /build/cubeshared + +COPY cubestore/Cargo.toml . +COPY cubestore/Cargo.lock . +COPY cubestore/cuberockstore cuberockstore +COPY cubestore/cubehll cubehll +COPY cubestore/cubezetasketch cubezetasketch +COPY cubestore/cubedatasketches cubedatasketches +COPY cubestore/cuberpc cuberpc +COPY cubestore/cubestore-sql-tests cubestore-sql-tests +COPY cubestore/cubestore/Cargo.toml cubestore/Cargo.toml RUN mkdir -p cubestore/src/bin && \ echo "fn main() {print!(\"Dummy main\");} // dummy file" > cubestore/src/bin/cubestored.rs @@ -18,8 +21,8 @@ RUN [ "$WITH_AVX2" -eq "1" ] && export RUSTFLAGS="-C target-feature=+avx2"; \ cargo build --release -p cubestore # Cube Store get version from his own package -COPY package.json package.json -COPY cubestore cubestore +COPY cubestore/package.json package.json +COPY cubestore/cubestore cubestore RUN [ "$WITH_AVX2" -eq "1" ] && export RUSTFLAGS="-C target-feature=+avx2"; \ cargo build --release -p cubestore diff --git a/rust/cubestore/cubestore/Cargo.toml b/rust/cubestore/cubestore/Cargo.toml index 2e3dc5f5caf2b..3e54d4428f00c 100644 --- a/rust/cubestore/cubestore/Cargo.toml +++ b/rust/cubestore/cubestore/Cargo.toml @@ -27,6 +27,7 @@ cuberockstore = { path = "../cuberockstore" } cubehll = { path = "../cubehll" } cubezetasketch = { path = "../cubezetasketch" } cubedatasketches = { path = "../cubedatasketches" } +cubeshared = { path = "../../cubeshared" } cuberpc = { path = "../cuberpc" } datafusion = { git = "https://github.com/cube-js/arrow-datafusion", branch = "cube", features = ["default_nulls_last"] } csv = "1.1.3" diff --git a/rust/cubestore/cubestore/src/http/mod.rs b/rust/cubestore/cubestore/src/http/mod.rs index f627a1f5f067e..e03fe51d0b425 100644 --- a/rust/cubestore/cubestore/src/http/mod.rs +++ b/rust/cubestore/cubestore/src/http/mod.rs @@ -4,11 +4,6 @@ use std::sync::Arc; use warp::{Filter, Rejection, Reply}; -use crate::codegen::{ - root_as_http_message, HttpColumnValue, HttpColumnValueArgs, HttpError, HttpErrorArgs, - HttpMessageArgs, HttpQuery, HttpQueryArgs, HttpResultSet, HttpResultSetArgs, HttpRow, - HttpRowArgs, -}; use crate::metastore::{Column, ColumnType, ImportFormat}; use crate::mysql::SqlAuthService; use crate::sql::{InlineTable, InlineTables, SqlQueryContext, SqlService}; @@ -17,6 +12,11 @@ use crate::table::{Row, TableValue}; use crate::util::WorkerLoop; use crate::CubeError; use async_std::fs::File; +use cubeshared::codegen::{ + root_as_http_message, HttpColumnValue, HttpColumnValueArgs, HttpError, HttpErrorArgs, + HttpMessageArgs, HttpQuery, HttpQueryArgs, HttpResultSet, HttpResultSetArgs, HttpRow, + HttpRowArgs, +}; use datafusion::cube_ext; use flatbuffers::{FlatBufferBuilder, ForwardsUOffset, Vector, WIPOffset}; use futures::{AsyncWriteExt, SinkExt, Stream, StreamExt}; @@ -603,10 +603,10 @@ impl HttpMessage { let args = HttpMessageArgs { message_id: self.message_id, command_type: match self.command { - HttpCommand::Query { .. } => crate::codegen::HttpCommand::HttpQuery, - HttpCommand::ResultSet { .. } => crate::codegen::HttpCommand::HttpResultSet, + HttpCommand::Query { .. } => cubeshared::codegen::HttpCommand::HttpQuery, + HttpCommand::ResultSet { .. } => cubeshared::codegen::HttpCommand::HttpResultSet, HttpCommand::CloseConnection { .. } | HttpCommand::Error { .. } => { - crate::codegen::HttpCommand::HttpError + cubeshared::codegen::HttpCommand::HttpError } }, command: match &self.command { @@ -666,7 +666,7 @@ impl HttpMessage { .as_ref() .map(|c| builder.create_string(c)), }; - let message = crate::codegen::HttpMessage::create(&mut builder, &args); + let message = cubeshared::codegen::HttpMessage::create(&mut builder, &args); builder.finish(message, None); builder.finished_data().to_vec() // TODO copy } @@ -762,7 +762,7 @@ impl HttpMessage { message_id: http_message.message_id(), connection_id: http_message.connection_id().map(|s| s.to_string()), command: match http_message.command_type() { - crate::codegen::HttpCommand::HttpQuery => { + cubeshared::codegen::HttpCommand::HttpQuery => { let query = http_message.command_as_http_query().unwrap(); let mut inline_tables = Vec::new(); if let Some(query_inline_tables) = query.inline_tables() { @@ -809,7 +809,7 @@ impl HttpMessage { trace_obj: query.trace_obj().map(|q| q.to_string()), } } - crate::codegen::HttpCommand::HttpResultSet => { + cubeshared::codegen::HttpCommand::HttpResultSet => { let result_set = http_message.command_as_http_result_set().unwrap(); let mut result_rows = Vec::new(); if let Some(rows) = result_set.rows() { @@ -857,7 +857,6 @@ impl HttpMessage { #[cfg(test)] mod tests { - use crate::codegen::{HttpMessageArgs, HttpQuery, HttpQueryArgs, HttpTable, HttpTableArgs}; use crate::config::{init_test_logger, Config}; use crate::http::{HttpCommand, HttpMessage, HttpServer}; use crate::metastore::{Column, ColumnType}; @@ -867,6 +866,9 @@ mod tests { use crate::table::{Row, TableValue}; use crate::CubeError; use async_trait::async_trait; + use cubeshared::codegen::{ + HttpMessageArgs, HttpQuery, HttpQueryArgs, HttpTable, HttpTableArgs, + }; use datafusion::cube_ext; use flatbuffers::{FlatBufferBuilder, ForwardsUOffset, Vector, WIPOffset}; use futures_util::{SinkExt, StreamExt}; @@ -973,11 +975,11 @@ mod tests { ); let args = HttpMessageArgs { message_id: 1234, - command_type: crate::codegen::HttpCommand::HttpQuery, + command_type: cubeshared::codegen::HttpCommand::HttpQuery, command: Some(query_value.as_union_value()), connection_id: Some(connection_id_offset), }; - let message = crate::codegen::HttpMessage::create(&mut builder, &args); + let message = cubeshared::codegen::HttpMessage::create(&mut builder, &args); builder.finish(message, None); let bytes = builder.finished_data().to_vec(); let message = HttpMessage::read(bytes).await.unwrap(); diff --git a/rust/cubestore/cubestore/src/lib.rs b/rust/cubestore/cubestore/src/lib.rs index 89ddb44e15599..05d24b86f0a14 100644 --- a/rust/cubestore/cubestore/src/lib.rs +++ b/rust/cubestore/cubestore/src/lib.rs @@ -38,7 +38,6 @@ use tokio::time::error::Elapsed; pub mod app_metrics; pub mod cachestore; pub mod cluster; -pub mod codegen; pub mod config; pub mod http; pub mod import;