diff --git a/packages/data-mate/package.json b/packages/data-mate/package.json index a583be4f0e3..5811bda2c9c 100644 --- a/packages/data-mate/package.json +++ b/packages/data-mate/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/data-mate", "displayName": "Data-Mate", - "version": "0.19.3", + "version": "0.19.4", "description": "Library of data validations/transformations", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/data-mate#readme", "repository": { diff --git a/packages/data-mate/src/aggregation-frame/AggregationFrame.ts b/packages/data-mate/src/aggregation-frame/AggregationFrame.ts index 5df9aaf27ad..b289c2e040c 100644 --- a/packages/data-mate/src/aggregation-frame/AggregationFrame.ts +++ b/packages/data-mate/src/aggregation-frame/AggregationFrame.ts @@ -5,7 +5,7 @@ import { KeyAggregation, ValueAggregation, valueAggMap, FieldAgg, KeyAggFn, keyAggMap, makeUniqueKeyAgg } from '../column'; -import { isNumberLike } from '../vector'; +import { getCommonTupleType, isNumberLike } from '../vector'; import { Builder } from '../builder'; import { getBuilderForField, getMaxColumnSize } from './utils'; import { @@ -128,10 +128,9 @@ export class AggregationFrame< field: keyof T, as?: A ): this|AggregationFrame> { - const { name, type } = this._ensureColumn(field, as); - if (!isNumberLike(type)) { - throw new Error(`${ValueAggregation.avg} requires a numeric field type`); - } + const { name } = this._ensureColumn(field, as); + this._ensureNumericLike(name, ValueAggregation.avg); + const aggObject = this._aggregations.get(name) ?? { }; aggObject.value = ValueAggregation.avg; this._aggregations.set(name, aggObject); @@ -154,10 +153,9 @@ export class AggregationFrame< field: keyof T, as?: A ): this|AggregationFrame> { - const { name, type } = this._ensureColumn(field, as); - if (!isNumberLike(type)) { - throw new Error(`${ValueAggregation.sum} requires a numeric field type`); - } + const { name } = this._ensureColumn(field, as); + this._ensureNumericLike(name, ValueAggregation.sum); + const aggObject = this._aggregations.get(name) ?? { }; aggObject.value = ValueAggregation.sum; this._aggregations.set(name, aggObject); @@ -180,10 +178,9 @@ export class AggregationFrame< field: keyof T, as?: A ): this|AggregationFrame> { - const { name, type } = this._ensureColumn(field, as); - if (!isNumberLike(type)) { - throw new Error(`${ValueAggregation.min} requires a numeric field type`); - } + const { name } = this._ensureColumn(field, as); + this._ensureNumericLike(name, ValueAggregation.min); + const aggObject = this._aggregations.get(name) ?? { }; aggObject.value = ValueAggregation.min; this._aggregations.set(name, aggObject); @@ -206,10 +203,9 @@ export class AggregationFrame< field: keyof T, as?: A ): this|AggregationFrame> { - const { name, type } = this._ensureColumn(field, as); - if (!isNumberLike(type)) { - throw new Error(`${ValueAggregation.max} requires a numeric field type`); - } + const { name } = this._ensureColumn(field, as); + this._ensureNumericLike(name, ValueAggregation.max); + const aggObject = this._aggregations.get(name) ?? { }; aggObject.value = ValueAggregation.max; this._aggregations.set(name, aggObject); @@ -461,8 +457,7 @@ export class AggregationFrame< name: keyof T, type: FieldType } { - const col = this.getColumn(field); - if (!col) throw new Error(`Unknown column named "${field}"`); + const col = this.getColumnOrThrow(field); if (as) { const columns: Column[] = []; @@ -486,6 +481,22 @@ export class AggregationFrame< }; } + private _ensureNumericLike(field: keyof T, ctx: string): void { + const column = this.getColumnOrThrow(field); + let fieldType: FieldType; + if (column.config.type === FieldType.Tuple) { + fieldType = getCommonTupleType( + field as string, column.vector.childConfig + ); + } else { + fieldType = column.config.type as FieldType; + } + + if (!isNumberLike(fieldType)) { + throw new Error(`${ctx} requires a numeric field type`); + } + } + /** * Execute the aggregations and flatten the grouped data. * Assigns the new columns to this. diff --git a/packages/data-mate/src/aggregation-frame/utils.ts b/packages/data-mate/src/aggregation-frame/utils.ts index 157a2d5eded..57d123d1108 100644 --- a/packages/data-mate/src/aggregation-frame/utils.ts +++ b/packages/data-mate/src/aggregation-frame/utils.ts @@ -3,7 +3,7 @@ import { Builder } from '../builder'; import { Column, ValueAggregation, KeyAggregation } from '../column'; import { WritableData } from '../core'; import { - isNumberLike, isFloatLike + isNumberLike, isFloatLike, getCommonTupleType } from '../vector'; export function getBuilderForField( @@ -29,7 +29,12 @@ export function getBuilderForField( }); } - const currentType = col.config.type as FieldType; + let currentType = col.config.type as FieldType; + if (currentType === FieldType.Tuple) { + currentType = getCommonTupleType( + col.name, col.vector.childConfig + ); + } let type: FieldType|undefined; if (valueAgg === ValueAggregation.avg) { if (currentType === FieldType.Long) { diff --git a/packages/data-mate/src/vector/utils.ts b/packages/data-mate/src/vector/utils.ts index e9dce5b4558..176b4a26fcc 100644 --- a/packages/data-mate/src/vector/utils.ts +++ b/packages/data-mate/src/vector/utils.ts @@ -1,8 +1,9 @@ import { + DataTypeFields, FieldType } from '@terascope/types'; import { - isNumber, isBigInt, getTypeOf, isArrayLike + isNumber, isBigInt, getTypeOf, isArrayLike, TSError } from '@terascope/utils'; import { ListVector } from './ListVector'; import { @@ -90,34 +91,34 @@ export function getNumericValues(value: unknown): { values: bigint[], type: 'bigint' } { - if (value == null) { - return { values: [], type: 'number' }; - } - - if (isNumber(value)) { - return { values: [value], type: 'number' }; - } - if (isBigInt(value)) { - return { values: [value], type: 'bigint' }; - } + let type: 'number'|'bigint' = 'number'; + const values: any[] = []; + function processValue(v: unknown): void { + if (v == null) return; - if (isArrayLike(value)) { - let type: 'number'|'bigint' = 'number'; - const values: any[] = []; - for (const v of value) { - if (v == null) continue; - if (type === 'number' && isBigInt(v)) { - type = 'bigint'; + if (isArrayLike(v)) { + for (const nested of v) { + processValue(nested); } - values.push(v); + return; + } + + if (!isNumber(v) && !isBigInt(v)) { + throw new Error(`Invalid to numeric values in ${v} (${getTypeOf(v)})`); + } + + if (type === 'number' && isBigInt(v)) { + type = 'bigint'; } - return { - values, - type - }; + + values.push(v); } + processValue(value); - throw new Error(`Unable to get numeric values from ${value} (${getTypeOf(value)})`); + return { + values, + type + }; } export function isNumberLike(type: FieldType): boolean { @@ -138,3 +139,32 @@ export function isIntLike(type: FieldType): boolean { if (type === FieldType.Integer) return true; return false; } + +export function getCommonTupleType( + tupleField: string, childConfig: DataTypeFields|undefined +): FieldType { + let fieldType: FieldType|undefined; + for (const config of Object.values(childConfig ?? {})) { + const type = config.type as FieldType; + + if (!fieldType || type === fieldType) { + fieldType = type; + } else if (isIntLike(fieldType) && isIntLike(type)) { + fieldType = FieldType.Integer; + } else if (isFloatLike(fieldType) && isFloatLike(type)) { + fieldType = FieldType.Float; + } else { + throw new TSError( + `Field "${tupleField}" has conflicting field types, ${fieldType} incompatible with ${type}`, + { statusCode: 400, context: { safe: true } } + ); + } + } + if (!fieldType) { + throw new TSError( + `Field "${tupleField}" has no child fields`, + { statusCode: 400, context: { safe: true } } + ); + } + return fieldType; +} diff --git a/packages/data-mate/test/aggregation-frame-spec.ts b/packages/data-mate/test/aggregation-frame-spec.ts index 6cbd88ab463..c016fbe1a9f 100644 --- a/packages/data-mate/test/aggregation-frame-spec.ts +++ b/packages/data-mate/test/aggregation-frame-spec.ts @@ -119,6 +119,21 @@ describe('AggregationFrame', () => { } ]); }); + + it('should get the right result when using tuple', async () => { + const grouped = dataFrame.createTupleFrom(['age'], 'age_tuple').aggregate(); + const resultFrame = await grouped.sum('age_tuple').run(); + expect(resultFrame.toJSON()).toEqual([ + { + name: 'Billy', + age: 64, + age_tuple: 310, + gender: 'M', + scores: [4, 9, 3], + date: '2020-09-15T17:39:11.195Z' + } + ]); + }); }); describe('->sum(scores)', () => { @@ -135,6 +150,21 @@ describe('AggregationFrame', () => { } ]); }); + + it('should get the right result when using tuple', async () => { + const grouped = dataFrame.createTupleFrom(['scores'], 'score_tuple').aggregate(); + const resultFrame = await grouped.sum('score_tuple').run(); + expect(resultFrame.toJSON()).toEqual([ + { + name: 'Billy', + age: 64, + gender: 'M', + scores: [4, 9, 3], + score_tuple: 177, + date: '2020-09-15T17:39:11.195Z' + } + ]); + }); }); describe('->avg(age)', () => { diff --git a/packages/elasticsearch-store/package.json b/packages/elasticsearch-store/package.json index bc3a0f4441c..7263eaf297c 100644 --- a/packages/elasticsearch-store/package.json +++ b/packages/elasticsearch-store/package.json @@ -1,7 +1,7 @@ { "name": "elasticsearch-store", "displayName": "Elasticsearch Store", - "version": "0.40.3", + "version": "0.40.4", "description": "An API for managing an elasticsearch index, with versioning and migration support.", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/elasticsearch-store#readme", "bugs": { @@ -24,7 +24,7 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/data-mate": "^0.19.3", + "@terascope/data-mate": "^0.19.4", "@terascope/data-types": "^0.25.1", "@terascope/types": "^0.7.0", "@terascope/utils": "^0.34.1", diff --git a/packages/ts-transforms/package.json b/packages/ts-transforms/package.json index a417bd12c49..b53744511c8 100644 --- a/packages/ts-transforms/package.json +++ b/packages/ts-transforms/package.json @@ -1,7 +1,7 @@ { "name": "ts-transforms", "displayName": "TS Transforms", - "version": "0.48.3", + "version": "0.48.4", "description": "An ETL framework built upon xlucene-evaluator", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/ts-transforms#readme", "bugs": { @@ -35,7 +35,7 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/data-mate": "^0.19.3", + "@terascope/data-mate": "^0.19.4", "@terascope/types": "^0.7.0", "@terascope/utils": "^0.34.1", "awesome-phonenumber": "^2.43.0",