diff --git a/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java b/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java index 87577cf2e24cc..e90665b14adbf 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java @@ -76,6 +76,9 @@ private static XContentBuilder createMinimalTestMapping(boolean invalidDim, bool .startObject() .field("name", "keyword_dv") .endObject() + .startObject() + .field("name", "unsignedLongDimension") // UnsignedLongDimension + .endObject() .endArray() .startArray("metrics") .startObject() @@ -117,6 +120,10 @@ private static XContentBuilder createMinimalTestMapping(boolean invalidDim, bool .field("type", "wildcard") .field("doc_values", false) .endObject() + .startObject("unsignedLongDimension") + .field("type", "unsigned_long") + .field("doc_values", true) + .endObject() .endObject() .endObject(); } catch (IOException e) { @@ -605,8 +612,11 @@ public void testValidCompositeIndex() { for (int i = 0; i < dateDim.getSortedCalendarIntervals().size(); i++) { assertEquals(expectedTimeUnits.get(i).shortName(), dateDim.getSortedCalendarIntervals().get(i).shortName()); } + assertEquals(4, starTreeFieldType.getDimensions().size()); assertEquals("numeric_dv", starTreeFieldType.getDimensions().get(1).getField()); assertEquals("keyword_dv", starTreeFieldType.getDimensions().get(2).getField()); + assertEquals("unsignedLongDimension", starTreeFieldType.getDimensions().get(3).getField()); + assertEquals("numeric_dv", starTreeFieldType.getMetrics().get(0).getField()); List expectedMetrics = Arrays.asList(MetricStat.VALUE_COUNT, MetricStat.SUM, MetricStat.AVG); assertEquals(expectedMetrics, starTreeFieldType.getMetrics().get(0).getMetrics()); diff --git a/server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912DocValuesReader.java b/server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912DocValuesReader.java index 7178ffbadf9f1..ebb6afae57f02 100644 --- a/server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912DocValuesReader.java +++ b/server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912DocValuesReader.java @@ -36,6 +36,7 @@ import org.opensearch.index.compositeindex.CompositeIndexMetadata; import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.DimensionConfig; import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.StarTreeMetadata; import org.opensearch.index.compositeindex.datacube.startree.index.CompositeIndexValues; import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; @@ -157,15 +158,15 @@ public Composite912DocValuesReader(DocValuesProducer producer, SegmentReadState compositeIndexInputMap.put(compositeFieldName, starTreeIndexInput); compositeIndexMetadataMap.put(compositeFieldName, starTreeMetadata); - Map dimensionFieldToDocValuesMap = starTreeMetadata.getDimensionFields(); + Map dimensionFieldToDocValuesMap = starTreeMetadata.getDimensionFields(); // generating star tree unique fields (fully qualified name for dimension and metrics) - for (Map.Entry dimensionEntry : dimensionFieldToDocValuesMap.entrySet()) { + for (Map.Entry dimensionEntry : dimensionFieldToDocValuesMap.entrySet()) { String dimName = fullyQualifiedFieldNameForStarTreeDimensionsDocValues( compositeFieldName, dimensionEntry.getKey() ); fields.add(dimName); - dimensionFieldTypeMap.put(dimName, dimensionEntry.getValue()); + dimensionFieldTypeMap.put(dimName, dimensionEntry.getValue().getDocValuesType()); } // adding metric fields for (Metric metric : starTreeMetadata.getMetrics()) { diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/Dimension.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/Dimension.java index 3d71b38881693..366a979946e5f 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/Dimension.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/Dimension.java @@ -12,6 +12,7 @@ import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.core.xcontent.ToXContent; +import java.util.Comparator; import java.util.List; import java.util.function.Consumer; @@ -34,8 +35,8 @@ public interface Dimension extends ToXContent { /** * Sets the dimension values with the consumer * - * @param value The value to be set - * @param dimSetter Consumer which sets the dimensions + * @param value The value to be set + * @param dimSetter Consumer which sets the dimensions */ void setDimensionValues(final Long value, final Consumer dimSetter); @@ -45,4 +46,19 @@ public interface Dimension extends ToXContent { List getSubDimensionNames(); DocValuesType getDocValuesType(); + + /** + * Returns the dimensionDataType used for comparing and parsing dimension values.
+ * This determines how numeric values are compared and parsed:
+ * - DimensionDataType.UNSIGNED_LONG for unsigned long values
+ * - DimensionDataType.LONG for all other numeric types (DEFAULT) + */ + default DimensionDataType getDimensionDataType() { + return DimensionDataType.LONG; + } + + default Comparator comparator() { + return (a, b) -> getDimensionDataType().compare(a, b); + } + } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionDataType.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionDataType.java new file mode 100644 index 0000000000000..67138b69c69fa --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionDataType.java @@ -0,0 +1,52 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube; + +import org.opensearch.common.annotation.ExperimentalApi; + +/** + * Represents the data type of the dimension value. + * + * @opensearch.experimental + */ +@ExperimentalApi +public enum DimensionDataType { + LONG { + @Override + int compare(Long a, Long b) { + if (a == null && b == null) { + return 0; + } + if (b == null) { + return -1; + } + if (a == null) { + return 1; + } + return Long.compare(a, b); + } + }, + UNSIGNED_LONG { + @Override + int compare(Long a, Long b) { + if (a == null && b == null) { + return 0; + } + if (b == null) { + return -1; + } + if (a == null) { + return 1; + } + return Long.compareUnsigned(a, b); + } + }; + + abstract int compare(Long a, Long b); +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionFactory.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionFactory.java index b1e78d78d3ad2..aed9d1d56a188 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionFactory.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionFactory.java @@ -45,6 +45,8 @@ public static Dimension parseAndCreateDimension( return parseAndCreateDateDimension(name, dimensionMap, c); case NumericDimension.NUMERIC: return new NumericDimension(name); + case UnsignedLongDimension.UNSIGNED_LONG: + return new UnsignedLongDimension(name); case ORDINAL: return new OrdinalDimension(name); case IP: @@ -72,6 +74,8 @@ public static Dimension parseAndCreateDimension( return parseAndCreateDateDimension(name, dimensionMap, c); case NUMERIC: return new NumericDimension(name); + case UNSIGNED_LONG: + return new UnsignedLongDimension(name); case ORDINAL: return new OrdinalDimension(name); case IP: diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionType.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionType.java index f7911e72f36fc..8f9db7df596ed 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionType.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionType.java @@ -23,6 +23,12 @@ public enum DimensionType { */ NUMERIC, + /** + * Represents an unsigned long dimension type. + * This is used for dimensions that contain numerical values of type unsigned long. + */ + UNSIGNED_LONG, + /** * Represents a date dimension type. * This is used for dimensions that contain date or timestamp values. diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/ReadDimension.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/ReadDimension.java index 384553a8f7e06..5a791188982ce 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/ReadDimension.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/ReadDimension.java @@ -26,15 +26,24 @@ public class ReadDimension implements Dimension { public static final String READ = "read"; private final String field; private final DocValuesType docValuesType; + private final DimensionDataType dimensionDataType; public ReadDimension(String field) { this.field = field; this.docValuesType = DocValuesType.SORTED_NUMERIC; + this.dimensionDataType = DimensionDataType.LONG; } public ReadDimension(String field, DocValuesType docValuesType) { this.field = field; this.docValuesType = docValuesType; + this.dimensionDataType = DimensionDataType.LONG; + } + + public ReadDimension(String field, DocValuesType docValuesType, DimensionDataType dimensionDataType) { + this.field = field; + this.docValuesType = docValuesType; + this.dimensionDataType = dimensionDataType; } public String getField() { @@ -82,4 +91,10 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(field); } + + @Override + public DimensionDataType getDimensionDataType() { + return dimensionDataType; + } + } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/UnsignedLongDimension.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/UnsignedLongDimension.java new file mode 100644 index 0000000000000..21f1b291f4821 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/UnsignedLongDimension.java @@ -0,0 +1,43 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube; + +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.index.mapper.CompositeDataCubeFieldType; + +import java.io.IOException; + +/** + * Unsigned Long dimension class + * + * @opensearch.experimental + */ +public class UnsignedLongDimension extends NumericDimension { + + public static final String UNSIGNED_LONG = "unsigned_long"; + + public UnsignedLongDimension(String field) { + super(field); + } + + @Override + public DimensionDataType getDimensionDataType() { + return DimensionDataType.UNSIGNED_LONG; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(CompositeDataCubeFieldType.NAME, getField()); + builder.field(CompositeDataCubeFieldType.TYPE, UNSIGNED_LONG); + builder.endObject(); + return builder; + } + +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java index cf36f2d7d4126..935c490b5a4dc 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java @@ -55,6 +55,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -112,6 +113,8 @@ public abstract class BaseStarTreeBuilder implements StarTreeBuilder { // This should be true for merge flows protected boolean isMerge = false; + protected final List> dimensionComparators = new ArrayList<>(); + /** * Reads all the configuration related to dimensions and metrics, builds a star-tree based on the different construction parameters. * @@ -136,6 +139,9 @@ protected BaseStarTreeBuilder( int numDims = 0; for (Dimension dim : starTreeField.getDimensionsOrder()) { numDims += dim.getNumSubDimensions(); + for (int i = 0; i < dim.getNumSubDimensions(); i++) { + dimensionComparators.add(dim.comparator()); + } dimensionsSplitOrder.add(dim); } this.numDimensions = numDims; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilder.java index 63659ef684744..da48559461c49 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilder.java @@ -235,7 +235,7 @@ private Iterator sortAndReduceDocuments(int[] sortedDocIds, in } catch (IOException e) { throw new UncheckedIOException(e); } - }); + }, dimensionComparators); } catch (UncheckedIOException ex) { // Unwrap UncheckedIOException and throw as IOException if (ex.getCause() != null) { @@ -308,6 +308,7 @@ public List getStarTreeDocuments() throws IOException { @Override public Long getDimensionValue(int docId, int dimensionId) throws IOException { return starTreeDocumentFileManager.getDimensionValue(docId, dimensionId); + } /** @@ -334,7 +335,8 @@ public Iterator generateStarTreeDocumentsForStarNode(int start } catch (IOException e) { throw new RuntimeException(e); } - }); + }, dimensionComparators); + // Create an iterator for aggregated documents return new Iterator() { boolean hasNext = true; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java index c91f4c5db98bb..a590b2b69cbc8 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java @@ -42,8 +42,8 @@ public class OnHeapStarTreeBuilder extends BaseStarTreeBuilder { /** * Constructor for OnHeapStarTreeBuilder * - * @param metaOut an index output to write star-tree metadata - * @param dataOut an index output to write star-tree data + * @param metaOut an index output to write star-tree metadata + * @param dataOut an index output to write star-tree data * @param starTreeField star-tree field * @param segmentWriteState segment write state * @param mapperService helps with the numeric type of field @@ -82,9 +82,8 @@ public Long getDimensionValue(int docId, int dimensionId) { * Sorts and aggregates all the documents of the segment based on dimension and metrics configuration * * @param dimensionReaders List of docValues readers to read dimensions from the segment - * @param metricReaders List of docValues readers to read metrics from the segment + * @param metricReaders List of docValues readers to read metrics from the segment * @return Iterator of star-tree documents - * */ @Override public Iterator sortAndAggregateSegmentDocuments( @@ -161,7 +160,7 @@ StarTreeDocument[] getSegmentsStarTreeDocuments(List starTreeVal Iterator sortAndAggregateStarTreeDocuments(StarTreeDocument[] starTreeDocuments, boolean isMerge) { // sort all the documents - sortStarTreeDocumentsFromDimensionId(starTreeDocuments, 0); + sortStarTreeDocumentsFromDimensionId(starTreeDocuments, -1); // merge the documents return mergeStarTreeDocuments(starTreeDocuments, isMerge); @@ -222,7 +221,7 @@ public Iterator generateStarTreeDocumentsForStarNode(int start } // sort star tree documents from given dimension id (as previous dimension ids have already been processed) - sortStarTreeDocumentsFromDimensionId(starTreeDocuments, dimensionId + 1); + sortStarTreeDocumentsFromDimensionId(starTreeDocuments, dimensionId); return new Iterator() { boolean hasNext = true; @@ -267,22 +266,13 @@ public StarTreeDocument next() { * Sorts the star-tree documents from the given dimension id * * @param starTreeDocuments star-tree documents - * @param dimensionId id of the dimension + * @param dimensionId id of the dimension */ private void sortStarTreeDocumentsFromDimensionId(StarTreeDocument[] starTreeDocuments, int dimensionId) { - Arrays.sort(starTreeDocuments, (o1, o2) -> { - for (int i = dimensionId; i < numDimensions; i++) { - if (!Objects.equals(o1.dimensions[i], o2.dimensions[i])) { - if (o1.dimensions[i] == null && o2.dimensions[i] == null) { - return 0; - } - if (o1.dimensions[i] == null) { - return 1; - } - if (o2.dimensions[i] == null) { - return -1; - } - return Long.compare(o1.dimensions[i], o2.dimensions[i]); + Arrays.sort(starTreeDocuments, (doc1, doc2) -> { + for (int i = dimensionId + 1; i < numDimensions; i++) { + if (!Objects.equals(doc1.dimensions[i], doc2.dimensions[i])) { + return dimensionComparators.get(i).compare(doc1.dimensions[i], doc2.dimensions[i]); } } return 0; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/StarTreeWriter.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/StarTreeWriter.java index e5890be3ccb5b..e888235f60ee2 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/StarTreeWriter.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/StarTreeWriter.java @@ -27,8 +27,11 @@ public class StarTreeWriter { /** Initial version for the star tree writer */ public static final int VERSION_START = 0; + /** Version for the star tree writer with updated metadata which handles unsigned long */ + public static final int VERSION_DIMENSION_DATA_TYPE = 1; + /** Current version for the star tree writer */ - public static final int VERSION_CURRENT = VERSION_START; + public static final int VERSION_CURRENT = VERSION_DIMENSION_DATA_TYPE; public StarTreeWriter() {} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/DimensionConfig.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/DimensionConfig.java new file mode 100644 index 0000000000000..4515cdde34add --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/DimensionConfig.java @@ -0,0 +1,38 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.fileformats.meta; + +import org.apache.lucene.index.DocValuesType; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.index.compositeindex.datacube.DimensionDataType; + +/** + * Class to store DocValuesType and DimensionDataType for a dimension. + * + * @opensearch.experimental + */ +@ExperimentalApi +public class DimensionConfig { + + private final DocValuesType docValuesType; + private final DimensionDataType dimensionDataType; + + public DimensionConfig(DocValuesType docValuesType, DimensionDataType dimensionDataType) { + this.docValuesType = docValuesType; + this.dimensionDataType = dimensionDataType; + } + + public DocValuesType getDocValuesType() { + return docValuesType; + } + + public DimensionDataType getDimensionDataType() { + return dimensionDataType; + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadata.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadata.java index 57e47b1a5b9d9..6a28fcff5eafa 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadata.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadata.java @@ -14,9 +14,11 @@ import org.apache.lucene.store.IndexInput; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.index.compositeindex.CompositeIndexMetadata; +import org.opensearch.index.compositeindex.datacube.DimensionDataType; import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; +import org.opensearch.index.compositeindex.datacube.startree.fileformats.StarTreeWriter; import org.opensearch.index.mapper.CompositeMappedFieldType; import java.io.IOException; @@ -63,10 +65,10 @@ public class StarTreeMetadata extends CompositeIndexMetadata { private final String starTreeFieldType; /** - * Map of dimension fields to their associated DocValuesType.Insertion order needs to be maintained + * Map of dimension fields to their associated DocValuesType. Insertion order needs to be maintained * as it dictates dimensionSplitOrder */ - LinkedHashMap dimensionFieldsToDocValuesMap; + private LinkedHashMap dimensionFieldToDimensionConfigMap; /** * List of metrics, containing field names and associated metric statistics. @@ -114,7 +116,7 @@ public class StarTreeMetadata extends CompositeIndexMetadata { * @param metaIn an index input to read star-tree meta * @param compositeFieldName name of the composite field. Here, name of the star-tree field. * @param compositeFieldType type of the composite field. Here, STAR_TREE field. - * @param version The version of the star tree stored in the segments. + * @param version The version of the star tree stored in the segments. * @throws IOException if unable to read star-tree metadata from the file */ public StarTreeMetadata( @@ -130,7 +132,7 @@ public StarTreeMetadata( this.starTreeFieldType = this.getCompositeFieldType().getName(); this.version = version; this.numberOfNodes = readNumberOfNodes(); - this.dimensionFieldsToDocValuesMap = readStarTreeDimensions(); + this.dimensionFieldToDimensionConfigMap = readStarTreeDimensions(); this.metrics = readMetricEntries(); this.segmentAggregatedDocCount = readSegmentAggregatedDocCount(); this.starTreeDocCount = readStarTreeDocCount(); @@ -149,19 +151,19 @@ public StarTreeMetadata( * A star tree metadata constructor to initialize star tree metadata. * Used for testing. * - * @param meta an index input to read star-tree meta - * @param compositeFieldName name of the composite field. Here, name of the star-tree field. - * @param compositeFieldType type of the composite field. Here, STAR_TREE field. - * @param version The version of the star tree stored in the segments. - * @param dimensionFieldsToDocValuesMap map of dimensionFields to docValues - * @param metrics list of metric entries - * @param segmentAggregatedDocCount segment aggregated doc count - * @param starTreeDocCount the total number of star tree documents for the segment - * @param maxLeafDocs max leaf docs - * @param skipStarNodeCreationInDims set of dimensions to skip star node creation - * @param starTreeBuildMode star tree build mode - * @param dataStartFilePointer star file pointer to the associated star tree data in (.cid) file - * @param dataLength length of the corresponding star-tree data in (.cid) file + * @param meta an index input to read star-tree meta + * @param compositeFieldName name of the composite field. Here, name of the star-tree field. + * @param compositeFieldType type of the composite field. Here, STAR_TREE field. + * @param version The version of the star tree stored in the segments. + * @param dimensionFieldToDimensionConfigMap map of dimensionFields to Dimension config + * @param metrics list of metric entries + * @param segmentAggregatedDocCount segment aggregated doc count + * @param starTreeDocCount the total number of star tree documents for the segment + * @param maxLeafDocs max leaf docs + * @param skipStarNodeCreationInDims set of dimensions to skip star node creation + * @param starTreeBuildMode star tree build mode + * @param dataStartFilePointer star file pointer to the associated star tree data in (.cid) file + * @param dataLength length of the corresponding star-tree data in (.cid) file */ public StarTreeMetadata( String compositeFieldName, @@ -169,7 +171,7 @@ public StarTreeMetadata( IndexInput meta, Integer version, Integer numberOfNodes, - LinkedHashMap dimensionFieldsToDocValuesMap, + LinkedHashMap dimensionFieldToDimensionConfigMap, List metrics, Integer segmentAggregatedDocCount, Integer starTreeDocCount, @@ -185,7 +187,7 @@ public StarTreeMetadata( this.starTreeFieldType = compositeFieldType.getName(); this.version = version; this.numberOfNodes = numberOfNodes; - this.dimensionFieldsToDocValuesMap = dimensionFieldsToDocValuesMap; + this.dimensionFieldToDimensionConfigMap = dimensionFieldToDimensionConfigMap; this.metrics = metrics; this.segmentAggregatedDocCount = segmentAggregatedDocCount; this.starTreeDocCount = starTreeDocCount; @@ -204,14 +206,24 @@ private int readDimensionsCount() throws IOException { return meta.readVInt(); } - private LinkedHashMap readStarTreeDimensions() throws IOException { + private LinkedHashMap readStarTreeDimensions() throws IOException { int dimensionCount = readDimensionsCount(); - LinkedHashMap dimensionFieldsToDocValuesMap = new LinkedHashMap<>(); + LinkedHashMap dimensionFieldToDimensionConfigMap = new LinkedHashMap<>(); for (int i = 0; i < dimensionCount; i++) { - dimensionFieldsToDocValuesMap.put(meta.readString(), getDocValuesType(meta, meta.readByte())); + if (getVersion() >= StarTreeWriter.VERSION_DIMENSION_DATA_TYPE) { + dimensionFieldToDimensionConfigMap.put( + meta.readString(), + new DimensionConfig(getDocValuesType(meta, meta.readByte()), getDimensionDataType(meta, meta.readByte())) + ); + } else { + dimensionFieldToDimensionConfigMap.put( + meta.readString(), + new DimensionConfig(getDocValuesType(meta, meta.readByte()), DimensionDataType.LONG) + ); + } } - return dimensionFieldsToDocValuesMap; + return dimensionFieldToDimensionConfigMap; } private int readMetricsCount() throws IOException { @@ -315,8 +327,8 @@ public String getStarTreeFieldType() { * * @return star-tree dimension field numbers */ - public Map getDimensionFields() { - return dimensionFieldsToDocValuesMap; + public Map getDimensionFields() { + return dimensionFieldToDimensionConfigMap; } /** @@ -393,6 +405,7 @@ public long getDataLength() { /** * Returns the version with which the star tree is stored in the segments + * * @return star-tree version */ public int getVersion() { @@ -401,6 +414,7 @@ public int getVersion() { /** * Returns the number of nodes in the star tree + * * @return number of nodes in the star tree */ public int getNumberOfNodes() { @@ -425,4 +439,15 @@ private static DocValuesType getDocValuesType(IndexInput input, byte b) throws I throw new CorruptIndexException("invalid docvalues byte: " + b, input); } } + + private static DimensionDataType getDimensionDataType(IndexInput input, byte b) throws IOException { + switch (b) { + case 0: + return DimensionDataType.LONG; + case 1: + return DimensionDataType.UNSIGNED_LONG; + default: + throw new CorruptIndexException("invalid dimensionDataType byte: " + b, input); + } + } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataWriter.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataWriter.java index 569692ce18893..5bf9c6174fd2f 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataWriter.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataWriter.java @@ -12,6 +12,8 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.store.IndexOutput; +import org.opensearch.index.compositeindex.datacube.Dimension; +import org.opensearch.index.compositeindex.datacube.DimensionDataType; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricAggregatorInfo; import org.opensearch.index.mapper.CompositeMappedFieldType; @@ -131,9 +133,20 @@ private static void writeMeta( metaOut.writeVInt(starTreeField.getDimensionNames().size()); // dimensions - for (int i = 0; i < starTreeField.getDimensionNames().size(); i++) { - metaOut.writeString(starTreeField.getDimensionNames().get(i)); - metaOut.writeByte(docValuesByte(starTreeField.getDimensionDocValueTypes().get(i))); + List dimensionsOrder = starTreeField.getDimensionsOrder(); + int docDimensionIndex = 0; + for (Dimension currentDimension : dimensionsOrder) { + int numSubDimensions = currentDimension.getNumSubDimensions(); + + // Process each sub-dimension + while (numSubDimensions > 0) { + metaOut.writeString(starTreeField.getDimensionNames().get(docDimensionIndex)); + metaOut.writeByte(docValuesByte(starTreeField.getDimensionDocValueTypes().get(docDimensionIndex))); + metaOut.writeByte(dimensionDataTypeByte(currentDimension.getDimensionDataType())); + + numSubDimensions--; + docDimensionIndex++; + } } // number of metrics @@ -174,6 +187,17 @@ private static void writeMeta( } + private static byte dimensionDataTypeByte(DimensionDataType dimensionDataType) { + switch (dimensionDataType) { + case LONG: + return 0; + case UNSIGNED_LONG: + return 1; + default: + throw new AssertionError("unhandled dimensionDataType: " + dimensionDataType); + } + } + private static byte docValuesByte(DocValuesType type) { switch (type) { case NONE: diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java index 6a13e6e789f3a..6658d53afd21a 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java @@ -24,6 +24,7 @@ import org.opensearch.index.compositeindex.datacube.ReadDimension; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; +import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.DimensionConfig; import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.StarTreeMetadata; import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeFactory; import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNode; @@ -131,13 +132,15 @@ public StarTreeValues( // build dimensions List readDimensions = new ArrayList<>(); - for (String dimension : starTreeMetadata.getDimensionFields().keySet()) { + for (Map.Entry dimensionEntry : starTreeMetadata.getDimensionFields().entrySet()) { + String dimension = dimensionEntry.getKey(); readDimensions.add( new ReadDimension( dimension, readState.fieldInfos.fieldInfo( fullyQualifiedFieldNameForStarTreeDimensionsDocValues(starTreeMetadata.getCompositeFieldName(), dimension) - ).getDocValuesType() + ).getDocValuesType(), + dimensionEntry.getValue().getDimensionDataType() ) ); } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNode.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNode.java index fce3e30e9ebf6..3767f6850002a 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNode.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNode.java @@ -89,13 +89,13 @@ public interface StarTreeNode { * *

The node type can be one of the following: *

    - *
  • Star Node: Represented by the value -2. - *
  • Null Node: Represented by the value -1. + *
  • Star Node: Represented by the value -1. + *
  • Null Node: Represented by the value 1. *
  • Default Node: Represented by the value 0. *
* @see StarTreeNodeType * - * @return The type of the current node, represented by the corresponding integer value (-2, -1, or 0). + * @return The type of the current node, represented by the corresponding integer value (-1, 1, 0). * @throws IOException if an I/O error occurs while reading the node type */ byte getStarTreeNodeType() throws IOException; @@ -103,7 +103,7 @@ public interface StarTreeNode { /** * Returns the child node for the given dimension value in the star-tree. * - * @param dimensionValue the dimension value + * @param dimensionValue the dimension value * @return the child node for the given dimension value or null if child is not present * @throws IOException if an I/O error occurs while retrieving the child node */ diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeDocumentsSorter.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeDocumentsSorter.java index 7b1c63bc611ee..09a653f13cf28 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeDocumentsSorter.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeDocumentsSorter.java @@ -10,6 +10,8 @@ import org.apache.lucene.util.IntroSorter; +import java.util.Comparator; +import java.util.List; import java.util.Objects; import java.util.function.IntFunction; @@ -24,7 +26,8 @@ public static void sort( final int[] sortedDocIds, final int dimensionId, final int numDocs, - final IntFunction dimensionsReader + final IntFunction dimensionsReader, + final List> dimensionComparators ) { new IntroSorter() { private Long[] dimensions; @@ -45,18 +48,8 @@ protected void setPivot(int i) { protected int comparePivot(int j) { Long[] currentDimensions = dimensionsReader.apply(j); for (int i = dimensionId + 1; i < dimensions.length; i++) { - Long dimension = currentDimensions[i]; - if (!Objects.equals(dimensions[i], dimension)) { - if (dimensions[i] == null && dimension == null) { - return 0; - } - if (dimension == null) { - return -1; - } - if (dimensions[i] == null) { - return 1; - } - return Long.compare(dimensions[i], dimension); + if (!Objects.equals(dimensions[i], currentDimensions[i])) { + return dimensionComparators.get(i).compare(dimensions[i], currentDimensions[i]); } } return 0; diff --git a/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java index 702e5db50e841..f3fc3f4b2aa95 100644 --- a/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java @@ -91,7 +91,7 @@ import org.roaringbitmap.RoaringBitmap; /** - * A {@link FieldMapper} for numeric types: byte, short, int, long, float and double. + * A {@link FieldMapper} for numeric types: byte, short, int, long, float, double and unsigned long. * * @opensearch.internal */ @@ -175,13 +175,9 @@ public NumberFieldMapper build(BuilderContext context) { @Override public Optional getSupportedDataCubeDimensionType() { - - // unsigned long is not supported as dimension for star tree - if (type.numericType.equals(NumericType.UNSIGNED_LONG)) { - return Optional.empty(); - } - - return Optional.of(DimensionType.NUMERIC); + return type.numericType.equals(NumericType.UNSIGNED_LONG) + ? Optional.of(DimensionType.UNSIGNED_LONG) + : Optional.of(DimensionType.NUMERIC); } @Override diff --git a/server/src/main/java/org/opensearch/search/startree/StarTreeFilter.java b/server/src/main/java/org/opensearch/search/startree/StarTreeFilter.java index ea2c43a40f330..1629b9d0c1db4 100644 --- a/server/src/main/java/org/opensearch/search/startree/StarTreeFilter.java +++ b/server/src/main/java/org/opensearch/search/startree/StarTreeFilter.java @@ -123,11 +123,8 @@ private static StarTreeResult traverseStarTree(StarTreeValues starTreeValues, Ma DocIdSetBuilder.BulkAdder adder; Set globalRemainingPredicateColumns = null; StarTreeNode starTree = starTreeValues.getRoot(); - List dimensionNames = starTreeValues.getStarTreeField() - .getDimensionsOrder() - .stream() - .map(Dimension::getField) - .collect(Collectors.toList()); + List dimensionsOrder = starTreeValues.getStarTreeField().getDimensionsOrder(); + List dimensionNames = dimensionsOrder.stream().map(Dimension::getField).collect(Collectors.toList()); boolean foundLeafNode = starTree.isLeaf(); assert foundLeafNode == false; // root node is never leaf Queue queue = new ArrayDeque<>(); diff --git a/server/src/test/java/org/opensearch/index/codec/composite912/datacube/startree/StarTreeDocValuesFormatTests.java b/server/src/test/java/org/opensearch/index/codec/composite912/datacube/startree/StarTreeDocValuesFormatTests.java index 03798c6e4ce55..b2572af042b9c 100644 --- a/server/src/test/java/org/opensearch/index/codec/composite912/datacube/startree/StarTreeDocValuesFormatTests.java +++ b/server/src/test/java/org/opensearch/index/codec/composite912/datacube/startree/StarTreeDocValuesFormatTests.java @@ -57,22 +57,26 @@ public void testStarTreeDocValues() throws IOException { conf.setMergePolicy(newLogMergePolicy()); RandomIndexWriter iw = new RandomIndexWriter(random(), directory, conf); Document doc = new Document(); + doc.add(new SortedNumericDocValuesField("unsignedLongDimension", 10)); doc.add(new SortedNumericDocValuesField("sndv", 1)); doc.add(new SortedNumericDocValuesField("dv1", 1)); doc.add(new SortedNumericDocValuesField("field1", -1)); iw.addDocument(doc); doc = new Document(); + doc.add(new SortedNumericDocValuesField("unsignedLongDimension", 10)); doc.add(new SortedNumericDocValuesField("sndv", 1)); doc.add(new SortedNumericDocValuesField("dv1", 1)); doc.add(new SortedNumericDocValuesField("field1", -1)); iw.addDocument(doc); doc = new Document(); iw.forceMerge(1); + doc.add(new SortedNumericDocValuesField("unsignedLongDimension", -20)); doc.add(new SortedNumericDocValuesField("sndv", 2)); doc.add(new SortedNumericDocValuesField("dv1", 2)); doc.add(new SortedNumericDocValuesField("field1", -2)); iw.addDocument(doc); doc = new Document(); + doc.add(new SortedNumericDocValuesField("unsignedLongDimension", -20)); doc.add(new SortedNumericDocValuesField("sndv", 2)); doc.add(new SortedNumericDocValuesField("dv1", 2)); doc.add(new SortedNumericDocValuesField("field1", -2)); @@ -86,35 +90,35 @@ public void testStarTreeDocValues() throws IOException { // Segment documents /** - * sndv dv field - * [1, 1, -1] - * [1, 1, -1] - * [2, 2, -2] - * [2, 2, -2] + * unsignedLongDimension sndv dv field + * [10, 1, 1, -1] + * [10, 1, 1, -1] + * [-20, 2, 2, -2] + * [-20, 2, 2, -2] */ - // Star tree docuements + // Star tree documents /** - * sndv dv | [ sum, value_count, min, max[field]] , [ sum, value_count, min, max[sndv]], doc_count - * [1, 1] | [-2.0, 2.0, -1.0, -1.0, 2.0, 2.0, 1.0, 1.0, 2.0] - * [2, 2] | [-4.0, 2.0, -2.0, -2.0, 4.0, 2.0, 2.0, 2.0, 2.0] - * [null, 1] | [-2.0, 2.0, -1.0, -1.0, 2.0, 2.0, 1.0, 1.0, 2.0] - * [null, 2] | [-4.0, 2.0, -2.0, -2.0, 4.0, 2.0, 2.0, 2.0, 2.0] + * unsignedLongDimension sndv dv | [ sum, value_count, min, max[field]] , [ sum, value_count, min, max[sndv]], doc_count + * [10, 1, 1] | [-2.0, 2.0, -1.0, -1.0, 2.0, 2.0, 1.0, 1.0, 2.0] + * [-20, 2, 2] | [-4.0, 2.0, -2.0, -2.0, 4.0, 2.0, 2.0, 2.0, 2.0] + * [null, 1, 1] | [-2.0, 2.0, -1.0, -1.0, 2.0, 2.0, 1.0, 1.0, 2.0] + * [null, 2, 2] | [-4.0, 2.0, -2.0, -2.0, 4.0, 2.0, 2.0, 2.0, 2.0] */ StarTreeDocument[] expectedStarTreeDocuments = new StarTreeDocument[4]; expectedStarTreeDocuments[0] = new StarTreeDocument( - new Long[] { 1L, 1L }, + new Long[] { 10L, 1L, 1L }, new Double[] { -2.0, 2.0, -1.0, -1.0, 2.0, 2.0, 1.0, 1.0, 2.0 } ); expectedStarTreeDocuments[1] = new StarTreeDocument( - new Long[] { 2L, 2L }, + new Long[] { -20L, 2L, 2L }, new Double[] { -4.0, 2.0, -2.0, -2.0, 4.0, 2.0, 2.0, 2.0, 2.0 } ); expectedStarTreeDocuments[2] = new StarTreeDocument( - new Long[] { null, 1L }, + new Long[] { null, 1L, 1L }, new Double[] { -2.0, 2.0, -1.0, -1.0, 2.0, 2.0, 1.0, 1.0, 2.0 } ); expectedStarTreeDocuments[3] = new StarTreeDocument( - new Long[] { null, 2L }, + new Long[] { null, 2L, 2L }, new Double[] { -4.0, 2.0, -2.0, -2.0, 4.0, 2.0, 2.0, 2.0, 2.0 } ); @@ -264,6 +268,9 @@ public static XContentBuilder getExpandedMapping() throws IOException { b.field("max_leaf_docs", 1); b.startArray("ordered_dimensions"); b.startObject(); + b.field("name", "unsignedLongDimension"); // UnsignedLongDimension + b.endObject(); + b.startObject(); b.field("name", "sndv"); b.endObject(); b.startObject(); @@ -305,6 +312,9 @@ public static XContentBuilder getExpandedMapping() throws IOException { b.startObject("field1"); b.field("type", "integer"); b.endObject(); + b.startObject("unsignedLongDimension"); + b.field("type", "unsigned_long"); + b.endObject(); b.endObject(); }); } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java index 44e40f1db4cc8..7e8ecf79d7443 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java @@ -161,7 +161,8 @@ public static void validateFileFormats( IndexInput dataIn, IndexInput metaIn, InMemoryTreeNode rootNode, - StarTreeMetadata expectedStarTreeMetadata + StarTreeMetadata expectedStarTreeMetadata, + StarTreeField starTreeField ) throws IOException { long magicMarker = metaIn.readLong(); assertEquals(COMPOSITE_FIELD_MARKER, magicMarker); @@ -201,11 +202,18 @@ public static void validateFileFormats( if (rootNode.getChildren() != null) { sortedChildren = new ArrayList<>(rootNode.getChildren().values()); } - - if (starTreeNode.getChildDimensionId() != -1) { + int dimensionId = starTreeNode.getChildDimensionId(); + List dimensionsOrder = starTreeField.getDimensionsOrder(); + if (dimensionId != -1) { assertFalse(sortedChildren.isEmpty()); int childCount = 0; boolean childStarNodeAsserted = false; + boolean nodeWithMinusOneValueFound = false; + /* + Since NULL nodes have a dimension value of -1, we need to track whether we have encountered any + default nodes with this dimension value. We will perform the assertNull() check only if we have not + yet found a default node with a dimension value of -1. + */ while (expectedChildrenIterator.hasNext()) { StarTreeNode child = expectedChildrenIterator.next(); InMemoryTreeNode resultChildNode = null; @@ -220,10 +228,13 @@ public static void validateFileFormats( resultChildNode = sortedChildren.get(childCount); assertNotNull(child); assertNotNull(resultChildNode); - if (child.getStarTreeNodeType() != StarTreeNodeType.NULL.getValue()) { - assertNotNull(starTreeNode.getChildForDimensionValue(child.getDimensionValue())); - } else { + if (child.getStarTreeNodeType() == StarTreeNodeType.NULL.getValue() && !nodeWithMinusOneValueFound) { assertNull(starTreeNode.getChildForDimensionValue(child.getDimensionValue())); + } else { + if (child.getDimensionValue() == -1L) { + nodeWithMinusOneValueFound = true; + } + assertNotNull(starTreeNode.getChildForDimensionValue(child.getDimensionValue())); } assertStarTreeNode(child, resultChildNode); assertNotEquals(child.getStarTreeNodeType(), StarTreeNodeType.STAR.getValue()); @@ -271,14 +282,21 @@ public static void assertStarTreeMetadata(StarTreeMetadata expectedStarTreeMetad assertEquals(expectedStarTreeMetadata.getCompositeFieldName(), resultStarTreeMetadata.getCompositeFieldName()); assertEquals(expectedStarTreeMetadata.getCompositeFieldType(), resultStarTreeMetadata.getCompositeFieldType()); + assertEquals(expectedStarTreeMetadata.getDimensionFields().size(), resultStarTreeMetadata.getDimensionFields().size()); - for (int i = 0; i < expectedStarTreeMetadata.getDimensionFields().size(); i++) { - assertEquals(expectedStarTreeMetadata.getDimensionFields().get(i), resultStarTreeMetadata.getDimensionFields().get(i)); - } - assertEquals(expectedStarTreeMetadata.getMetrics().size(), resultStarTreeMetadata.getMetrics().size()); + expectedStarTreeMetadata.getDimensionFields().forEach((dimensionField, dimensionConfig) -> { + assertEquals( + dimensionConfig.getDocValuesType(), + resultStarTreeMetadata.getDimensionFields().get(dimensionField).getDocValuesType() + ); + assertEquals( + dimensionConfig.getDimensionDataType(), + resultStarTreeMetadata.getDimensionFields().get(dimensionField).getDimensionDataType() + ); + }); + assertEquals(expectedStarTreeMetadata.getMetrics().size(), resultStarTreeMetadata.getMetrics().size()); for (int i = 0; i < expectedStarTreeMetadata.getMetrics().size(); i++) { - Metric expectedMetric = expectedStarTreeMetadata.getMetrics().get(i); Metric resultMetric = resultStarTreeMetadata.getMetrics().get(i); assertEquals(expectedMetric.getField(), resultMetric.getField()); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BuilderTestsUtils.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BuilderTestsUtils.java index e7c3d50c9572a..06c04ce67ea05 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BuilderTestsUtils.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BuilderTestsUtils.java @@ -36,6 +36,7 @@ import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.StarTreeTestUtils; +import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.DimensionConfig; import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.StarTreeMetadata; import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; import org.opensearch.index.compositeindex.datacube.startree.node.InMemoryTreeNode; @@ -439,7 +440,7 @@ public static void validateStarTreeFileFormats( StarTreeDocument[] expectedStarTreeDocumentsArray = expectedStarTreeDocuments.toArray(new StarTreeDocument[0]); StarTreeTestUtils.assertStarTreeDocuments(starTreeDocuments, expectedStarTreeDocumentsArray); - validateFileFormats(dataIn, metaIn, rootNode, expectedStarTreeMetadata); + validateFileFormats(dataIn, metaIn, rootNode, expectedStarTreeMetadata, starTreeField); dataIn.close(); metaIn.close(); @@ -448,7 +449,7 @@ public static void validateStarTreeFileFormats( public static SegmentReadState getReadState( int numDocs, - Map dimensionFields, + Map dimensionFields, List metrics, StarTreeField compositeField, SegmentWriteState writeState, @@ -471,7 +472,7 @@ public static SegmentReadState getReadState( false, true, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, - dimensionFields.get(dimension), + dimensionFields.get(dimension).getDocValuesType(), DocValuesSkipIndexType.RANGE, -1, Collections.emptyMap(), diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuildMetricTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuildMetricTests.java index 72ee197a93e18..5690f1d9ef07e 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuildMetricTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuildMetricTests.java @@ -30,12 +30,15 @@ import org.opensearch.common.settings.Settings; import org.opensearch.index.codec.composite.LuceneDocValuesConsumerFactory; import org.opensearch.index.codec.composite.composite912.Composite912DocValuesFormat; +import org.opensearch.index.compositeindex.datacube.Dimension; +import org.opensearch.index.compositeindex.datacube.DimensionDataType; import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.NumericDimension; import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; +import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.DimensionConfig; import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.StarTreeMetadata; import org.opensearch.index.compositeindex.datacube.startree.node.InMemoryTreeNode; import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNodeType; @@ -440,6 +443,130 @@ public void test_build_longMetrics() throws IOException { ); } + public void test_build_unsigned_longMetrics() throws IOException { + + mapperService = mock(MapperService.class); + DocumentMapper documentMapper = mock(DocumentMapper.class); + when(mapperService.documentMapper()).thenReturn(documentMapper); + Settings settings = Settings.builder().put(settings(org.opensearch.Version.CURRENT).build()).build(); + NumberFieldMapper numberFieldMapper1 = new NumberFieldMapper.Builder( + "field2", + NumberFieldMapper.NumberType.UNSIGNED_LONG, + false, + true + ).build(new Mapper.BuilderContext(settings, new ContentPath())); + NumberFieldMapper numberFieldMapper2 = new NumberFieldMapper.Builder( + "field4", + NumberFieldMapper.NumberType.UNSIGNED_LONG, + false, + true + ).build(new Mapper.BuilderContext(settings, new ContentPath())); + NumberFieldMapper numberFieldMapper3 = new NumberFieldMapper.Builder( + "field6", + NumberFieldMapper.NumberType.UNSIGNED_LONG, + false, + true + ).build(new Mapper.BuilderContext(settings, new ContentPath())); + NumberFieldMapper numberFieldMapper4 = new NumberFieldMapper.Builder( + "field9", + NumberFieldMapper.NumberType.UNSIGNED_LONG, + false, + true + ).build(new Mapper.BuilderContext(settings, new ContentPath())); + NumberFieldMapper numberFieldMapper5 = new NumberFieldMapper.Builder( + "field10", + NumberFieldMapper.NumberType.UNSIGNED_LONG, + false, + true + ).build(new Mapper.BuilderContext(settings, new ContentPath())); + MappingLookup fieldMappers = new MappingLookup( + Set.of(numberFieldMapper1, numberFieldMapper2, numberFieldMapper3, numberFieldMapper4, numberFieldMapper5), + Collections.emptyList(), + Collections.emptyList(), + 0, + null + ); + when(documentMapper.mappers()).thenReturn(fieldMappers); + + int noOfStarTreeDocuments = 5; + StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + + starTreeDocuments[0] = new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Long[] { 12L, 10L, randomLong(), 8L, -1L }); + starTreeDocuments[1] = new StarTreeDocument( + new Long[] { 3L, 4L, 2L, 1L }, + new Long[] { -2L, -9223372036854775808L, randomLong(), 12L, 10L } + ); + starTreeDocuments[2] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Long[] { 14L, 12L, randomLong(), 6L, 24L }); + starTreeDocuments[3] = new StarTreeDocument( + new Long[] { 2L, 4L, 3L, 4L }, + new Long[] { 9L, 4L, randomLong(), -9223372036854775806L, 12L } + ); + starTreeDocuments[4] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Long[] { 11L, 16L, randomLong(), 8L, 13L }); + + StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + for (int i = 0; i < noOfStarTreeDocuments; i++) { + long metric1 = (Long) starTreeDocuments[i].metrics[0]; + long metric2 = (Long) starTreeDocuments[i].metrics[1]; + long metric3 = (Long) starTreeDocuments[i].metrics[2]; + long metric4 = (Long) starTreeDocuments[i].metrics[3]; + long metric5 = (Long) starTreeDocuments[i].metrics[4]; + segmentStarTreeDocuments[i] = new StarTreeDocument( + starTreeDocuments[i].dimensions, + new Long[] { metric1, metric2, metric3, metric4, metric5, null } + ); + } + + SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); + List metricsIterators = getMetricIterators(segmentStarTreeDocuments); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite912DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite912DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite912DocValuesFormat.META_DOC_VALUES_CODEC, + Composite912DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); + Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( + dimsIterators, + metricsIterators + ); + builder.build(segmentStarTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); + + List resultStarTreeDocuments = builder.getStarTreeDocuments(); + assertEquals(7, resultStarTreeDocuments.size()); + + Iterator expectedStarTreeDocumentIterator = getExpectedStarTreeDocumentIteratorForUnsignedLong().iterator(); + assertStarTreeDocuments(resultStarTreeDocuments, expectedStarTreeDocumentIterator); + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = new StarTreeMetadata( + "test", + STAR_TREE, + mock(IndexInput.class), + VERSION_CURRENT, + builder.numStarTreeNodes, + getStarTreeDimensionNames(compositeField.getDimensionsOrder()), + compositeField.getMetrics(), + 2, + getExpectedStarTreeDocumentIterator().size(), + 1, + Set.of("field8"), + getBuildMode(), + 0, + 330 + ); + + validateStarTreeFileFormats( + builder.getRootNode(), + getExpectedStarTreeDocumentIteratorForUnsignedLong().size(), + starTreeMetadata, + getExpectedStarTreeDocumentIteratorForUnsignedLong() + ); + } + public void test_build_multipleStarTrees() throws IOException { int noOfStarTreeDocuments = 5; @@ -594,11 +721,11 @@ public void test_build_multipleStarTrees() throws IOException { metaOut.close(); dataOut.close(); - LinkedHashMap fieldsMap = new LinkedHashMap<>(); - fieldsMap.put("field1", DocValuesType.SORTED_NUMERIC); - fieldsMap.put("field3", DocValuesType.SORTED_NUMERIC); - fieldsMap.put("field5", DocValuesType.SORTED_NUMERIC); - fieldsMap.put("field8", DocValuesType.SORTED_NUMERIC); + LinkedHashMap fieldsMap = new LinkedHashMap<>(); + fieldsMap.put("field1", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); + fieldsMap.put("field3", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); + fieldsMap.put("field5", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); + fieldsMap.put("field8", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); StarTreeMetadata starTreeMetadata = new StarTreeMetadata( "test", @@ -623,10 +750,10 @@ public void test_build_multipleStarTrees() throws IOException { 330 ); - LinkedHashMap fieldsMap1 = new LinkedHashMap<>(); - fieldsMap1.put("fieldC", DocValuesType.SORTED_NUMERIC); - fieldsMap1.put("fieldB", DocValuesType.SORTED_NUMERIC); - fieldsMap1.put("fieldL", DocValuesType.SORTED_NUMERIC); + LinkedHashMap fieldsMap1 = new LinkedHashMap<>(); + fieldsMap1.put("fieldC", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); + fieldsMap1.put("fieldB", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); + fieldsMap1.put("fieldL", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); StarTreeMetadata starTreeMetadata2 = new StarTreeMetadata( "test", @@ -645,7 +772,7 @@ public void test_build_multipleStarTrees() throws IOException { 1287 ); - LinkedHashMap totalDimensionFields = new LinkedHashMap<>(starTreeMetadata.getDimensionFields()); + LinkedHashMap totalDimensionFields = new LinkedHashMap<>(starTreeMetadata.getDimensionFields()); totalDimensionFields.putAll(starTreeMetadata2.getDimensionFields()); List metrics = new ArrayList<>(); @@ -656,9 +783,32 @@ public void test_build_multipleStarTrees() throws IOException { IndexInput dataIn = readState.directory.openInput(dataFileName, IOContext.DEFAULT); IndexInput metaIn = readState.directory.openInput(metaFileName, IOContext.DEFAULT); + List dimensionsOrder1 = List.of( + new NumericDimension("field1"), + new NumericDimension("field3"), + new NumericDimension("field5"), + new NumericDimension("field8") + ); + List dimensionsOrder2 = List.of( + new NumericDimension("fieldC"), + new NumericDimension("fieldB"), + new NumericDimension("fieldL") + ); + StarTreeField compositeField1 = new StarTreeField( + "test", + dimensionsOrder1, + metrics, + new StarTreeFieldConfiguration(1, Set.of(), getBuildMode()) + ); + StarTreeField compositeField2 = new StarTreeField( + "test", + dimensionsOrder2, + metrics, + new StarTreeFieldConfiguration(1, Set.of(), getBuildMode()) + ); - validateFileFormats(dataIn, metaIn, rootNode1, starTreeMetadata); - validateFileFormats(dataIn, metaIn, rootNode2, starTreeMetadata2); + validateFileFormats(dataIn, metaIn, rootNode1, starTreeMetadata, compositeField1); + validateFileFormats(dataIn, metaIn, rootNode2, starTreeMetadata2, compositeField2); dataIn.close(); metaIn.close(); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderFlushFlowTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderFlushFlowTests.java index 7ecf175b5eb09..6bc75d01bc9d3 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderFlushFlowTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderFlushFlowTests.java @@ -20,14 +20,17 @@ import org.opensearch.index.codec.composite.LuceneDocValuesConsumerFactory; import org.opensearch.index.codec.composite.composite912.Composite912DocValuesFormat; import org.opensearch.index.compositeindex.datacube.Dimension; +import org.opensearch.index.compositeindex.datacube.DimensionDataType; import org.opensearch.index.compositeindex.datacube.IpDimension; import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.NumericDimension; import org.opensearch.index.compositeindex.datacube.OrdinalDimension; +import org.opensearch.index.compositeindex.datacube.UnsignedLongDimension; import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; +import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.DimensionConfig; import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.StarTreeMetadata; import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator; import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator; @@ -132,9 +135,9 @@ public void testFlushFlow() throws IOException { metaOut.close(); dataOut.close(); docValuesConsumer.close(); - LinkedHashMap docValues = new LinkedHashMap<>(); - docValues.put("field1", DocValuesType.SORTED_NUMERIC); - docValues.put("field3", DocValuesType.SORTED_NUMERIC); + LinkedHashMap docValues = new LinkedHashMap<>(); + docValues.put("field1", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); + docValues.put("field3", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); StarTreeMetadata starTreeMetadata = new StarTreeMetadata( "sf", STAR_TREE, @@ -233,9 +236,9 @@ public void testFlushFlowDimsReverse() throws IOException { dataOut.close(); docValuesConsumer.close(); - LinkedHashMap docValues = new LinkedHashMap<>(); - docValues.put("field1", DocValuesType.SORTED_NUMERIC); - docValues.put("field3", DocValuesType.SORTED_NUMERIC); + LinkedHashMap docValues = new LinkedHashMap<>(); + docValues.put("field1", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); + docValues.put("field3", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); StarTreeMetadata starTreeMetadata = new StarTreeMetadata( "sf", STAR_TREE, @@ -261,6 +264,121 @@ public void testFlushFlowDimsReverse() throws IOException { ); } + public void testFlushFlowWithUnsignedLongDimensions() throws IOException { + List dimList = List.of(0L, -1L, 9223372036854775806L, 4987L, -9223372036854775807L); + List docsWithField = List.of(0, 1, 3, 4, 5); + List dimList2 = List.of(0L, -1L, 2L, 9223372036854775806L, 4987L, -9223372036854775807L); + List docsWithField2 = List.of(0, 1, 2, 3, 4, 5); + + List metricsList = List.of( + getLongFromDouble(0.0), + getLongFromDouble(10.0), + getLongFromDouble(20.0), + getLongFromDouble(30.0), + getLongFromDouble(40.0), + getLongFromDouble(50.0) + ); + List metricsWithField = List.of(0, 1, 2, 3, 4, 5); + + compositeField = getStarTreeFieldWithUnsignedLongField(); + SortedNumericStarTreeValuesIterator d1sndv = new SortedNumericStarTreeValuesIterator(getSortedNumericMock(dimList, docsWithField)); + SortedNumericStarTreeValuesIterator d2sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(dimList2, docsWithField2) + ); + SortedNumericStarTreeValuesIterator m1sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(metricsList, metricsWithField) + ); + SortedNumericStarTreeValuesIterator m2sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(metricsList, metricsWithField) + ); + + writeState = getWriteState(6, writeState.segmentInfo.getId()); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); + SequentialDocValuesIterator[] dimDvs = { new SequentialDocValuesIterator(d1sndv), new SequentialDocValuesIterator(d2sndv) }; + Iterator starTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( + dimDvs, + List.of(new SequentialDocValuesIterator(m1sndv), new SequentialDocValuesIterator(m2sndv)) + ); + + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite912DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite912DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite912DocValuesFormat.META_DOC_VALUES_CODEC, + Composite912DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); + List starTreeDocuments = builder.getStarTreeDocuments(); + + /* + Asserting following dim / metrics [ dim1, dim2 / Sum [metric], count [metric] ] + [0, 0] | [0.0, 1] + [4987, 4987] | [40.0, 1] + [9223372036854775806, 9223372036854775806] | [30.0, 1] + [-9223372036854775807, -9223372036854775807] | [50.0, 1] + [-1, -1] | [10.0, 1] + [null, 2] | [20.0, 1] + */ + Object[][] expectedSortedDimensions = { + { 0L, 0L }, + { 4987L, 4987L }, + { 9223372036854775806L, 9223372036854775806L }, + { -9223372036854775807L, -9223372036854775807L }, + { -1L, -1L }, + { null, 2L } }; + + double[] expectedSumMetrics = { 0.0, 40.0, 30.0, 50.0, 10.0, 20.0 }; + long expectedCountMetric = 1L; + + int count = 0; + for (StarTreeDocument starTreeDocument : starTreeDocuments) { + if (count < 6) { + assertEquals(expectedSumMetrics[count], starTreeDocument.metrics[0]); + assertEquals(expectedCountMetric, starTreeDocument.metrics[1]); + + Long dim1 = starTreeDocument.dimensions[0]; + Long dim2 = starTreeDocument.dimensions[1]; + assertEquals(expectedSortedDimensions[count][0], dim1); + assertEquals(expectedSortedDimensions[count][1], dim2); + } + count++; + } + assertEquals(13, count); + validateStarTree(builder.getRootNode(), 2, 1000, builder.getStarTreeDocuments()); + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + LinkedHashMap docValues = new LinkedHashMap<>(); + docValues.put("field1", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.UNSIGNED_LONG)); + docValues.put("field3", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.UNSIGNED_LONG)); + StarTreeMetadata starTreeMetadata = new StarTreeMetadata( + "sf", + STAR_TREE, + mock(IndexInput.class), + VERSION_CURRENT, + builder.numStarTreeNodes, + docValues, + List.of(new Metric("field2", List.of(MetricStat.SUM, MetricStat.VALUE_COUNT, MetricStat.AVG))), + 6, + builder.numStarTreeDocs, + 1000, + Set.of(), + getBuildMode(), + 0, + 264 + ); + + // validateStarTreeFileFormats( + // builder.getRootNode(), + // builder.getStarTreeDocuments().size(), + // starTreeMetadata, + // builder.getStarTreeDocuments() + // ); + + // TODO: Fix this post 2.19 [Handling search for unsigned-long as part of star-tree] + } + public void testFlushFlowBuild() throws IOException { List dimList = new ArrayList<>(100); List docsWithField = new ArrayList<>(100); @@ -337,9 +455,9 @@ public void testFlushFlowBuild() throws IOException { dataOut.close(); docValuesConsumer.close(); - LinkedHashMap map = new LinkedHashMap<>(); - map.put("field1", DocValuesType.SORTED_NUMERIC); - map.put("field3", DocValuesType.SORTED_NUMERIC); + LinkedHashMap map = new LinkedHashMap<>(); + map.put("field1", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); + map.put("field3", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); StarTreeMetadata starTreeMetadata = getStarTreeMetadata(map, 100, 1, 6699); validateStarTreeFileFormats( @@ -496,9 +614,9 @@ public void testFlushFlowForKeywords() throws IOException { metaOut.close(); dataOut.close(); docValuesConsumer.close(); - LinkedHashMap docValues = new LinkedHashMap<>(); - docValues.put("field1", DocValuesType.SORTED_SET); - docValues.put("field3", DocValuesType.SORTED_SET); + LinkedHashMap docValues = new LinkedHashMap<>(); + docValues.put("field1", new DimensionConfig(DocValuesType.SORTED_SET, DimensionDataType.LONG)); + docValues.put("field3", new DimensionConfig(DocValuesType.SORTED_SET, DimensionDataType.LONG)); StarTreeMetadata starTreeMetadata = new StarTreeMetadata( "sf", STAR_TREE, @@ -537,6 +655,18 @@ private StarTreeField getStarTreeFieldWithMultipleMetrics() { return new StarTreeField("sf", dims, metrics, c); } + private StarTreeField getStarTreeFieldWithUnsignedLongField() { + Dimension d1 = new UnsignedLongDimension("field1"); + Dimension d2 = new UnsignedLongDimension("field3"); + Metric m1 = new Metric("field2", List.of(MetricStat.SUM)); + Metric m2 = new Metric("field2", List.of(MetricStat.VALUE_COUNT)); + Metric m3 = new Metric("field2", List.of(MetricStat.AVG)); + List dims = List.of(d1, d2); + List metrics = List.of(m1, m2, m3); + StarTreeFieldConfiguration c = new StarTreeFieldConfiguration(1000, new HashSet<>(), getBuildMode()); + return new StarTreeField("sf", dims, metrics, c); + } + private StarTreeField getStarTreeFieldWithKeywordField(boolean isIp) { Dimension d1 = isIp ? new IpDimension("field1") : new OrdinalDimension("field1"); Dimension d2 = isIp ? new IpDimension("field3") : new OrdinalDimension("field3"); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderMergeFlowTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderMergeFlowTests.java index 74ecff04076b1..3ed2881b72490 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderMergeFlowTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderMergeFlowTests.java @@ -19,12 +19,15 @@ import org.opensearch.index.codec.composite.composite912.Composite912DocValuesFormat; import org.opensearch.index.compositeindex.CompositeIndexConstants; import org.opensearch.index.compositeindex.datacube.Dimension; +import org.opensearch.index.compositeindex.datacube.DimensionDataType; import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.NumericDimension; +import org.opensearch.index.compositeindex.datacube.ReadDimension; import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; +import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.DimensionConfig; import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.StarTreeMetadata; import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator; @@ -94,13 +97,6 @@ public void testMergeFlow() throws IOException { docsWithField4.add(i); } - List dimList5 = new ArrayList<>(1000); - List docsWithField5 = new ArrayList<>(1000); - for (int i = 0; i < 1000; i++) { - dimList5.add((long) i); - docsWithField5.add(i); - } - List metricsList = new ArrayList<>(1000); List metricsWithField = new ArrayList<>(1000); for (int i = 0; i < 1000; i++) { @@ -119,7 +115,6 @@ public void testMergeFlow() throws IOException { Dimension d2 = new NumericDimension("field3"); Dimension d3 = new NumericDimension("field5"); Dimension d4 = new NumericDimension("field8"); - // Dimension d5 = new NumericDimension("field5"); Metric m1 = new Metric("field2", List.of(MetricStat.SUM, MetricStat.AVG, MetricStat.VALUE_COUNT)); Metric m2 = new Metric("_doc_count", List.of(MetricStat.DOC_COUNT)); List dims = List.of(d1, d2, d3, d4); @@ -216,12 +211,16 @@ public void testMergeFlow() throws IOException { ... [999, 999, 999, 999] | [19980.0] */ + builder.appendDocumentsToStarTree(starTreeDocumentIterator); + builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); + int count = 0; for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { - assertEquals(starTreeDocument.dimensions[0] * 20.0, starTreeDocument.metrics[0]); - assertEquals(2L, starTreeDocument.metrics[1]); + if (count < 1000) { + assertEquals(starTreeDocument.dimensions[0] * 20.0, starTreeDocument.metrics[0]); + assertEquals(2L, starTreeDocument.metrics[2]); + } + count++; } - builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); - // Validate the star tree structure validateStarTree(builder.getRootNode(), 4, 1, builder.getStarTreeDocuments()); @@ -244,6 +243,195 @@ public void testMergeFlow() throws IOException { ); } + public void testMergeFlowForUnsignedLong() throws IOException { + int numDocs = 1000; + List dimList1 = new ArrayList<>(numDocs); + List docsWithField1 = new ArrayList<>(numDocs); + for (int i = 0; i < numDocs; i++) { + dimList1.add((long) (i % 2 == 0 ? i : -i)); + docsWithField1.add(i); + } + + List dimList2 = new ArrayList<>(numDocs); + List docsWithField2 = new ArrayList<>(numDocs); + for (int i = 0; i < numDocs; i++) { + dimList2.add((long) (i % 2 == 0 ? i : -i)); + docsWithField2.add(i); + } + + List dimList3 = new ArrayList<>(numDocs); + List docsWithField3 = new ArrayList<>(numDocs); + for (int i = 0; i < numDocs; i++) { + dimList3.add((long) (i % 2 == 0 ? i : -i)); + docsWithField3.add(i); + } + + List dimList4 = new ArrayList<>(numDocs); + List docsWithField4 = new ArrayList<>(numDocs); + for (int i = 0; i < numDocs; i++) { + dimList4.add((long) (i % 2 == 0 ? i : -i)); + docsWithField4.add(i); + } + + List metricsList = new ArrayList<>(1000); + List metricsWithField = new ArrayList<>(1000); + for (int i = 0; i < 1000; i++) { + metricsList.add(getLongFromDouble(i * 10.0)); + metricsWithField.add(i); + } + + List metricsListValueCount = new ArrayList<>(1000); + List metricsWithFieldValueCount = new ArrayList<>(1000); + for (int i = 0; i < 1000; i++) { + metricsListValueCount.add((long) i); + metricsWithFieldValueCount.add(i); + } + + Dimension d1 = new ReadDimension("field1", DocValuesType.SORTED_NUMERIC, DimensionDataType.UNSIGNED_LONG); + Dimension d2 = new ReadDimension("field3", DocValuesType.SORTED_NUMERIC, DimensionDataType.UNSIGNED_LONG); + Dimension d3 = new ReadDimension("field5", DocValuesType.SORTED_NUMERIC, DimensionDataType.UNSIGNED_LONG); + Dimension d4 = new ReadDimension("field8", DocValuesType.SORTED_NUMERIC, DimensionDataType.UNSIGNED_LONG); + + Metric m1 = new Metric("field2", List.of(MetricStat.SUM, MetricStat.AVG, MetricStat.VALUE_COUNT)); + Metric m2 = new Metric("_doc_count", List.of(MetricStat.DOC_COUNT)); + + List dims = List.of(d1, d2, d3, d4); + List metrics = List.of(m1, m2); + StarTreeFieldConfiguration c = new StarTreeFieldConfiguration(1, new HashSet<>(), getBuildMode()); + compositeField = new StarTreeField("sf", dims, metrics, c); + SortedNumericDocValues d1sndv = getSortedNumericMock(dimList1, docsWithField1); + SortedNumericDocValues d2sndv = getSortedNumericMock(dimList2, docsWithField2); + SortedNumericDocValues d3sndv = getSortedNumericMock(dimList3, docsWithField3); + SortedNumericDocValues d4sndv = getSortedNumericMock(dimList4, docsWithField4); + SortedNumericDocValues m1sndv = getSortedNumericMock(metricsList, metricsWithField); + SortedNumericDocValues valucountsndv = getSortedNumericMock(metricsListValueCount, metricsWithFieldValueCount); + SortedNumericDocValues m2sndv = DocValues.emptySortedNumeric(); + Map> dimDocIdSetIterators = Map.of( + "field1", + () -> new SortedNumericStarTreeValuesIterator(d1sndv), + "field3", + () -> new SortedNumericStarTreeValuesIterator(d2sndv), + "field5", + () -> new SortedNumericStarTreeValuesIterator(d3sndv), + "field8", + () -> new SortedNumericStarTreeValuesIterator(d4sndv) + ); + + Map> metricDocIdSetIterators = Map.of( + "sf_field2_sum_metric", + () -> new SortedNumericStarTreeValuesIterator(m1sndv), + "sf_field2_value_count_metric", + () -> new SortedNumericStarTreeValuesIterator(valucountsndv), + "sf__doc_count_doc_count_metric", + () -> new SortedNumericStarTreeValuesIterator(m2sndv) + ); + + StarTreeValues starTreeValues = new StarTreeValues( + compositeField, + null, + dimDocIdSetIterators, + metricDocIdSetIterators, + getAttributes(1000), + null + ); + + SortedNumericDocValues f2d1sndv = getSortedNumericMock(dimList1, docsWithField1); + SortedNumericDocValues f2d2sndv = getSortedNumericMock(dimList2, docsWithField2); + SortedNumericDocValues f2d3sndv = getSortedNumericMock(dimList3, docsWithField3); + SortedNumericDocValues f2d4sndv = getSortedNumericMock(dimList4, docsWithField4); + SortedNumericDocValues f2m1sndv = getSortedNumericMock(metricsList, metricsWithField); + SortedNumericDocValues f2valucountsndv = getSortedNumericMock(metricsListValueCount, metricsWithFieldValueCount); + SortedNumericDocValues f2m2sndv = DocValues.emptySortedNumeric(); + Map> f2dimDocIdSetIterators = Map.of( + "field1", + () -> new SortedNumericStarTreeValuesIterator(f2d1sndv), + "field3", + () -> new SortedNumericStarTreeValuesIterator(f2d2sndv), + "field5", + () -> new SortedNumericStarTreeValuesIterator(f2d3sndv), + "field8", + () -> new SortedNumericStarTreeValuesIterator(f2d4sndv) + ); + + Map> f2metricDocIdSetIterators = Map.of( + "sf_field2_sum_metric", + () -> new SortedNumericStarTreeValuesIterator(f2m1sndv), + "sf_field2_value_count_metric", + () -> new SortedNumericStarTreeValuesIterator(f2valucountsndv), + "sf__doc_count_doc_count_metric", + () -> new SortedNumericStarTreeValuesIterator(f2m2sndv) + ); + StarTreeValues starTreeValues2 = new StarTreeValues( + compositeField, + null, + f2dimDocIdSetIterators, + f2metricDocIdSetIterators, + getAttributes(1000), + null + ); + + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite912DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite912DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite912DocValuesFormat.META_DOC_VALUES_CODEC, + Composite912DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); + Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); + /** + [0, 0, 0, 0] | [0.0, 2] + [-1, -1, -1, -1] | [20.0, 2] + [2, 2, 2, 2] | [40.0, 2] + [-3, -3, -3, -3] | [60.0, 2] + [4, 4, 4, 4] | [80.0, 2] + [-5, -5, -5, -5] | [100.0, 2] + ... + */ + builder.appendDocumentsToStarTree(starTreeDocumentIterator); + builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); + int count = 0; + List actualDimensionValues = new ArrayList<>(numDocs); + for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { + if (count < 1000) { + actualDimensionValues.add(starTreeDocument.dimensions[0]); + } + count++; + } + + List expectedDimensionValues = new ArrayList<>(1000); + for (int i = 0; i < numDocs; i++) { + if (i <= 499) { + expectedDimensionValues.add((long) i * 2); // Positive even numbers + } else { + expectedDimensionValues.add((long) -(numDocs - i) * 2 + 1); // Negative odd numbers in decreasing order + } + } + assertEquals(expectedDimensionValues, actualDimensionValues); + + // Validate the star tree structure + validateStarTree(builder.getRootNode(), 4, 1, builder.getStarTreeDocuments()); + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = getStarTreeMetadata( + getStarTreeDimensionNames(compositeField.getDimensionsOrder()), + 1000, + compositeField.getStarTreeConfig().maxLeafDocs(), + 132165 + ); + + // validateStarTreeFileFormats( + // builder.getRootNode(), + // builder.getStarTreeDocuments().size(), + // starTreeMetadata, + // builder.getStarTreeDocuments() + // ); + // TODO: Fix this post 2.19 [Handling search for unsigned-long as part of star-tree] + } + public void testMergeFlow_randomNumberTypes() throws Exception { DocumentMapper documentMapper = mock(DocumentMapper.class); @@ -352,9 +540,95 @@ public void testMergeFlowWithSum() throws IOException { metaOut.close(); dataOut.close(); docValuesConsumer.close(); - LinkedHashMap map = new LinkedHashMap<>(); - map.put("field1", DocValuesType.SORTED_NUMERIC); - map.put("field3", DocValuesType.SORTED_NUMERIC); + LinkedHashMap map = new LinkedHashMap<>(); + map.put("field1", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); + map.put("field3", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); + StarTreeMetadata starTreeMetadata = getStarTreeMetadata(map, 6, 1000, 264); + + validateStarTreeFileFormats( + builder.getRootNode(), + builder.getStarTreeDocuments().size(), + starTreeMetadata, + builder.getStarTreeDocuments() + ); + } + + public void testMergeFlowForUnsignedLongWithSum() throws IOException { + List dimList = List.of(0L, 1L, 3L, 4L, 5L, 6L); + List docsWithField = List.of(0, 1, 3, 4, 5, 6); + List dimList2 = List.of(0L, 1L, 2L, 3L, 4L, 5L, -1L); + List docsWithField2 = List.of(0, 1, 2, 3, 4, 5, 6); + + List metricsList = List.of( + getLongFromDouble(0.0), + getLongFromDouble(10.0), + getLongFromDouble(20.0), + getLongFromDouble(30.0), + getLongFromDouble(40.0), + getLongFromDouble(50.0), + getLongFromDouble(60.0) + ); + List metricsWithField = List.of(0, 1, 2, 3, 4, 5, 6); + + compositeField = getStarTreeFieldForUnsignedLong(MetricStat.SUM); + StarTreeValues starTreeValues = getStarTreeValues( + getSortedNumericMock(dimList, docsWithField), + getSortedNumericMock(dimList2, docsWithField2), + getSortedNumericMock(metricsList, metricsWithField), + compositeField, + "6" + ); + + StarTreeValues starTreeValues2 = getStarTreeValues( + getSortedNumericMock(dimList, docsWithField), + getSortedNumericMock(dimList2, docsWithField2), + getSortedNumericMock(metricsList, metricsWithField), + compositeField, + "6" + ); + writeState = getWriteState(6, writeState.segmentInfo.getId()); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite912DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite912DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite912DocValuesFormat.META_DOC_VALUES_CODEC, + Composite912DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); + Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); + /** + * Asserting following dim / metrics [ dim1, dim2 / Sum [ metric] ] + * [0, 0] | [0.0] + * [1, 1] | [20.0] + * [3, 3] | [60.0] + * [4, 4] | [80.0] + * [5, 5] | [100.0] + * [null, 2] | [40.0] + * ------------------ We only take non-star docs + * [6,-1] | [120.0] + */ + builder.appendDocumentsToStarTree(starTreeDocumentIterator); + assertEquals(6, builder.getStarTreeDocuments().size()); + builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); + int count = 0; + for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { + count++; + if (count <= 6) { + assertEquals( + starTreeDocument.dimensions[0] != null ? starTreeDocument.dimensions[0] * 2 * 10.0 : 40.0, + starTreeDocument.metrics[0] + ); + } + } + + validateStarTree(builder.getRootNode(), 2, 1000, builder.getStarTreeDocuments()); + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + LinkedHashMap map = new LinkedHashMap<>(); + map.put("field1", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.UNSIGNED_LONG)); + map.put("field3", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.UNSIGNED_LONG)); StarTreeMetadata starTreeMetadata = getStarTreeMetadata(map, 6, 1000, 264); validateStarTreeFileFormats( @@ -427,9 +701,9 @@ public void testMergeFlowWithCount() throws IOException { metaOut.close(); dataOut.close(); docValuesConsumer.close(); - LinkedHashMap map = new LinkedHashMap<>(); - map.put("field1", DocValuesType.SORTED_NUMERIC); - map.put("field3", DocValuesType.SORTED_NUMERIC); + LinkedHashMap map = new LinkedHashMap<>(); + map.put("field1", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); + map.put("field3", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); StarTreeMetadata starTreeMetadata = getStarTreeMetadata(map, 6, 1000, 264); validateStarTreeFileFormats( @@ -576,9 +850,9 @@ public void testMergeFlowWithMissingDocs() throws IOException { metaOut.close(); dataOut.close(); docValuesConsumer.close(); - LinkedHashMap map = new LinkedHashMap<>(); - map.put("field1", DocValuesType.SORTED_NUMERIC); - map.put("field3", DocValuesType.SORTED_NUMERIC); + LinkedHashMap map = new LinkedHashMap<>(); + map.put("field1", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); + map.put("field3", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); StarTreeMetadata starTreeMetadata = getStarTreeMetadata(map, 10, 1000, 363); validateStarTreeFileFormats( @@ -666,9 +940,9 @@ public void testMergeFlowWithMissingDocsWithZero() throws IOException { metaOut.close(); dataOut.close(); docValuesConsumer.close(); - LinkedHashMap map = new LinkedHashMap<>(); - map.put("field1", DocValuesType.SORTED_NUMERIC); - map.put("field3", DocValuesType.SORTED_NUMERIC); + LinkedHashMap map = new LinkedHashMap<>(); + map.put("field1", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); + map.put("field3", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); StarTreeMetadata starTreeMetadata = getStarTreeMetadata(map, 6, 1000, 231); validateStarTreeFileFormats( @@ -759,9 +1033,9 @@ public void testMergeFlowWithMissingDocsWithZeroComplexCase() throws IOException metaOut.close(); dataOut.close(); docValuesConsumer.close(); - LinkedHashMap map = new LinkedHashMap<>(); - map.put("field1", DocValuesType.SORTED_NUMERIC); - map.put("field3", DocValuesType.SORTED_NUMERIC); + LinkedHashMap map = new LinkedHashMap<>(); + map.put("field1", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); + map.put("field3", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); StarTreeMetadata starTreeMetadata = getStarTreeMetadata(map, 7, 1000, 231); validateStarTreeFileFormats( @@ -848,9 +1122,9 @@ public void testMergeFlowWithMissingDocsInSecondDim() throws IOException { metaOut.close(); dataOut.close(); docValuesConsumer.close(); - LinkedHashMap map = new LinkedHashMap<>(); - map.put("field1", DocValuesType.SORTED_NUMERIC); - map.put("field3", DocValuesType.SORTED_NUMERIC); + LinkedHashMap map = new LinkedHashMap<>(); + map.put("field1", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); + map.put("field3", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); StarTreeMetadata starTreeMetadata = getStarTreeMetadata(map, 10, 1000, 363); validateStarTreeFileFormats( @@ -935,9 +1209,9 @@ public void testMergeFlowWithDocsMissingAtTheEnd() throws IOException { metaOut.close(); dataOut.close(); docValuesConsumer.close(); - LinkedHashMap map = new LinkedHashMap<>(); - map.put("field1", DocValuesType.SORTED_NUMERIC); - map.put("field3", DocValuesType.SORTED_NUMERIC); + LinkedHashMap map = new LinkedHashMap<>(); + map.put("field1", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); + map.put("field3", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); StarTreeMetadata starTreeMetadata = getStarTreeMetadata(map, 10, 1000, 363); validateStarTreeFileFormats( @@ -1010,9 +1284,9 @@ public void testMergeFlowWithEmptyFieldsInOneSegment() throws IOException { metaOut.close(); dataOut.close(); docValuesConsumer.close(); - LinkedHashMap map = new LinkedHashMap<>(); - map.put("field1", DocValuesType.SORTED_NUMERIC); - map.put("field3", DocValuesType.SORTED_NUMERIC); + LinkedHashMap map = new LinkedHashMap<>(); + map.put("field1", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); + map.put("field3", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); StarTreeMetadata starTreeMetadata = getStarTreeMetadata(map, 6, 1000, 264); validateStarTreeFileFormats( @@ -1411,9 +1685,9 @@ public void testMergeFlowWithDifferentDocsFromSegments() throws IOException { metaOut.close(); dataOut.close(); docValuesConsumer.close(); - LinkedHashMap map = new LinkedHashMap<>(); - map.put("field1", DocValuesType.SORTED_NUMERIC); - map.put("field3", DocValuesType.SORTED_NUMERIC); + LinkedHashMap map = new LinkedHashMap<>(); + map.put("field1", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); + map.put("field3", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); StarTreeMetadata starTreeMetadata = getStarTreeMetadata(map, 9, 1000, 330); validateStarTreeFileFormats( diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderTestCase.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderTestCase.java index 843a79d0877ce..7dd4f59312c82 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderTestCase.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderTestCase.java @@ -26,6 +26,7 @@ import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.NumericUtils; +import org.opensearch.common.Numbers; import org.opensearch.common.Rounding; import org.opensearch.common.settings.Settings; import org.opensearch.index.codec.composite.composite912.Composite912DocValuesFormat; @@ -38,9 +39,11 @@ import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.NumericDimension; import org.opensearch.index.compositeindex.datacube.OrdinalDimension; +import org.opensearch.index.compositeindex.datacube.UnsignedLongDimension; import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; +import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.DimensionConfig; import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.StarTreeMetadata; import org.opensearch.index.compositeindex.datacube.startree.node.InMemoryTreeNode; import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitAdapter; @@ -249,7 +252,7 @@ SegmentWriteState getWriteState(int numDocs, byte[] id) { return BuilderTestsUtils.getWriteState(numDocs, id, fieldsInfo, directory); } - SegmentReadState getReadState(int numDocs, Map dimensionFields, List metrics) { + SegmentReadState getReadState(int numDocs, Map dimensionFields, List metrics) { return BuilderTestsUtils.getReadState(numDocs, dimensionFields, metrics, compositeField, writeState, directory); } @@ -257,11 +260,11 @@ protected Map getAttributes(int numSegmentDocs) { return Map.of(CompositeIndexConstants.SEGMENT_DOCS_COUNT, String.valueOf(numSegmentDocs)); } - protected LinkedHashMap getStarTreeDimensionNames(List dimensionsOrder) { - LinkedHashMap dimensionNames = new LinkedHashMap<>(); + protected LinkedHashMap getStarTreeDimensionNames(List dimensionsOrder) { + LinkedHashMap dimensionNames = new LinkedHashMap<>(); for (Dimension dimension : dimensionsOrder) { for (String dimensionName : dimension.getSubDimensionNames()) { - dimensionNames.put(dimensionName, dimension.getDocValuesType()); + dimensionNames.put(dimensionName, new DimensionConfig(dimension.getDocValuesType(), dimension.getDimensionDataType())); } } return dimensionNames; @@ -277,6 +280,16 @@ protected StarTreeField getStarTreeField(MetricStat count) { return new StarTreeField("sf", dims, metrics, c); } + protected StarTreeField getStarTreeFieldForUnsignedLong(MetricStat count) { + Dimension d1 = new UnsignedLongDimension("field1"); + Dimension d2 = new UnsignedLongDimension("field3"); + Metric m1 = new Metric("field2", List.of(count)); + List dims = List.of(d1, d2); + List metrics = List.of(m1); + StarTreeFieldConfiguration c = new StarTreeFieldConfiguration(1000, new HashSet<>(), getBuildMode()); + return new StarTreeField("sf", dims, metrics, c); + } + protected StarTreeField getStarTreeFieldWithDocCount(int maxLeafDocs, boolean includeDocCountMetric) { Dimension d1 = new NumericDimension("field1"); Dimension d2 = new NumericDimension("field3"); @@ -327,12 +340,66 @@ protected static List getExpectedStarTreeDocumentIterator() { ); } + protected static List getExpectedStarTreeDocumentIteratorForUnsignedLong() { + return List.of( + new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Object[] { 21.0, 14.0, 2L, 8.0, Numbers.unsignedLongToDouble(-1), 2L }), + new StarTreeDocument( + new Long[] { 3L, 4L, 2L, 1L }, + new Object[] { + 20.0 + Numbers.unsignedLongToDouble(-2L), + 28.0 + Numbers.unsignedLongToDouble(-9223372036854775808L), + 3L, + 6.0, + 24.0, + 3L } + ), + new StarTreeDocument( + new Long[] { null, 4L, 2L, 1L }, + new Object[] { + 20.0 + Numbers.unsignedLongToDouble(-2L), + 28.0 + Numbers.unsignedLongToDouble(-9223372036854775808L), + 3L, + 6.0, + 24.0, + 3L } + ), + new StarTreeDocument( + new Long[] { null, 4L, 3L, 4L }, + new Object[] { 21.0, 14.0, 2L, 8.0, Numbers.unsignedLongToDouble(-1), 2L } + ), + new StarTreeDocument( + new Long[] { null, 4L, null, 1L }, + new Object[] { + 20.0 + Numbers.unsignedLongToDouble(-2L), + 28.0 + Numbers.unsignedLongToDouble(-9223372036854775808L), + 3L, + 6.0, + 24.0, + 3L } + ), + new StarTreeDocument( + new Long[] { null, 4L, null, 4L }, + new Object[] { 21.0, 14.0, 2L, 8.0, Numbers.unsignedLongToDouble(-1), 2L } + ), + new StarTreeDocument( + new Long[] { null, 4L, null, null }, + new Object[] { + 46.0 + Numbers.unsignedLongToDouble(-2L), + 42.0 + Numbers.unsignedLongToDouble(-9223372036854775808L), + 5L, + 6.0, + Numbers.unsignedLongToDouble(-1), + 5L } + ) + ); + } + protected long getLongFromDouble(double value) { return NumericUtils.doubleToSortableLong(value); } protected StarTreeMetadata getStarTreeMetadata( - LinkedHashMap fields, + LinkedHashMap fields, int segmentAggregatedDocCount, int maxLeafDocs, int dataLength diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataTests.java index 51dee33662290..5622864c53ada 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataTests.java @@ -50,6 +50,7 @@ import static org.opensearch.index.compositeindex.CompositeIndexConstants.COMPOSITE_FIELD_MARKER; import static org.opensearch.index.compositeindex.datacube.startree.fileformats.StarTreeWriter.VERSION_CURRENT; import static org.opensearch.index.mapper.CompositeMappedFieldType.CompositeFieldType.STAR_TREE; +import static org.junit.Assert.assertEquals; public class StarTreeMetadataTests extends OpenSearchTestCase { @@ -184,15 +185,16 @@ public void test_starTreeMetadata() throws IOException { assertEquals(starTreeMetadata.getNumberOfNodes(), numberOfNodes); assertNotNull(starTreeMetadata); - assertEquals(dimensionsOrder.size(), starTreeMetadata.dimensionFieldsToDocValuesMap.size()); - int k = 0; - for (Map.Entry entry : starTreeMetadata.dimensionFieldsToDocValuesMap.entrySet()) { - assertEquals(dimensionsOrder.get(k).getField(), entry.getKey()); - k++; + assertEquals(dimensionsOrder.size(), starTreeMetadata.getDimensionFields().size()); + int index = 0; + for (Map.Entry entry : starTreeMetadata.getDimensionFields().entrySet()) { + Dimension dimension = dimensionsOrder.get(index++); + assertEquals(dimension.getField(), entry.getKey()); + assertEquals(dimension.getDocValuesType(), entry.getValue().getDocValuesType()); + assertEquals(dimension.getDimensionDataType(), entry.getValue().getDimensionDataType()); } assertEquals(starTreeField.getMetrics().size(), starTreeMetadata.getMetrics().size()); - for (int i = 0; i < starTreeField.getMetrics().size(); i++) { Metric expectedMetric = starTreeField.getMetrics().get(i); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeDocumentsSorterTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeDocumentsSorterTests.java index b485ea1a4fe3e..ce1d36024524b 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeDocumentsSorterTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeDocumentsSorterTests.java @@ -9,52 +9,140 @@ package org.opensearch.index.compositeindex.datacube.startree.utils; import org.opensearch.common.Randomness; +import org.opensearch.common.Rounding; +import org.opensearch.index.compositeindex.datacube.DateDimension; +import org.opensearch.index.compositeindex.datacube.NumericDimension; +import org.opensearch.index.compositeindex.datacube.UnsignedLongDimension; +import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitAdapter; +import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitRounding; +import org.opensearch.index.mapper.DateFieldMapper; import org.opensearch.test.OpenSearchTestCase; import org.junit.Before; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Random; +import java.util.concurrent.atomic.AtomicInteger; /** * Tests for {@link StarTreeDocumentsSorter}. */ public class StarTreeDocumentsSorterTests extends OpenSearchTestCase { + private Map testData; + private List> comparatorList; @Before public void setUp() throws Exception { super.setUp(); testData = new HashMap<>(); - testData.put(0, new Long[] { -1L, 2L, 3L }); - testData.put(1, new Long[] { 1L, 2L, 2L }); - testData.put(2, new Long[] { -1L, -1L, 3L }); - testData.put(3, new Long[] { 1L, 2L, null }); - testData.put(4, new Long[] { 1L, null, 3L }); + comparatorList = new ArrayList<>(); + + List intervals = Arrays.asList( + new DateTimeUnitAdapter(Rounding.DateTimeUnit.YEAR_OF_CENTURY), + new DateTimeUnitAdapter(Rounding.DateTimeUnit.MONTH_OF_YEAR), + new DateTimeUnitAdapter(Rounding.DateTimeUnit.DAY_OF_MONTH), + new DateTimeUnitAdapter(Rounding.DateTimeUnit.HOUR_OF_DAY) + ); + DateDimension dateDimension = new DateDimension("timestamp", intervals, DateFieldMapper.Resolution.MILLISECONDS); + Long[] date_dims = new Long[4]; + Long testValue = 1609459200000L; // 2021-01-01 00:00:00 UTC + AtomicInteger dimIndex = new AtomicInteger(0); + dateDimension.setDimensionValues(testValue, value -> { date_dims[dimIndex.getAndIncrement()] = value; }); + + // 10 documents with 6 dimensions each + testData.put(0, new Long[] { date_dims[0], date_dims[1], date_dims[2], date_dims[3], null, 150L, 100L, 300L, null }); + testData.put(1, new Long[] { date_dims[0], date_dims[1], date_dims[2], date_dims[3], 1L, null, -9223372036854775807L, 200L, 300L }); + testData.put(2, new Long[] { date_dims[0], date_dims[1], date_dims[2], date_dims[3], 2L, -100L, -15L, 250L, null }); + testData.put( + 3, + new Long[] { date_dims[0], date_dims[1], date_dims[2], date_dims[3], 2L, -100L, -10L, 210L, -9223372036854775807L } + ); + testData.put(4, new Long[] { date_dims[0], date_dims[1], date_dims[2], date_dims[3], 1L, 120L, null, null, 305L }); + testData.put(5, new Long[] { date_dims[0], date_dims[1], date_dims[2], date_dims[3], 2L, 150L, -5L, 200L, 295L }); + testData.put(6, new Long[] { date_dims[0], date_dims[1], date_dims[2], date_dims[3], 3L, 105L, null, -200L, -315L }); + testData.put(7, new Long[] { date_dims[0], date_dims[1], date_dims[2], date_dims[3], 1L, 120L, -10L, 205L, 310L }); + testData.put( + 8, + new Long[] { date_dims[0], date_dims[1], date_dims[2], date_dims[3], null, -100L, 9223372036854775807L, 200L, -300L } + ); + testData.put(9, new Long[] { date_dims[0], date_dims[1], date_dims[2], date_dims[3], 2L, null, -10L, 210L, 325L }); + + comparatorList.addAll(Collections.nCopies(4, dateDimension.comparator())); + comparatorList.add(new NumericDimension("dim1").comparator()); + comparatorList.add(new UnsignedLongDimension("dim2").comparator()); + comparatorList.add(new NumericDimension("dim3").comparator()); + comparatorList.add(new UnsignedLongDimension("dim4").comparator()); + comparatorList.add(new NumericDimension("dim5").comparator()); + } - public void testSortDocumentsOffHeap_FirstDimension() { - int[] sortedDocIds = { 0, 1, 2, 3, 4 }; + public void testSortDocumentsOffHeap_StartFromFirstDimension() { + int[] sortedDocIds = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; int dimensionId = -1; - int numDocs = 5; + int numDocs = 10; - StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i])); + StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i]), comparatorList); + assertArrayEquals(new int[] { 7, 4, 1, 5, 2, 3, 9, 6, 0, 8 }, sortedDocIds); + } - assertArrayEquals(new int[] { 2, 0, 1, 3, 4 }, sortedDocIds); + public void testSortDocumentsOffHeap_StartFromSecondDimension() { + int[] sortedDocIds = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + int dimensionId = 0; + int numDocs = 10; + + StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i]), comparatorList); + assertArrayEquals(new int[] { 7, 4, 1, 5, 2, 3, 9, 6, 0, 8 }, sortedDocIds); } - public void testSortDocumentsOffHeap_ThirdDimension() { - int[] sortedDocIds = { 0, 1, 2, 3, 4 }; + public void testSortDocumentsOffHeap_StartFromThirdDimension() { + int[] sortedDocIds = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; int dimensionId = 1; - int numDocs = 5; + int numDocs = 10; + + StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i]), comparatorList); + assertArrayEquals(new int[] { 7, 4, 1, 5, 2, 3, 9, 6, 0, 8 }, sortedDocIds); + } + + public void testSortDocumentsOffHeap_StartFromFourthDimension() { + int[] sortedDocIds = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + int dimensionId = 2; + int numDocs = 10; + + StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i]), comparatorList); + assertArrayEquals(new int[] { 7, 4, 1, 5, 2, 3, 9, 6, 0, 8 }, sortedDocIds); + } + + public void testSortDocumentsOffHeap_StartFromFifthDimension() { + int[] sortedDocIds = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + int dimensionId = 3; + int numDocs = 10; + + StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i]), comparatorList); + assertArrayEquals(new int[] { 7, 4, 1, 5, 2, 3, 9, 6, 0, 8 }, sortedDocIds); + } + + public void testSortDocumentsOffHeap_StartFromSixthDimension() { + int[] sortedDocIds = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + int dimensionId = 4; + int numDocs = 10; - StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i])); + StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i]), comparatorList); + assertArrayEquals(new int[] { 6, 7, 4, 5, 0, 2, 3, 8, 1, 9 }, sortedDocIds); + } + + public void testSortDocumentsOffHeap_StartFromSeventhDimension() { + int[] sortedDocIds = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + int dimensionId = 5; + int numDocs = 10; - assertArrayEquals(new int[] { 1, 0, 2, 4, 3 }, sortedDocIds); + StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i]), comparatorList); + assertArrayEquals(new int[] { 1, 2, 7, 3, 9, 5, 0, 8, 6, 4 }, sortedDocIds); } public void testSortDocumentsOffHeap_SingleElement() { @@ -62,8 +150,7 @@ public void testSortDocumentsOffHeap_SingleElement() { int dimensionId = -1; int numDocs = 1; - StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i])); - + StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i]), comparatorList); assertArrayEquals(new int[] { 0 }, sortedDocIds); } @@ -72,32 +159,21 @@ public void testSortDocumentsOffHeap_EmptyArray() { int dimensionId = -1; int numDocs = 0; - StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i])); - + StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i]), comparatorList); assertArrayEquals(new int[] {}, sortedDocIds); } - public void testSortDocumentsOffHeap_SecondDimensionId() { - int[] sortedDocIds = { 0, 1, 2, 3, 4 }; - int dimensionId = 0; - int numDocs = 5; - - StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i])); - - assertArrayEquals(new int[] { 2, 1, 0, 3, 4 }, sortedDocIds); - } - public void testSortDocumentsOffHeap_AllNulls() { Map testData = new HashMap<>(); - testData.put(0, new Long[] { null, null, null }); - testData.put(1, new Long[] { null, null, null }); - testData.put(2, new Long[] { null, null, null }); + testData.put(0, new Long[] { null, null, null, null, null, null, null, null, null }); + testData.put(1, new Long[] { null, null, null, null, null, null, null, null, null }); + testData.put(2, new Long[] { null, null, null, null, null, null, null, null, null }); int[] sortedDocIds = { 0, 1, 2 }; int dimensionId = -1; int numDocs = 3; - StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i])); + StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i]), comparatorList); // The order should remain unchanged as all elements are equal (null) assertArrayEquals(new int[] { 0, 1, 2 }, sortedDocIds); @@ -105,23 +181,21 @@ public void testSortDocumentsOffHeap_AllNulls() { public void testSortDocumentsOffHeap_Negatives() { Map testData = new HashMap<>(); - testData.put(0, new Long[] { -10L, 0L }); - testData.put(1, new Long[] { -9L, 0L }); - testData.put(2, new Long[] { -8L, 0L }); - testData.put(3, new Long[] { -7L, -0L }); - testData.put(4, new Long[] { -15L, -0L }); + testData.put(0, new Long[] { -1L, -2L, -3L, -4L, -10L, 0L, null, 0L, -5L }); + testData.put(1, new Long[] { -5L, -2L, -3L, -4L, -9L, 0L, null, 0L, -10L }); + testData.put(2, new Long[] { -5L, -3L, -3L, -4L, -9L, 0L, null, 0L, 15L }); + testData.put(3, new Long[] { -9L, -2L, -3L, -4L, -7L, 0L, null, 0L, -20L }); + testData.put(4, new Long[] { -8L, -2L, -3L, -4L, -15L, 0L, null, 0L, -25L }); int[] sortedDocIds = { 0, 1, 2, 3, 4 }; int dimensionId = -1; int numDocs = 5; - StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i])); - - // The order should remain unchanged as all elements are equal (null) - assertArrayEquals(new int[] { 4, 0, 1, 2, 3 }, sortedDocIds); + StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i]), comparatorList); + assertArrayEquals(new int[] { 3, 4, 2, 1, 0 }, sortedDocIds); } - public void testRandomSort() { + public void testTheRandomSort() { int i = 0; while (i < 10) { testRandomizedSort(); @@ -156,9 +230,16 @@ private void testRandomizedSort() { // sort dimensionId + 1 to numDimensions // for example to start from dimension in 0th index, we need to pass -1 to sort method int dimensionId = random.nextInt(numDimensions) - 1; + List> comparatorList = new ArrayList<>(); + + for (int i = 0; i < numDimensions; i++) { + Boolean isUnsignedLong = random.nextBoolean(); + if (!isUnsignedLong) comparatorList.add(new NumericDimension("fieldName").comparator()); + else comparatorList.add(new UnsignedLongDimension("fieldName").comparator()); + } // Sort using StarTreeDocumentsSorter - StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i])); + StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i]), comparatorList); // Verify the sorting for (int i = 1; i < numDocs; i++) { @@ -166,7 +247,7 @@ private void testRandomizedSort() { Long[] curr = testData.get(sortedDocIds[i]); boolean isCorrectOrder = true; for (int j = dimensionId + 1; j < numDimensions; j++) { - int comparison = compareLongs(prev[j], curr[j]); + int comparison = comparatorList.get(j).compare(prev[j], curr[j]); if (comparison < 0) { break; } else if (comparison > 0) { @@ -186,16 +267,4 @@ private void testRandomizedSort() { } } - private int compareLongs(Long a, Long b) { - if (!Objects.equals(a, b)) { - if (a == null) { - return 1; - } else if (b == null) { - return -1; - } else { - return a.compareTo(b); - } - } - return 0; - } } diff --git a/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java index 684704ad65b0a..78edd7c6fe4e7 100644 --- a/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java @@ -25,6 +25,7 @@ import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.NumericDimension; import org.opensearch.index.compositeindex.datacube.ReadDimension; +import org.opensearch.index.compositeindex.datacube.UnsignedLongDimension; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings; @@ -77,7 +78,7 @@ public void testValidStarTree() throws IOException { Set compositeFieldTypes = mapperService.getCompositeFieldTypes(); for (CompositeMappedFieldType type : compositeFieldTypes) { StarTreeMapper.StarTreeFieldType starTreeFieldType = (StarTreeMapper.StarTreeFieldType) type; - assertEquals(2, starTreeFieldType.getDimensions().size()); + assertEquals(3, starTreeFieldType.getDimensions().size()); assertEquals("@timestamp", starTreeFieldType.getDimensions().get(0).getField()); assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension); DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0); @@ -89,6 +90,11 @@ public void testValidStarTree() throws IOException { assertEquals(expectedTimeUnits.get(i).shortName(), dateDim.getIntervals().get(i).shortName()); } assertEquals("status", starTreeFieldType.getDimensions().get(1).getField()); + assertTrue(starTreeFieldType.getDimensions().get(1) instanceof NumericDimension); + + assertEquals("unsignedLongDimension", starTreeFieldType.getDimensions().get(2).getField()); + assertTrue(starTreeFieldType.getDimensions().get(2) instanceof UnsignedLongDimension); + assertEquals(2, starTreeFieldType.getMetrics().size()); assertEquals("size", starTreeFieldType.getMetrics().get(0).getField()); @@ -154,6 +160,11 @@ public void testMetricsWithJustSum() throws IOException { assertEquals(new DateTimeUnitAdapter(expectedTimeUnits.get(i)), dateDim.getIntervals().get(i)); } assertEquals("status", starTreeFieldType.getDimensions().get(1).getField()); + assertTrue(starTreeFieldType.getDimensions().get(1) instanceof NumericDimension); + + assertEquals("unsignedLongDimension", starTreeFieldType.getDimensions().get(2).getField()); + assertTrue(starTreeFieldType.getDimensions().get(2) instanceof UnsignedLongDimension); + assertEquals("size", starTreeFieldType.getMetrics().get(0).getField()); // Assert AVG gets added when both of its base metrics is already present @@ -162,7 +173,7 @@ public void testMetricsWithJustSum() throws IOException { assertEquals(100, starTreeFieldType.getStarTreeConfig().maxLeafDocs()); assertEquals(StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP, starTreeFieldType.getStarTreeConfig().getBuildMode()); assertEquals( - new HashSet<>(Arrays.asList("@timestamp", "status")), + new HashSet<>(Arrays.asList("@timestamp", "status", "unsignedLongDimension")), starTreeFieldType.getStarTreeConfig().getSkipStarNodeCreationInDims() ); } @@ -548,6 +559,7 @@ public void testStarTreeField() { DateDimension d1 = new DateDimension("name", d1CalendarIntervals, DateFieldMapper.Resolution.MILLISECONDS); NumericDimension n1 = new NumericDimension("numeric"); NumericDimension n2 = new NumericDimension("name1"); + UnsignedLongDimension n3 = new UnsignedLongDimension("name2"); List metrics = List.of(metric1); List dims = List.of(d1, n2); @@ -561,6 +573,11 @@ public void testStarTreeField() { StarTreeField field2 = new StarTreeField("starTree", dims, metrics, config); assertEquals(field1, field2); + List dims1 = List.of(d1, n1, n2, n3); + StarTreeField field3 = new StarTreeField("starTree", dims1, metrics, config); + StarTreeField field4 = new StarTreeField("starTree", dims1, metrics, config); + assertEquals(field3, field4); + dims = List.of(d1, n2, n1); field2 = new StarTreeField("starTree", dims, metrics, config); assertNotEquals(field1, field2); @@ -677,6 +694,9 @@ private XContentBuilder getExpandedMappingWithJustAvg(String dim, String metric) b.startObject(); b.field("name", dim); b.endObject(); + b.startObject(); + b.field("name", "unsignedLongDimension"); // UnsignedLongDimension + b.endObject(); b.endArray(); b.startArray("metrics"); b.startObject(); @@ -702,6 +722,9 @@ private XContentBuilder getExpandedMappingWithJustAvg(String dim, String metric) b.startObject("keyword1"); b.field("type", "keyword"); b.endObject(); + b.startObject("unsignedLongDimension"); + b.field("type", "unsigned_long"); + b.endObject(); b.endObject(); }); } @@ -768,6 +791,7 @@ private XContentBuilder getExpandedMappingWithJustSum(String dim, String metric) { b.value("@timestamp"); b.value("status"); + b.value("unsignedLongDimension"); } b.endArray(); b.startObject("date_dimension"); @@ -781,6 +805,9 @@ private XContentBuilder getExpandedMappingWithJustSum(String dim, String metric) b.startObject(); b.field("name", dim); b.endObject(); + b.startObject(); + b.field("name", "unsignedLongDimension"); // UnsignedLongDimension + b.endObject(); b.endArray(); b.startArray("metrics"); b.startObject(); @@ -806,6 +833,9 @@ private XContentBuilder getExpandedMappingWithJustSum(String dim, String metric) b.startObject("keyword1"); b.field("type", "keyword"); b.endObject(); + b.startObject("unsignedLongDimension"); + b.field("type", "unsigned_long"); + b.endObject(); b.endObject(); }); }