Skip to content

Commit

Permalink
Add value ranges for GeoTile aggregation metrics metrics in the meta …
Browse files Browse the repository at this point in the history 
…layer (#71611)
  • Loading branch information
@iverase
iverase authored Apr 22, 2021
1 parent 9a86dde commit d6cfec0
Show file tree
Hide file tree
Showing 5 changed files with 123 additions and 80 deletions.
32 changes: 31 additions & 1 deletion .../main/java/org/elasticsearch/xpack/spatial/vectortile/AbstractVectorTileSearchAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,14 +28,22 @@
import org.elasticsearch.rest.RestResponse;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.rest.action.RestResponseListener;
import org.elasticsearch.search.aggregations.AggregationBuilder;
import org.elasticsearch.search.aggregations.AggregatorFactories;
import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils;
import org.elasticsearch.search.aggregations.metrics.AvgAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.CardinalityAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.MinAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.FieldAndFormat;

import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.function.Supplier;
Expand Down Expand Up @@ -226,9 +234,31 @@ public AggregatorFactories.Builder getAggBuilder() {
}

public void setAggBuilder(AggregatorFactories.Builder aggBuilder) {
// TODO: validation
for (AggregationBuilder aggregation : aggBuilder.getAggregatorFactories()) {
final String type = aggregation.getType();
switch (type) {
case MinAggregationBuilder.NAME:
case MaxAggregationBuilder.NAME:
case AvgAggregationBuilder.NAME:
case SumAggregationBuilder.NAME:
case CardinalityAggregationBuilder.NAME:
break;
default:
// top term and percentile should be supported
throw new IllegalArgumentException("Unsupported aggregation of type [" + type + "]");
}
}
for (PipelineAggregationBuilder aggregation : aggBuilder.getPipelineAggregatorFactories()) {
// should not have pipeline aggregations
final String type = aggregation.getType();
throw new IllegalArgumentException("Unsupported pipeline aggregation of type [" + type + "]");
}
this.aggBuilder = aggBuilder;
}

public Collection<AggregationBuilder> getAggregations() {
return aggBuilder == null ? emptyList() : aggBuilder.getAggregatorFactories();
}
}

protected AbstractVectorTileSearchAction(Supplier<R> emptyRequestProvider) {
Expand Down
100 changes: 34 additions & 66 deletions ...patial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestVectorTileAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,43 +9,36 @@
import com.wdtinc.mapbox_vector_tile.VectorTile;
import com.wdtinc.mapbox_vector_tile.adapt.jts.IUserDataConverter;
import com.wdtinc.mapbox_vector_tile.build.MvtLayerProps;
import com.wdtinc.mapbox_vector_tile.encoding.MvtValue;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchResponseSections;
import org.elasticsearch.common.document.DocumentField;
import org.elasticsearch.common.geo.GeoBoundingBox;
import org.elasticsearch.common.geo.GeoPoint;
import org.elasticsearch.common.geo.GeometryParser;
import org.elasticsearch.common.util.Maps;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.geometry.Geometry;
import org.elasticsearch.geometry.Rectangle;
import org.elasticsearch.rest.RestRequest;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.AggregationBuilder;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils;
import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoGridBucket;
import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoTileGrid;
import org.elasticsearch.search.aggregations.metrics.AvgAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.CardinalityAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.GeoBoundsAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.InternalGeoBounds;
import org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.MinAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation;
import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder;
import org.elasticsearch.search.aggregations.pipeline.MaxBucketPipelineAggregationBuilder;
import org.elasticsearch.search.aggregations.pipeline.MinBucketPipelineAggregationBuilder;
import org.elasticsearch.search.fetch.subphase.FieldAndFormat;
import org.elasticsearch.search.profile.SearchProfileShardResults;

import java.io.IOException;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

import static org.elasticsearch.rest.RestRequest.Method.GET;
Expand All @@ -59,8 +52,11 @@ public class RestVectorTileAction extends AbstractVectorTileSearchAction<Abstrac
private static final String GRID_FIELD = "grid";
private static final String BOUNDS_FIELD = "bounds";

private static final String COUNT_TAG = "count";
private static final String ID_TAG = "id";
private static final String COUNT_TAG = "_count";
private static final String ID_TAG = "_id";

private static final String COUNT_MIN = "_count.min";
private static final String COUNT_MAX = "_count.max";

public RestVectorTileAction() {
super(Request::new);
Expand Down Expand Up @@ -115,13 +111,13 @@ protected ResponseBuilder doParseRequest(RestRequest restRequest, Request reques
s.getClusters()
);
if (hits.length > 0) {
tileBuilder.addLayers(getHitsLayer(s, request));
tileBuilder.addLayers(buildHitsLayer(s, request));
}
// TODO: should be expose the total number of buckets on InternalGeoTileGrid?
if (grid != null && grid.getBuckets().size() > 0) {
tileBuilder.addLayers(getAggsLayer(grid, request, geomBuilder));
tileBuilder.addLayers(buildAggsLayer(grid, request, geomBuilder));
}
tileBuilder.addLayers(getMetaLayer(meta, bounds, request, geomBuilder));
tileBuilder.addLayers(buildMetaLayer(meta, bounds, request, geomBuilder));
tileBuilder.build().writeTo(b);
};
}
Expand Down Expand Up @@ -150,6 +146,15 @@ private static SearchRequestBuilder searchBuilder(SearchRequestBuilder searchReq
aBuilder.subAggregations(request.getAggBuilder());
}
searchRequestBuilder.addAggregation(aBuilder);
searchRequestBuilder.addAggregation(new MaxBucketPipelineAggregationBuilder(COUNT_MAX, GRID_FIELD + "._count"));
searchRequestBuilder.addAggregation(new MinBucketPipelineAggregationBuilder(COUNT_MIN, GRID_FIELD + "._count"));
final Collection<AggregationBuilder> aggregations = request.getAggregations();
for (AggregationBuilder aggregation : aggregations) {
searchRequestBuilder.addAggregation(
new MaxBucketPipelineAggregationBuilder(aggregation.getName() + ".max", GRID_FIELD + ">" + aggregation.getName()));
searchRequestBuilder.addAggregation(
new MinBucketPipelineAggregationBuilder(aggregation.getName() + ".min", GRID_FIELD + ">" + aggregation.getName()));
}
}
if (request.getExactBounds()) {
final GeoBoundsAggregationBuilder boundsBuilder = new GeoBoundsAggregationBuilder(BOUNDS_FIELD).field(request.getField())
Expand All @@ -159,7 +164,7 @@ private static SearchRequestBuilder searchBuilder(SearchRequestBuilder searchReq
return searchRequestBuilder;
}

private VectorTile.Tile.Layer.Builder getHitsLayer(SearchResponse response, Request request) {
private VectorTile.Tile.Layer.Builder buildHitsLayer(SearchResponse response, Request request) {
final FeatureFactory featureFactory = new FeatureFactory(request.getZ(), request.getX(), request.getY(), request.getExtent());
final GeometryParser parser = new GeometryParser(true, false, false);
final VectorTile.Tile.Layer.Builder hitsLayerBuilder = VectorTileUtils.createLayerBuilder(HITS_LAYER, request.getExtent());
Expand All @@ -168,12 +173,12 @@ private VectorTile.Tile.Layer.Builder getHitsLayer(SearchResponse response, Requ
final IUserDataConverter tags = (userData, layerProps, featureBuilder) -> {
// TODO: It would be great if we can add the centroid information for polygons. That information can be
// used to place labels inside those geometries
addPropertyToFeature(featureBuilder, layerProps, ID_TAG, searchHit.getId());
VectorTileUtils.addPropertyToFeature(featureBuilder, layerProps, ID_TAG, searchHit.getId());
if (fields != null) {
for (FieldAndFormat field : fields) {
DocumentField documentField = searchHit.field(field.field);
final DocumentField documentField = searchHit.field(field.field);
if (documentField != null) {
addPropertyToFeature(featureBuilder, layerProps, field.field, documentField.getValue());
VectorTileUtils.addPropertyToFeature(featureBuilder, layerProps, field.field, documentField.getValue());
}
}
}
Expand All @@ -182,16 +187,16 @@ private VectorTile.Tile.Layer.Builder getHitsLayer(SearchResponse response, Requ
final Geometry geometry = parser.parseGeometry(searchHit.field(request.getField()).getValue());
hitsLayerBuilder.addAllFeatures(featureFactory.getFeatures(geometry, tags));
}
addPropertiesToLayer(hitsLayerBuilder, featureFactory.getLayerProps());
VectorTileUtils.addPropertiesToLayer(hitsLayerBuilder, featureFactory.getLayerProps());
return hitsLayerBuilder;
}

private VectorTile.Tile.Layer.Builder getAggsLayer(InternalGeoTileGrid grid, Request request, VectorTileGeometryBuilder geomBuilder) {
private VectorTile.Tile.Layer.Builder buildAggsLayer(InternalGeoTileGrid grid, Request request, VectorTileGeometryBuilder geomBuilder)
throws IOException{
final VectorTile.Tile.Layer.Builder aggLayerBuilder = VectorTileUtils.createLayerBuilder(AGGS_LAYER, request.getExtent());
final MvtLayerProps layerProps = new MvtLayerProps();
final VectorTile.Tile.Feature.Builder featureBuilder = VectorTile.Tile.Feature.newBuilder();
for (InternalGeoGridBucket<?> bucket : grid.getBuckets()) {
final long count = bucket.getDocCount();
featureBuilder.clear();
// Add geometry
if (request.getGridType() == GRID_TYPE.GRID) {
Expand All @@ -203,40 +208,22 @@ private VectorTile.Tile.Layer.Builder getAggsLayer(InternalGeoTileGrid grid, Req
geomBuilder.point(featureBuilder, point.lon(), point.lat());
}
// Add count as key value pair
addPropertyToFeature(featureBuilder, layerProps, COUNT_TAG, count);
// Add aggregations results as key value pair
VectorTileUtils.addPropertyToFeature(featureBuilder, layerProps, COUNT_TAG, bucket.getDocCount());
for (Aggregation aggregation : bucket.getAggregations()) {
final String type = aggregation.getType();
switch (type) {
case MinAggregationBuilder.NAME:
case MaxAggregationBuilder.NAME:
case AvgAggregationBuilder.NAME:
case SumAggregationBuilder.NAME:
case CardinalityAggregationBuilder.NAME:
final NumericMetricsAggregation.SingleValue metric = (NumericMetricsAggregation.SingleValue) aggregation;
addPropertyToFeature(featureBuilder, layerProps, "aggs." + aggregation.getName(), metric.value());
break;
default:
// top term and percentile should be supported
throw new IllegalArgumentException("Unknown feature type [" + type + "]");
}
VectorTileUtils.addToXContentToFeature(featureBuilder, layerProps, aggregation);
}
aggLayerBuilder.addFeatures(featureBuilder);
}
addPropertiesToLayer(aggLayerBuilder, layerProps);
VectorTileUtils.addPropertiesToLayer(aggLayerBuilder, layerProps);
return aggLayerBuilder;
}

private VectorTile.Tile.Layer.Builder getMetaLayer(
private VectorTile.Tile.Layer.Builder buildMetaLayer(
SearchResponse response,
InternalGeoBounds bounds,
Request request,
VectorTileGeometryBuilder geomBuilder
) throws IOException {
Map<String, Object> responseMap = Maps.flatten(
XContentHelper.convertToMap(XContentHelper.toXContent(response, XContentType.CBOR, false), true, XContentType.CBOR).v2(),
true
);
final VectorTile.Tile.Layer.Builder metaLayerBuilder = VectorTileUtils.createLayerBuilder(META_LAYER, request.getExtent());
final MvtLayerProps layerProps = new MvtLayerProps();
final VectorTile.Tile.Feature.Builder featureBuilder = VectorTile.Tile.Feature.newBuilder();
Expand All @@ -248,31 +235,12 @@ private VectorTile.Tile.Layer.Builder getMetaLayer(
final Rectangle tile = request.getBoundingBox();
geomBuilder.box(featureBuilder, tile.getMinLon(), tile.getMaxLon(), tile.getMinLat(), tile.getMaxLat());
}
for (Map.Entry<String, Object> entry : responseMap.entrySet()) {
if (entry.getValue() != null) {
addPropertyToFeature(featureBuilder, layerProps, entry.getKey(), entry.getValue());
}
}
VectorTileUtils.addToXContentToFeature(featureBuilder, layerProps, response);
metaLayerBuilder.addFeatures(featureBuilder);
addPropertiesToLayer(metaLayerBuilder, layerProps);
VectorTileUtils.addPropertiesToLayer(metaLayerBuilder, layerProps);
return metaLayerBuilder;
}

private void addPropertyToFeature(VectorTile.Tile.Feature.Builder feature, MvtLayerProps layerProps, String key, Object value) {
feature.addTags(layerProps.addKey(key));
feature.addTags(layerProps.addValue(value));
}

private void addPropertiesToLayer(VectorTile.Tile.Layer.Builder layer, MvtLayerProps layerProps) {
// Add keys
layer.addAllKeys(layerProps.getKeys());
// Add values
final Iterable<Object> values = layerProps.getVals();
for (Object value : values) {
layer.addValues(MvtValue.toValue(value));
}
}

@Override
public String getName() {
return "vectortile_action";
Expand Down
4 changes: 2 additions & 2 deletions ...l/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileGeometryBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,10 +57,10 @@ public void box(VectorTile.Tile.Feature.Builder featureBuilder, double minLon, d
}

private int lat(double lat) {
return (int) (pointYScale * (VectorTileUtils.latToSphericalMercator(lat) - rectangle.getMinY())) + extent;
return (int) Math.round(pointYScale * (VectorTileUtils.latToSphericalMercator(lat) - rectangle.getMinY())) + extent;
}

private int lon(double lon) {
return (int) (pointXScale * (VectorTileUtils.lonToSphericalMercator(lon) - rectangle.getMinX()));
return (int) Math.round(pointXScale * (VectorTileUtils.lonToSphericalMercator(lon) - rectangle.getMinX()));
}
}
46 changes: 46 additions & 0 deletions ...gin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,26 @@
package org.elasticsearch.xpack.spatial.vectortile;

import com.wdtinc.mapbox_vector_tile.VectorTile;
import com.wdtinc.mapbox_vector_tile.build.MvtLayerProps;
import com.wdtinc.mapbox_vector_tile.encoding.MvtValue;
import org.elasticsearch.common.util.Maps;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.geometry.Rectangle;
import org.locationtech.jts.geom.Envelope;

import java.io.IOException;
import java.util.Map;

/**
* Utility methods For vector tiles. Transforms WGS84 into spherical mercator.
*/
public class VectorTileUtils {

/**
* Creates a vector layer builder with the provided name and extent.
*/
public static VectorTile.Tile.Layer.Builder createLayerBuilder(String layerName, int extent) {
final VectorTile.Tile.Layer.Builder layerBuilder = VectorTile.Tile.Layer.newBuilder();
layerBuilder.setVersion(2);
Expand All @@ -24,6 +36,40 @@ public static VectorTile.Tile.Layer.Builder createLayerBuilder(String layerName,
return layerBuilder;
}

/**
* Adds the flatten elements of toXContent into the feature as tags.
*/
public static void addToXContentToFeature(VectorTile.Tile.Feature.Builder feature, MvtLayerProps layerProps, ToXContent toXContent)
throws IOException {
final Map<String, Object> map = Maps.flatten(
XContentHelper.convertToMap(XContentHelper.toXContent(toXContent, XContentType.CBOR, false), true, XContentType.CBOR).v2(),
true
);
for (Map.Entry<String, Object> entry : map.entrySet()) {
if (entry.getValue() != null) {
addPropertyToFeature(feature, layerProps, entry.getKey(), entry.getValue());
}
}
}

/**
* Adds the provided key / value pair into the feature as tags.
*/
public static void addPropertyToFeature(VectorTile.Tile.Feature.Builder feature, MvtLayerProps layerProps, String key, Object value) {
feature.addTags(layerProps.addKey(key));
feature.addTags(layerProps.addValue(value));
}

public static void addPropertiesToLayer(VectorTile.Tile.Layer.Builder layer, MvtLayerProps layerProps) {
// Add keys
layer.addAllKeys(layerProps.getKeys());
// Add values
final Iterable<Object> values = layerProps.getVals();
for (Object value : values) {
layer.addValues(MvtValue.toValue(value));
}
}

/**
* Gets the JTS envelope for z/x/y/ tile in spherical mercator projection.
*/
Expand Down
Loading

0 comments on commit d6cfec0

Please sign in to comment.