Skip to content

Commit

Permalink
Vector Tiles: add support for array serialization in meta layer (#72136)
Browse files Browse the repository at this point in the history 
Adds support for array serialization in meta layer
  • Loading branch information
@imotov
imotov authored Apr 24, 2021
1 parent d6cfec0 commit c4f919d
Show file tree
Hide file tree
Showing 4 changed files with 104 additions and 17 deletions.
31 changes: 25 additions & 6 deletions server/src/main/java/org/elasticsearch/common/util/Maps.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@

import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.TreeMap;
Expand Down Expand Up @@ -127,23 +128,41 @@ public static <K, V> boolean deepEquals(Map<K, V> left, Map<K, V> right) {
.allMatch(e -> right.containsKey(e.getKey()) && Objects.deepEquals(e.getValue(), right.get(e.getKey())));
}

public static Map<String, Object> flatten(Map<String, Object> map, boolean ordered) {
return flatten(map, ordered, null);
public static Map<String, Object> flatten(Map<String, Object> map, boolean flattenArrays, boolean ordered) {
return flatten(map, flattenArrays, ordered, null);
}

@SuppressWarnings("unchecked")
private static Map<String, Object> flatten(Map<String, Object> map, boolean ordered, String parentPath) {
Map<String, Object> flatMap = ordered ? new TreeMap() : new HashMap<>();
private static Map<String, Object> flatten(Map<String, Object> map, boolean flattenArrays, boolean ordered, String parentPath) {
Map<String, Object> flatMap = ordered ? new TreeMap<>() : new HashMap<>();
String prefix = parentPath != null ? parentPath + "." : "";

for (Map.Entry<String, Object> entry : map.entrySet()) {
if (entry.getValue() instanceof Map) {
flatMap.putAll(flatten((Map<String, Object>) entry.getValue(), ordered, prefix + entry.getKey()));
flatMap.putAll(flatten((Map<String, Object>) entry.getValue(), flattenArrays, ordered, prefix + entry.getKey()));
} else if (flattenArrays && entry.getValue() instanceof List) {
flatMap.putAll(flatten((List<Object>) entry.getValue(), ordered, prefix + entry.getKey()));
} else {
flatMap.put(prefix + entry.getKey(), entry.getValue());
}
}
return flatMap;
}

@SuppressWarnings("unchecked")
private static Map<String, Object> flatten(List<Object> list, boolean ordered, String parentPath) {
Map<String, Object> flatMap = ordered ? new TreeMap<>() : new HashMap<>();
String prefix = parentPath != null ? parentPath + "." : "";
for (int i = 0; i < list.size(); i++) {
Object cur = list.get(i);
if (cur instanceof Map) {
flatMap.putAll(flatten((Map<String, Object>) cur, true, ordered, prefix + i));
}
if (cur instanceof List) {
flatMap.putAll(flatten((List<Object>) cur, ordered, prefix + i));
} else {
flatMap.put(prefix + i, cur);
}
}
return flatMap;
}
}
65 changes: 65 additions & 0 deletions server/src/test/java/org/elasticsearch/common/util/MapsTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.function.Supplier;
import java.util.stream.Collectors;
Expand Down Expand Up @@ -122,6 +123,70 @@ public void testDeepEquals() {
assertFalse(Maps.deepEquals(map, mapModified));
}

public void testFlatten() {
Map<String, Object> map = randomNestedMap(10);
Map<String, Object> flatten = Maps.flatten(map, true, true);
assertThat(flatten.size(), equalTo(deepCount(map.values())));
for (Map.Entry<String, Object> entry : flatten.entrySet()) {
assertThat(entry.getKey(), entry.getValue(), equalTo(deepGet(entry.getKey(), map)));
}
}

@SuppressWarnings("unchecked")
private static Object deepGet(String path, Object obj) {
Object cur = obj;
String[] keys = path.split("\\.");
for (String key : keys) {
if (Character.isDigit(key.charAt(0))) {
List<Object> list = (List<Object>) cur;
cur = list.get(Integer.parseInt(key));
} else {
Map<String, Object> map = (Map<String, Object>) cur;
cur = map.get(key);
}
}
return cur;
}

@SuppressWarnings("unchecked")
private int deepCount(Collection<Object> map) {
int sum = 0;
for (Object val : map) {
if (val instanceof Map) {
sum += deepCount(((Map<String, Object>) val).values());
} else if (val instanceof List) {
sum += deepCount((List<Object>) val);
} else {
sum ++;
}
}
return sum;
}

private Map<String, Object> randomNestedMap(int level) {
final Supplier<String> keyGenerator = () -> randomAlphaOfLengthBetween(1, 5);
final Supplier<Object> arrayValueGenerator = () -> random().ints(randomInt(5))
.boxed()
.map(s -> (Object) s)
.collect(Collectors.toList());

final Supplier<Object> mapSupplier;
if (level > 0) {
mapSupplier = () -> randomNestedMap(level - 1);
} else {
mapSupplier = ESTestCase::randomLong;
}
final Supplier<Supplier<Object>> valueSupplier = () -> randomFrom(
ESTestCase::randomBoolean,
ESTestCase::randomDouble,
ESTestCase::randomLong,
arrayValueGenerator,
mapSupplier
);
final Supplier<Object> valueGenerator = () -> valueSupplier.get().get();
return randomMap(randomInt(5), keyGenerator, valueGenerator);
}

private void assertMapEntries(final Map<String, String> map, final Collection<Map.Entry<String, String>> entries) {
for (var entry : entries) {
assertThat("map [" + map + "] does not contain key [" + entry.getKey() + "]", map.keySet(), hasItem(entry.getKey()));
Expand Down
1 change: 1 addition & 0 deletions ...gin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ public static void addToXContentToFeature(VectorTile.Tile.Feature.Builder featur
throws IOException {
final Map<String, Object> map = Maps.flatten(
XContentHelper.convertToMap(XContentHelper.toXContent(toXContent, XContentType.CBOR, false), true, XContentType.CBOR).v2(),
true,
true
);
for (Map.Entry<String, Object> entry : map.entrySet()) {
Expand Down
24 changes: 13 additions & 11 deletions ...lugin/spatial/src/yamlRestTest/java/org/elasticsearch/xpack/spatial/VectorTileRestIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,15 +132,15 @@ public void testBasicGet() throws Exception {
assertThat(tile.getLayersCount(), Matchers.equalTo(3));
assertLayer(tile, HITS_LAYER, 4096, 33, 1);
assertLayer(tile, AGGS_LAYER, 4096, 1, 1);
assertLayer(tile, META_LAYER, 4096, 1, 15);
assertLayer(tile, META_LAYER, 4096, 1, 13);
}

public void testEmpty() throws Exception {
final int newY = (1 << z) - 1 == y ? y - 1 : y + 1;
final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + newY);
final VectorTile.Tile tile = execute(mvtRequest);
assertThat(tile.getLayersCount(), Matchers.equalTo(1));
assertLayer(tile, META_LAYER, 4096, 1, 12);
assertLayer(tile, META_LAYER, 4096, 1, 8);
}

public void testGridPrecision() throws Exception {
Expand All @@ -151,7 +151,7 @@ public void testGridPrecision() throws Exception {
assertThat(tile.getLayersCount(), Matchers.equalTo(3));
assertLayer(tile, HITS_LAYER, 4096, 33, 1);
assertLayer(tile, AGGS_LAYER, 4096, 1, 1);
assertLayer(tile, META_LAYER, 4096, 1, 15);
assertLayer(tile, META_LAYER, 4096, 1, 13);
}
{
final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y);
Expand All @@ -169,7 +169,7 @@ public void testGridType() throws Exception {
assertThat(tile.getLayersCount(), Matchers.equalTo(3));
assertLayer(tile, HITS_LAYER, 4096, 33, 1);
assertLayer(tile, AGGS_LAYER, 4096, 1, 1);
assertLayer(tile, META_LAYER, 4096, 1, 15);
assertLayer(tile, META_LAYER, 4096, 1, 13);
}
{
final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y);
Expand All @@ -178,7 +178,7 @@ public void testGridType() throws Exception {
assertThat(tile.getLayersCount(), Matchers.equalTo(3));
assertLayer(tile, HITS_LAYER, 4096, 33, 1);
assertLayer(tile, AGGS_LAYER, 4096, 1, 1);
assertLayer(tile, META_LAYER, 4096, 1, 15);
assertLayer(tile, META_LAYER, 4096, 1, 13);
}
{
final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y);
Expand All @@ -194,7 +194,7 @@ public void testNoAggLayer() throws Exception {
final VectorTile.Tile tile = execute(mvtRequest);
assertThat(tile.getLayersCount(), Matchers.equalTo(2));
assertLayer(tile, HITS_LAYER, 4096, 33, 1);
assertLayer(tile, META_LAYER, 4096, 1, 11);
assertLayer(tile, META_LAYER, 4096, 1, 9);
}

public void testNoHitsLayer() throws Exception {
Expand All @@ -203,7 +203,7 @@ public void testNoHitsLayer() throws Exception {
final VectorTile.Tile tile = execute(mvtRequest);
assertThat(tile.getLayersCount(), Matchers.equalTo(2));
assertLayer(tile, AGGS_LAYER, 4096, 1, 1);
assertLayer(tile, META_LAYER, 4096, 1, 14);
assertLayer(tile, META_LAYER, 4096, 1, 12);
}

public void testBasicQueryGet() throws Exception {
Expand All @@ -221,7 +221,7 @@ public void testBasicQueryGet() throws Exception {
assertThat(tile.getLayersCount(), Matchers.equalTo(3));
assertLayer(tile, HITS_LAYER, 4096, 1, 1);
assertLayer(tile, AGGS_LAYER, 4096, 1, 1);
assertLayer(tile, META_LAYER, 4096, 1, 15);
assertLayer(tile, META_LAYER, 4096, 1, 13);
}

public void testBasicShape() throws Exception {
Expand All @@ -230,7 +230,9 @@ public void testBasicShape() throws Exception {
assertThat(tile.getLayersCount(), Matchers.equalTo(3));
assertLayer(tile, HITS_LAYER, 4096, 1, 1);
assertLayer(tile, AGGS_LAYER, 4096, 256 * 256, 1);
assertLayer(tile, META_LAYER, 4096, 1, 15);
// The same max and min values are present in all buckets so min and max aggs return all buckets as max and mins
// as an array at the moment hence 2 additional values and 256 * 256 * 2 additional keys
assertLayer(tile, META_LAYER, 4096, 1, 9 + 2 + 256 * 256 * 2);
}

public void testWithFields() throws Exception {
Expand All @@ -240,7 +242,7 @@ public void testWithFields() throws Exception {
assertThat(tile.getLayersCount(), Matchers.equalTo(3));
assertLayer(tile, HITS_LAYER, 4096, 1, 3);
assertLayer(tile, AGGS_LAYER, 4096, 256 * 256, 1);
assertLayer(tile, META_LAYER, 4096, 1, 15);
assertLayer(tile, META_LAYER, 4096, 1, 9 + 2 + 256 * 256 * 2);
}

public void testMinAgg() throws Exception {
Expand All @@ -258,7 +260,7 @@ public void testMinAgg() throws Exception {
assertThat(tile.getLayersCount(), Matchers.equalTo(3));
assertLayer(tile, HITS_LAYER, 4096, 1, 1);
assertLayer(tile, AGGS_LAYER, 4096, 256 * 256, 2);
assertLayer(tile, META_LAYER, 4096, 1, 19);
assertLayer(tile, META_LAYER, 4096, 1, 9 + 4 + 256 * 256 * 4);
}

private void assertLayer(VectorTile.Tile tile, String name, int extent, int numFeatures, int numTags) {
Expand Down

0 comments on commit c4f919d

Please sign in to comment.