Skip to content
This repository has been archived by the owner on Aug 2, 2022. It is now read-only.

Commit

Permalink
support Elasticsearch geo_point and ip data type (#719)
Browse files Browse the repository at this point in the history
  • Loading branch information
penghuo authored Sep 1, 2020
1 parent 3f3fbfd commit fe5d539
Show file tree
Hide file tree
Showing 24 changed files with 434 additions and 175 deletions.
4 changes: 2 additions & 2 deletions docs/experiment/ppl/cmd/stats.rst
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ PPL query::
+----------+--------------------+
| gender | avg(age) |
|----------+--------------------|
| F | 28 |
| F | 28.0 |
| M | 33.666666666666664 |
+----------+--------------------+

Expand All @@ -68,7 +68,7 @@ PPL query::
+----------+--------------------+------------+
| gender | avg(age) | sum(age) |
|----------+--------------------+------------|
| F | 28 | 28 |
| F | 28.0 | 28 |
| M | 33.666666666666664 | 101 |
+----------+--------------------+------------+

2 changes: 1 addition & 1 deletion docs/experiment/ppl/interfaces/endpoint.rst
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@ PPL query::
"type": "string"
}
],
"total": 4,
"datarows": [
[
"Amber",
Expand All @@ -62,6 +61,7 @@ PPL query::
"Adams"
]
],
"total": 4,
"size": 4
}

4 changes: 2 additions & 2 deletions docs/experiment/ppl/interfaces/protocol.rst
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@ PPL query::
"type": "string"
}
],
"total": 4,
"datarows": [
[
"Amber",
Expand All @@ -62,6 +61,7 @@ PPL query::
"Adams"
]
],
"total": 4,
"size": 4
}

Expand Down Expand Up @@ -94,7 +94,6 @@ PPL query::
"type": "string"
}
],
"total": 4,
"datarows": [
[
"Amber",
Expand All @@ -113,6 +112,7 @@ PPL query::
"Adams"
]
],
"total": 4,
"size": 4
}

Expand Down
16 changes: 8 additions & 8 deletions docs/user/dql/functions.rst
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ Example::
+-----------+
| ASIN(0) |
|-----------|
| 0 |
| 0.0 |
+-----------+


Expand Down Expand Up @@ -229,7 +229,7 @@ Example::
+----------+
| COS(0) |
|----------|
| 1 |
| 1.0 |
+----------+


Expand Down Expand Up @@ -676,7 +676,7 @@ Example::
+-------------+--------------+--------------------+
| POW(3, 2) | POW(-3, 2) | POW(3, -2) |
|-------------+--------------+--------------------|
| 9 | 9 | 0.1111111111111111 |
| 9.0 | 9.0 | 0.1111111111111111 |
+-------------+--------------+--------------------+


Expand All @@ -699,7 +699,7 @@ Example::
+---------------+----------------+--------------------+
| POWER(3, 2) | POWER(-3, 2) | POWER(3, -2) |
|---------------+----------------+--------------------|
| 9 | 9 | 0.1111111111111111 |
| 9.0 | 9.0 | 0.1111111111111111 |
+---------------+----------------+--------------------+


Expand Down Expand Up @@ -804,7 +804,7 @@ Example::
+----------------+-------------------+--------------------+----------------+
| ROUND(12.34) | ROUND(12.34, 1) | ROUND(12.34, -1) | ROUND(12, 1) |
|----------------+-------------------+--------------------+----------------|
| 12 | 12.3 | 10 | 12 |
| 12.0 | 12.3 | 10.0 | 12 |
+----------------+-------------------+--------------------+----------------+


Expand Down Expand Up @@ -872,7 +872,7 @@ Example::
+----------+
| SIN(0) |
|----------|
| 0 |
| 0.0 |
+----------+


Expand Down Expand Up @@ -909,7 +909,7 @@ Example::
+-----------+--------------+
| SQRT(4) | SQRT(4.41) |
|-----------+--------------|
| 2 | 2.1 |
| 2.0 | 2.1 |
+-----------+--------------+


Expand Down Expand Up @@ -954,7 +954,7 @@ Example::
+----------+
| TAN(0) |
|----------|
| 0 |
| 0.0 |
+----------+


Expand Down
10 changes: 10 additions & 0 deletions docs/user/general/datatypes.rst
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,10 @@ The ODFE SQL Engine support the following data types.
+---------------+
| interval |
+---------------+
| ip |
+---------------+
| geo_point |
+---------------+
| struct |
+---------------+
| array |
Expand All @@ -66,6 +70,8 @@ The table below list the mapping between Elasticsearch Data Type, ODFE SQL Data
+--------------------+---------------+-----------+
| float | float | FLOAT |
+--------------------+---------------+-----------+
| half_float | float | FLOAT |
+--------------------+---------------+-----------+
| double | double | DOUBLE |
+--------------------+---------------+-----------+
| keyword | string | VARCHAR |
Expand All @@ -74,6 +80,10 @@ The table below list the mapping between Elasticsearch Data Type, ODFE SQL Data
+--------------------+---------------+-----------+
| date | timestamp | TIMESTAMP |
+--------------------+---------------+-----------+
| ip | ip | IP |
+--------------------+---------------+-----------+
| date | timestamp | TIMESTAMP |
+--------------------+---------------+-----------+
| object | struct | STRUCT |
+--------------------+---------------+-----------+
| nested | array | TBD |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
package com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.type;

import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING;
import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.UNKNOWN;

import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType;
import java.util.Arrays;
Expand All @@ -40,7 +41,12 @@ public enum ElasticsearchDataType implements ExprType {
* Elasticsearch multi-fields which has text and keyword.
* Ref: https://www.elastic.co/guide/en/elasticsearch/reference/current/multi-fields.html
*/
ES_TEXT_KEYWORD(Arrays.asList(STRING, ES_TEXT), "string");
ES_TEXT_KEYWORD(Arrays.asList(STRING, ES_TEXT), "string"),


ES_IP(Arrays.asList(UNKNOWN), "ip"),

ES_GEO_POINT(Arrays.asList(UNKNOWN), "geo_point");

/**
* Parent of current type.
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
/*
*
* Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*
*/

package com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.value;

import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.type.ElasticsearchDataType.ES_GEO_POINT;

import com.amazon.opendistroforelasticsearch.sql.data.model.AbstractExprValue;
import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue;
import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType;
import java.util.Objects;
import lombok.Data;

/**
* Elasticsearch GeoPointValue.
* Todo, add this to avoid the unknown value type exception, the implementation will be changed.
*/
public class ElasticsearchExprGeoPointValue extends AbstractExprValue {

private final GeoPoint geoPoint;

public ElasticsearchExprGeoPointValue(Double lat, Double lon) {
this.geoPoint = new GeoPoint(lat, lon);
}

@Override
public Object value() {
return geoPoint;
}

@Override
public ExprType type() {
return ES_GEO_POINT;
}

@Override
public int compare(ExprValue other) {
return geoPoint.toString()
.compareTo((((ElasticsearchExprGeoPointValue) other).geoPoint).toString());
}

@Override
public boolean equal(ExprValue other) {
return geoPoint.equals(((ElasticsearchExprGeoPointValue) other).geoPoint);
}

@Override
public int hashCode() {
return Objects.hashCode(geoPoint);
}

@Data
public static class GeoPoint {

private final Double lat;

private final Double lon;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
/*
*
* Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*
*/

package com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.value;

import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.type.ElasticsearchDataType.ES_IP;

import com.amazon.opendistroforelasticsearch.sql.data.model.AbstractExprValue;
import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue;
import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType;
import java.util.Objects;
import lombok.RequiredArgsConstructor;

/**
* Elasticsearch IP ExprValue.
* Todo, add this to avoid the unknown value type exception, the implementation will be changed.
*/
@RequiredArgsConstructor
public class ElasticsearchExprIpValue extends AbstractExprValue {

private final String ip;

@Override
public Object value() {
return ip;
}

@Override
public ExprType type() {
return ES_IP;
}

@Override
public int compare(ExprValue other) {
return ip.compareTo(((ElasticsearchExprIpValue) other).ip);
}

@Override
public boolean equal(ExprValue other) {
return ip.equals(((ElasticsearchExprIpValue) other).ip);
}

@Override
public int hashCode() {
return Objects.hashCode(ip);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@
import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING;
import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRUCT;
import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.TIMESTAMP;
import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.type.ElasticsearchDataType.ES_GEO_POINT;
import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.type.ElasticsearchDataType.ES_IP;
import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.type.ElasticsearchDataType.ES_TEXT;
import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.type.ElasticsearchDataType.ES_TEXT_KEYWORD;
import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.value.ElasticsearchDateFormatters.SQL_LITERAL_DATE_TIME_FORMAT;
Expand Down Expand Up @@ -120,6 +122,11 @@ private ExprValue construct(String field, JsonNode value) {
return new ElasticsearchExprTextValue(value.asText());
} else if (type.equals(ES_TEXT_KEYWORD)) {
return new ElasticsearchExprTextKeywordValue(value.asText());
} else if (type.equals(ES_IP)) {
return new ElasticsearchExprIpValue(value.asText());
} else if (type.equals(ES_GEO_POINT)) {
return new ElasticsearchExprGeoPointValue(value.get("lat").doubleValue(),
value.get("lon").doubleValue());
} else {
throw new IllegalStateException(
String.format(
Expand All @@ -130,6 +137,7 @@ private ExprValue construct(String field, JsonNode value) {
/**
* Construct ExprValue from field and its value object. Throw exception if trying
* to construct from field of unsupported type.
* Todo, add IP, GeoPoint support after we have function implementation around it.
*
* @param field field name
* @param value value object
Expand Down Expand Up @@ -167,8 +175,8 @@ public ExprValue construct(String field, Object value) {
return new ElasticsearchExprTextKeywordValue((String) value);
} else {
throw new IllegalStateException(String.format(
"Unsupported type %s to construct expression value from object for "
+ "field: %s, value: %s.", type.typeName(), field, value));
"Unsupported type %s to construct expression value from object for "
+ "field: %s, value: %s.", type.typeName(), field, value));
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType;
import com.amazon.opendistroforelasticsearch.sql.elasticsearch.client.ElasticsearchClient;
import com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.type.ElasticsearchDataType;
import com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.value.ElasticsearchExprIpValue;
import com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.value.ElasticsearchExprValueFactory;
import com.amazon.opendistroforelasticsearch.sql.elasticsearch.mapping.IndexMapping;
import com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage.script.filter.FilterQueryBuilder;
Expand Down Expand Up @@ -53,11 +54,14 @@ public class ElasticsearchIndex implements Table {
.put("integer", ExprCoreType.INTEGER)
.put("long", ExprCoreType.LONG)
.put("float", ExprCoreType.FLOAT)
.put("half_float", ExprCoreType.FLOAT)
.put("double", ExprCoreType.DOUBLE)
.put("boolean", ExprCoreType.BOOLEAN)
.put("nested", ExprCoreType.ARRAY)
.put("object", ExprCoreType.STRUCT)
.put("date", ExprCoreType.TIMESTAMP)
.put("ip", ElasticsearchDataType.ES_IP)
.put("geo_point", ElasticsearchDataType.ES_GEO_POINT)
.build();

/** Elasticsearch client connection. */
Expand Down
Loading

0 comments on commit fe5d539

Please sign in to comment.