Skip to content
This repository has been archived by the owner on Aug 2, 2022. It is now read-only.

Merge develop to master for ODFE 1.9.0.1 release #633

Merged
merged 36 commits into from
Jul 29, 2020
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
36 commits
Select commit Hold shift + click to select a range
b071f51
Merge all SQL repos and adjust workflows (#549)
joshuali925 Jul 9, 2020
0622896
adding filter for columns in SQLColumns call (#556)
rupal-bq Jul 10, 2020
4b139c4
Keep mismatch results when error occurs in comparison test (#557)
dai-chen Jul 10, 2020
13afbe5
Support mathematical functions ceil/ceiling, exp, floor, ln, log (#540)
chloe-zh Jul 12, 2020
bc02233
add date and time support (#568)
penghuo Jul 13, 2020
eea2920
ODBC: Build driver files in parallel (#570)
jordanw-bq Jul 13, 2020
4888d37
ODBC: Remove catalog support from driver (#566)
jordanw-bq Jul 13, 2020
7fbe349
ODBC: Add documents for `Refresh` & `Export as CSV files` options in …
rupal-bq Jul 13, 2020
51f353d
ODBC: Update documentation for using Microsoft Excel with Open Distro…
rupal-bq Jul 14, 2020
0b57d93
Add workflow to draft release on push (#572)
joshuali925 Jul 16, 2020
9d72867
Support mathematical functions: conv, crc32, mod, pow/power, round, s…
chloe-zh Jul 20, 2020
d74cb81
Fix object/nested field select issue (#584)
dai-chen Jul 20, 2020
c66b85f
ODBC: Add user documentation for using Microsoft Excel on Mac (#594)
rupal-bq Jul 20, 2020
af4ff91
ODBC: add ODBC 2.x functions called by Excel for Mac (#592)
jordanw-bq Jul 20, 2020
8262994
ODBC: Updating Microsoft Excel connection documents (#581)
rupal-bq Jul 20, 2020
c9ea038
Extra fixes for Mac ODBC driver (#602)
jordanw-bq Jul 21, 2020
277c966
bumped lodash version (#598)
chloe-zh Jul 21, 2020
0a878ab
Support SELECT * and FROM clause in new SQL parser (#573)
dai-chen Jul 21, 2020
5365ca0
ODBC: Adding Power BI M connector (#596)
rupal-bq Jul 21, 2020
e8f0539
Using UTC as default timezone for date_format function if not provide…
penghuo Jul 21, 2020
7372a44
Fix CAST bool field to integer issue (#600)
dai-chen Jul 22, 2020
78a51a4
Support mathematical functions rand and constants e, pi (#591)
chloe-zh Jul 22, 2020
49883b9
Support trigonometric functions acos, asin, atan, atan2, cos, cot, de…
chloe-zh Jul 22, 2020
353c1c8
Add ElasticsearchExprValueFactory in StorageEngine (#608)
penghuo Jul 23, 2020
6ba778f
Update docs after merging repos (#563)
joshuali925 Jul 24, 2020
3368d60
ODBC: Report error from Excel when executing an invalid query (#611)
jordanw-bq Jul 25, 2020
f3517ee
ODBC: adding manual test plan for Microsoft Excel testing (#604)
rupal-bq Jul 25, 2020
9dac0ac
ODBC: Adding BASIC & AWS_SIGV4 auth in M Connector (#610)
rupal-bq Jul 25, 2020
e370ce9
ODBC: Change Tableau connector version (#622)
rupal-bq Jul 28, 2020
5a7084f
Issue 623, fix security vulnerability regarding to depedencies common…
penghuo Jul 28, 2020
0211d7e
Support queries end with semi colon (#609)
dai-chen Jul 29, 2020
bee2ca2
ODBC: Fix for data loading failure in Power BI Desktop (#627)
rupal-bq Jul 29, 2020
b258fda
Add Text and Keyword Data Type (#620)
penghuo Jul 29, 2020
e61b5fd
Move workbench down in kibana nav (#578)
joshuali925 Jul 29, 2020
caf21a6
Bump versions to 1.9.0.1 and add release notes (#632)
joshuali925 Jul 29, 2020
baac103
resolve conflict
joshuali925 Jul 29, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Using UTC as default timezone for date_format function if not provided (
#605)

* Using UTC as default Zone for date_format function

* update

* update
  • Loading branch information
penghuo authored Jul 21, 2020
commit e8f0539d39fc00f5b6c7dbc595b72c85416461cc
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,10 @@

package com.amazon.opendistroforelasticsearch.sql.legacy;

import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.rows;
import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.schema;
import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.verifyDataRows;
import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.verifySchema;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.is;

Expand Down Expand Up @@ -116,6 +120,17 @@ public void and() throws SqlParseException {
);
}

@Test
public void andWithDefaultTimeZone() throws SqlParseException {
assertThat(
dateQuery(SELECT_FROM +
"WHERE date_format(insert_time, 'yyyy-MM-dd HH:mm:ss') >= '2014-08-17 16:13:12' " +
"AND date_format(insert_time, 'yyyy-MM-dd HH:mm:ss') <= '2014-08-17 16:13:13'",
"yyyy-MM-dd HH:mm:ss"),
contains("2014-08-17 16:13:12")
);
}

@Test
public void or() throws SqlParseException {
assertThat(
Expand Down Expand Up @@ -157,6 +172,17 @@ public void sortByAliasedDateFormat() throws IOException {
is(new DateTime("2014-08-24T00:00:41.221Z", DateTimeZone.UTC)));
}

@Test
public void selectDateTimeWithDefaultTimeZone() throws SqlParseException {
JSONObject response = executeJdbcRequest("SELECT date_format(insert_time, 'yyyy-MM-dd') as date " +
" FROM " + TestsConstants.TEST_INDEX_ONLINE +
" WHERE date_format(insert_time, 'yyyy-MM-dd HH:mm:ss') >= '2014-08-17 16:13:12' " +
" AND date_format(insert_time, 'yyyy-MM-dd HH:mm:ss') <= '2014-08-17 16:13:13'");

verifySchema(response, schema("date", "", "text"));
verifyDataRows(response, rows("2014-08-17"));
}

@Test
public void groupByAndSort() throws IOException {
JSONObject aggregations = executeQuery(
Expand Down Expand Up @@ -209,17 +235,19 @@ private void checkAggregations(JSONObject aggregations, String key,
}

private Set<Object> dateQuery(String sql) throws SqlParseException {
return dateQuery(sql, TestsConstants.SIMPLE_DATE_FORMAT);
}

private Set<Object> dateQuery(String sql, String format) throws SqlParseException {
try {
JSONObject response = executeQuery(sql);
return getResult(response, "insert_time");
return getResult(response, "insert_time", DateTimeFormat.forPattern(format));
} catch (IOException e) {
throw new SqlParseException(String.format("Unable to process query '%s'", sql));
}
}

private Set<Object> getResult(JSONObject response, String fieldName) {
DateTimeFormatter formatter = DateTimeFormat.forPattern(TestsConstants.SIMPLE_DATE_FORMAT);

private Set<Object> getResult(JSONObject response, String fieldName, DateTimeFormatter formatter) {
JSONArray hits = getHits(response);
Set<Object> result = new TreeSet<>(); // Using TreeSet so order is maintained
for (int i = 0; i < hits.length(); i++) {
Expand All @@ -242,4 +270,8 @@ public static String getScriptAggregationKey(JSONObject aggregation, String pref
.orElseThrow(() -> new RuntimeException(
"Can't find key" + prefix + " in aggregation " + aggregation));
}

private JSONObject executeJdbcRequest(String query) {
return new JSONObject(executeQuery(query, "jdbc"));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,11 @@

public abstract class Maker {

/**
* UTC.
*/
private static final ZoneId UTC = ZoneId.of("UTC");

public static final Object NONE = new Object();

public static final Set<String> queryFunctions = Sets.newHashSet(
Expand Down Expand Up @@ -409,7 +414,8 @@ private ToXContent makeForDateFormat(SQLMethodInvokeExpr nameExpr, SQLCharExpr v
if (params.size() > 2) {
zoneId = ZoneId.of(removeSingleQuote(params.get(2).toString())).toString();
} else {
zoneId = ZoneId.systemDefault().toString();
// Using UTC, if there is no Zone provided.
zoneId = UTC.getId();
}

RangeQueryBuilder rangeQuery = QueryBuilders.rangeQuery(field).format(format).timeZone(zoneId);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -527,7 +527,7 @@ private Tuple<String, String> date_format(SQLExpr field, String pattern, String
String name = nextId("date_format");
if (valueName == null) {
return new Tuple<>(name, "def " + name + " = DateTimeFormatter.ofPattern('" + pattern + "').withZone("
+ (zoneId != null ? "ZoneId.of('" + zoneId + "')" : "ZoneId.systemDefault()")
+ (zoneId != null ? "ZoneId.of('" + zoneId + "')" : "ZoneId.of(\"UTC\")")
+ ").format(Instant.ofEpochMilli(" + getPropertyOrValue(field) + ".toInstant().toEpochMilli()))");
} else {
return new Tuple<>(name, exprString(field) + "; "
Expand Down