[PPL] Support index name with date suffix (#983)

* Change grammar and UT

* Add more UT and IT

* Support wildcard in index name with dots

* Update doc with wildcard case

* Add assertion for IT

* Disable nested field in PPL grammar

* Remove commented out code

docs/experiment/ppl/general/identifiers.rst

@@ -30,6 +30,8 @@ For Elasticsearch, the following identifiers are supported extensionally:
 3. Identifiers with ``-`` in the middle: this is mostly the case for index name with date information.
 4. Identifiers with star ``*`` present: this is mostly an index pattern for wildcard match.
 
+Index name with date suffix separated by dash or dots, such as ``cwl-2020.01.11`` or ``logs-7.0-2020.01.11``, is common for those created by Logstash or FileBeat ingestion. So, this kind of identifier used as index name is also supported without the need of being quoted for user convenience. In this case, wildcard within date pattern is also allowed to search for data across indices of different date range. For example, you can use ``logs-2020.1*`` to search in indices for October, November and December 2020.
+
 Examples
 --------
 

integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/SearchCommandIT.java

@@ -15,6 +15,7 @@
 
 package com.amazon.opendistroforelasticsearch.sql.ppl;
 
+import org.elasticsearch.client.Request;
 import org.elasticsearch.client.ResponseException;
 import org.json.JSONObject;
 import org.junit.jupiter.api.Test;
@@ -49,6 +50,15 @@ public void testSearchCommandWithoutSearchKeyword() throws IOException {
         executeQueryToString(String.format("source=%s", TEST_INDEX_BANK)));
   }
 
+  @Test
+  public void testSearchCommandWithSpecialIndexName() throws IOException {
+    executeRequest(new Request("PUT", "/logs-2021.01.11"));
+    verifyDataRows(executeQuery("search source=logs-2021.01.11"));
+
+    executeRequest(new Request("PUT", "/logs-7.10.0-2021.01.11"));
+    verifyDataRows(executeQuery("search source=logs-7.10.0-2021.01.11"));
+  }
+
   @Test
   public void testSearchCommandWithLogicalExpression() throws IOException {
     JSONObject result =

ppl/src/main/antlr/OpenDistroPPLLexer.g4

@@ -241,7 +241,9 @@ ID:                                 ID_LITERAL;
 INTEGER_LITERAL:                    DEC_DIGIT+;
 DECIMAL_LITERAL:                    (DEC_DIGIT+)? '.' DEC_DIGIT+;
 
+fragment DATE_SUFFIX:               ([\-.][*0-9]+)*;
 fragment ID_LITERAL:                [@*A-Z]+?[*A-Z_\-0-9]*;
+ID_DATE_SUFFIX:                     ID_LITERAL DATE_SUFFIX;
 DQUOTA_STRING:                      '"' ( '\\'. | '""' | ~('"'| '\\') )* '"';
 SQUOTA_STRING:                      '\'' ('\\'. | '\'\'' | ~('\'' | '\\'))* '\'';
 BQUOTA_STRING:                      '`' ( '\\'. | '``' | ~('`'|'\\'))* '`';

ppl/src/main/antlr/OpenDistroPPLParser.g4

@@ -178,6 +178,7 @@ booleanExpression
 /** tables */
 tableSource
     : qualifiedName
+    | ID_DATE_SUFFIX
     ;
 
 /** fields */
@@ -311,11 +312,11 @@ valueList
     ;
 
 qualifiedName
-    : ident (DOT ident)*                                            #identsAsQualifiedName
+    : ident                                          #identsAsQualifiedName
     ;
 
 wcQualifiedName
-    : wildcard (DOT wildcard)*                                      #identsAsWildcardQualifiedName
+    : wildcard                                       #identsAsWildcardQualifiedName
     ;
 
 ident

ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstBuilder.java

@@ -267,7 +267,7 @@ public UnresolvedPlan visitTopCommand(TopCommandContext ctx) {
    */
   @Override
   public UnresolvedPlan visitFromClause(FromClauseContext ctx) {
-    return new Relation(visitExpression(ctx.tableSource().qualifiedName()));
+    return new Relation(visitExpression(ctx.tableSource()));
   }
 
   /**

ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstExpressionBuilder.java

@@ -18,8 +18,10 @@
 import static com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName.IS_NOT_NULL;
 import static com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName.IS_NULL;
 import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.BinaryArithmeticContext;
+import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.BooleanFunctionCallContext;
 import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.BooleanLiteralContext;
 import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.CompareExprContext;
+import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.CountAllFunctionCallContext;
 import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.DecimalLiteralContext;
 import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.EvalClauseContext;
 import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.EvalFunctionCallContext;
@@ -38,6 +40,7 @@
 import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.SortFieldContext;
 import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.StatsFunctionCallContext;
 import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.StringLiteralContext;
+import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.TableSourceContext;
 import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.WcFieldExpressionContext;
 
 import com.amazon.opendistroforelasticsearch.sql.ast.expression.AggregateFunction;
@@ -59,7 +62,6 @@
 import com.amazon.opendistroforelasticsearch.sql.ast.expression.UnresolvedExpression;
 import com.amazon.opendistroforelasticsearch.sql.ast.expression.Xor;
 import com.amazon.opendistroforelasticsearch.sql.common.utils.StringUtils;
-import com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser;
 import com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParserBaseVisitor;
 import com.amazon.opendistroforelasticsearch.sql.ppl.utils.ArgumentFactory;
 import com.google.common.collect.ImmutableMap;
@@ -183,8 +185,7 @@ public UnresolvedExpression visitStatsFunctionCall(StatsFunctionCallContext ctx)
   }
 
   @Override
-  public UnresolvedExpression visitCountAllFunctionCall(
-      OpenDistroPPLParser.CountAllFunctionCallContext ctx) {
+  public UnresolvedExpression visitCountAllFunctionCall(CountAllFunctionCallContext ctx) {
     return new AggregateFunction("count", AllFields.of());
   }
 
@@ -198,8 +199,7 @@ public UnresolvedExpression visitPercentileAggFunction(PercentileAggFunctionCont
    * Eval function.
    */
   @Override
-  public UnresolvedExpression visitBooleanFunctionCall(
-      OpenDistroPPLParser.BooleanFunctionCallContext ctx) {
+  public UnresolvedExpression visitBooleanFunctionCall(BooleanFunctionCallContext ctx) {
     final String functionName = ctx.conditionFunctionBase().getText();
 
     return new Function(
@@ -225,30 +225,23 @@ public UnresolvedExpression visitEvalFunctionCall(EvalFunctionCallContext ctx) {
             .collect(Collectors.toList()));
   }
 
+  @Override
+  public UnresolvedExpression visitTableSource(TableSourceContext ctx) {
+    return visitIdentifier(ctx);
+  }
+
   /**
    * Literal and value.
    */
   @Override
   public UnresolvedExpression visitIdentsAsQualifiedName(IdentsAsQualifiedNameContext ctx) {
-    return new QualifiedName(
-        ctx.ident()
-            .stream()
-            .map(ParserRuleContext::getText)
-            .map(StringUtils::unquoteText)
-            .collect(Collectors.toList())
-    );
+    return visitIdentifier(ctx.ident());
   }
 
   @Override
   public UnresolvedExpression visitIdentsAsWildcardQualifiedName(
       IdentsAsWildcardQualifiedNameContext ctx) {
-    return new QualifiedName(
-        ctx.wildcard()
-            .stream()
-            .map(ParserRuleContext::getText)
-            .map(StringUtils::unquoteText)
-            .collect(Collectors.toList())
-    );
+    return visitIdentifier(ctx.wildcard());
   }
 
   @Override
@@ -277,4 +270,9 @@ public UnresolvedExpression visitBooleanLiteral(BooleanLiteralContext ctx) {
     return new Literal(Boolean.valueOf(ctx.getText()), DataType.BOOLEAN);
   }
 
+  private UnresolvedExpression visitIdentifier(ParserRuleContext ctx) {
+    return new QualifiedName(Collections.singletonList(
+        StringUtils.unquoteIdentifier(ctx.getText())));
+  }
+
 }

ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstBuilderTest.java

@@ -50,7 +50,6 @@
 
 import com.amazon.opendistroforelasticsearch.sql.ast.Node;
 import com.amazon.opendistroforelasticsearch.sql.ast.tree.RareTopN.CommandType;
-import com.amazon.opendistroforelasticsearch.sql.common.antlr.SyntaxCheckException;
 import com.amazon.opendistroforelasticsearch.sql.ppl.antlr.PPLSyntaxParser;
 import org.junit.Ignore;
 import org.junit.Rule;
@@ -379,9 +378,9 @@ public void testIdentifierAsIndexNameStartWithDot() {
   }
 
   @Test
-  public void identifierAsIndexNameWithDotInTheMiddleThrowException() {
-    exceptionRule.expect(SyntaxCheckException.class);
-    plan("source=log.2020.10.10");
+  public void testIdentifierAsIndexNameWithDotInTheMiddle() {
+    assertEqual("source=log.2020.10.10", relation("log.2020.10.10"));
+    assertEqual("source=log-7.10-2020.10.10", relation("log-7.10-2020.10.10"));
   }
 
   @Test
@@ -396,6 +395,13 @@ public void testIdentifierAsIndexNameContainStar() {
         relation("log-2020-10-*"));
   }
 
+  @Test
+  public void testIdentifierAsIndexNameContainStarAndDots() {
+    assertEqual("source=log-2020.10.*", relation("log-2020.10.*"));
+    assertEqual("source=log-2020.*.01", relation("log-2020.*.01"));
+    assertEqual("source=log-2020.*.*", relation("log-2020.*.*"));
+  }
+
   @Test
   public void testIdentifierAsFieldNameStartWithAt() {
     assertEqual("source=log-2020 | fields @timestamp",

ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstExpressionBuilderTest.java

@@ -376,6 +376,7 @@ public void testEvalFuncCallExpr() {
         ));
   }
 
+  @Ignore("Nested field is not supported in backend yet")
   @Test
   public void testNestedFieldName() {
     assertEqual("source=t | fields field0.field1.field2",
@@ -390,6 +391,19 @@ public void testNestedFieldName() {
 
   @Test
   public void testFieldNameWithSpecialChars() {
+    assertEqual("source=t | fields `field-0`",
+        projectWithArg(
+            relation("t"),
+            defaultFieldsArgs(),
+            field(
+                qualifiedName("field-0")
+            )
+        ));
+  }
+
+  @Ignore("Nested field is not supported in backend yet")
+  @Test
+  public void testNestedFieldNameWithSpecialChars() {
     assertEqual("source=t | fields `field-0`.`field#1`.`field*2`",
         projectWithArg(
             relation("t"),
@@ -486,9 +500,9 @@ public void testKeywordsAsIdentifiers() {
   @Test
   public void canBuildKeywordsAsIdentInQualifiedName() {
     assertEqual(
-        "source=test.timestamp | fields timestamp",
+        "source=test | fields timestamp",
         projectWithArg(
-            relation("test.timestamp"),
+            relation("test"),
             defaultFieldsArgs(),
             field("timestamp")
         )

ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java

@@ -151,8 +151,8 @@ public void testNotExpression() {
 
   @Test
   public void testQualifiedName() {
-    assertEquals("source=t | fields + field0.field1",
-        anonymize("source=t | fields field0.field1")
+    assertEquals("source=t | fields + field0",
+        anonymize("source=t | fields field0")
     );
   }