Merge branch '2.x' into backport2.x-1930

Signed-off-by: Mitchell Gale <Mitchell.gale@improving.com>

core/src/main/java/org/opensearch/sql/analysis/Analyzer.java

@@ -161,7 +161,7 @@ public LogicalPlan visitRelation(Relation node, AnalysisContext context) {
     table.getFieldTypes().forEach((k, v) -> curEnv.define(new Symbol(Namespace.FIELD_NAME, k), v));
     table
         .getReservedFieldTypes()
-        .forEach((k, v) -> curEnv.addReservedWord(new Symbol(Namespace.FIELD_NAME, k), v));
+        .forEach((k, v) -> curEnv.define(new Symbol(Namespace.HIDDEN_FIELD_NAME, k), v));
 
     // Put index name or its alias in index namespace on type environment so qualifier
     // can be removed when analyzing qualified name. The value (expr type) here doesn't matter.
@@ -215,7 +215,7 @@ public LogicalPlan visitTableFunction(TableFunction node, AnalysisContext contex
     table.getFieldTypes().forEach((k, v) -> curEnv.define(new Symbol(Namespace.FIELD_NAME, k), v));
     table
         .getReservedFieldTypes()
-        .forEach((k, v) -> curEnv.addReservedWord(new Symbol(Namespace.FIELD_NAME, k), v));
+        .forEach((k, v) -> curEnv.define(new Symbol(Namespace.HIDDEN_FIELD_NAME, k), v));
     curEnv.define(
         new Symbol(
             Namespace.INDEX_NAME, dataSourceSchemaIdentifierNameResolver.getIdentifierName()),

core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java

@@ -378,7 +378,7 @@ public Expression visitQualifiedName(QualifiedName node, AnalysisContext context
           typeEnv != null;
           typeEnv = typeEnv.getParent()) {
         Optional<ExprType> exprType =
-            typeEnv.getReservedSymbolTable().lookup(new Symbol(Namespace.FIELD_NAME, part));
+            Optional.ofNullable(typeEnv.lookupAllFields(Namespace.HIDDEN_FIELD_NAME).get(part));
         if (exprType.isPresent()) {
           return visitMetadata(
               qualifierAnalyzer.unqualified(node), (ExprCoreType) exprType.get(), context);

core/src/main/java/org/opensearch/sql/analysis/TypeEnvironment.java

@@ -25,8 +25,6 @@ public class TypeEnvironment implements Environment<Symbol, ExprType> {
   @Getter private final TypeEnvironment parent;
   private final SymbolTable symbolTable;
 
-  @Getter private final SymbolTable reservedSymbolTable;
-
   /**
    * Constructor with empty symbol tables.
    *
@@ -35,7 +33,6 @@ public class TypeEnvironment implements Environment<Symbol, ExprType> {
   public TypeEnvironment(TypeEnvironment parent) {
     this.parent = parent;
     this.symbolTable = new SymbolTable();
-    this.reservedSymbolTable = new SymbolTable();
   }
 
   /**
@@ -47,7 +44,6 @@ public TypeEnvironment(TypeEnvironment parent) {
   public TypeEnvironment(TypeEnvironment parent, SymbolTable symbolTable) {
     this.parent = parent;
     this.symbolTable = symbolTable;
-    this.reservedSymbolTable = new SymbolTable();
   }
 
   /**
@@ -123,8 +119,4 @@ public void remove(ReferenceExpression ref) {
   public void clearAllFields() {
     lookupAllFields(FIELD_NAME).keySet().forEach(v -> remove(new Symbol(Namespace.FIELD_NAME, v)));
   }
-
-  public void addReservedWord(Symbol symbol, ExprType type) {
-    reservedSymbolTable.store(symbol, type);
-  }
 }

core/src/main/java/org/opensearch/sql/analysis/symbol/Namespace.java

@@ -9,6 +9,7 @@
 public enum Namespace {
   INDEX_NAME("Index"),
   FIELD_NAME("Field"),
+  HIDDEN_FIELD_NAME("HiddenField"),
   FUNCTION_NAME("Function");
 
   private final String name;

core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java

@@ -234,8 +234,8 @@ public void qualified_name_with_qualifier() {
   public void qualified_name_with_reserved_symbol() {
     analysisContext.push();
 
-    analysisContext.peek().addReservedWord(new Symbol(Namespace.FIELD_NAME, "_reserved"), STRING);
-    analysisContext.peek().addReservedWord(new Symbol(Namespace.FIELD_NAME, "_priority"), FLOAT);
+    analysisContext.peek().define(new Symbol(Namespace.HIDDEN_FIELD_NAME, "_reserved"), STRING);
+    analysisContext.peek().define(new Symbol(Namespace.HIDDEN_FIELD_NAME, "_priority"), FLOAT);
     analysisContext.peek().define(new Symbol(Namespace.INDEX_NAME, "index_alias"), STRUCT);
     assertAnalyzeEqual(
         DSL.ref("_priority", FLOAT),
@@ -246,7 +246,7 @@ public void qualified_name_with_reserved_symbol() {
         qualifiedName("index_alias", "_reserved")
     );
 
-    // reserved fields take priority over symbol table
+    // cannot replace an existing field type
     analysisContext.peek().define(new Symbol(Namespace.FIELD_NAME, "_reserved"), LONG);
     assertAnalyzeEqual(
         DSL.ref("_reserved", STRING),

doctest/test_data/nested_objects.json

@@ -1,4 +1,2 @@
-{"index":{"_id":"1"}}
 {"message":{"info":"a","author":"e","dayOfWeek":1},"comment":{"data":"ab","likes":3},"myNum":1,"someField":"b"}
-{"index":{"_id":"2"}}
 {"message":{"info":"b","author":"f","dayOfWeek":2},"comment":{"data":"aa","likes":2},"myNum":2,"someField":"a"}

doctest/test_data/wildcard.json

@@ -1,22 +1,11 @@
-{"index":{"_id":"0"}}
 {"Body":"test wildcard"}
-{"index":{"_id":"1"}}
 {"Body":"test wildcard in the end of the text%"}
-{"index":{"_id":"2"}}
 {"Body":"%test wildcard in the beginning of the text"}
-{"index":{"_id":"3"}}
 {"Body":"test wildcard in % the middle of the text"}
-{"index":{"_id":"4"}}
 {"Body":"test wildcard %% beside each other"}
-{"index":{"_id":"5"}}
 {"Body":"test wildcard in the end of the text_"}
-{"index":{"_id":"6"}}
 {"Body":"_test wildcard in the beginning of the text"}
-{"index":{"_id":"7"}}
 {"Body":"test wildcard in _ the middle of the text"}
-{"index":{"_id":"8"}}
 {"Body":"test wildcard __ beside each other"}
-{"index":{"_id":"9"}}
 {"Body":"test backslash wildcard \\_"}
-{"index":{"_id":"10"}}
 {"Body":"tEsT wIlDcArD sensitive cases"}