From f28df64fbbb0fff2e4d892c6e9b2720be80f6d40 Mon Sep 17 00:00:00 2001 From: kaiGo-Li Date: Mon, 28 Mar 2022 02:55:13 +0800 Subject: [PATCH] Update LexerUtils.scala MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Optimized the logic of LexerUtils.scala for blank line judgment. When the last word on the previous line of the cursor is ";", the complex judgment logic that follows will no longer be executed. Update LexerUtilsTest Added unit tests for corresponding optimizations。 And modified an error in the loginfo of the AutoSuggestContext.scala file --- .../autosuggest/AutoSuggestContext.scala | 2 +- .../autosuggest/statement/LexerUtils.scala | 81 ++++++++++++------- .../antlr4/autosuggest/LexerUtilsTest.scala | 36 ++++++++- 3 files changed, 86 insertions(+), 33 deletions(-) diff --git a/external/mlsql-autosuggest/src/main/java/tech/mlsql/autosuggest/AutoSuggestContext.scala b/external/mlsql-autosuggest/src/main/java/tech/mlsql/autosuggest/AutoSuggestContext.scala index 43c229f2c..00fdcba0c 100644 --- a/external/mlsql-autosuggest/src/main/java/tech/mlsql/autosuggest/AutoSuggestContext.scala +++ b/external/mlsql-autosuggest/src/main/java/tech/mlsql/autosuggest/AutoSuggestContext.scala @@ -167,7 +167,7 @@ class AutoSuggestContext(val session: SparkSession, private[autosuggest] def _suggest(tokenPos: TokenPos): List[SuggestItem] = { assert(_rawColumnNum != 0 || _rawLineNum != 0, "lineNum and columnNum should be set") if (isInDebugMode) { - logInfo("Global Pos::" + tokenPos.str + s"::${rawTokens(tokenPos.pos)}") + logInfo("Global Pos::" + tokenPos.str + s"::${if(tokenPos.pos == -1) null else rawTokens(tokenPos.pos)}") } if (tokenPos.pos == -1) { return firstWords diff --git a/external/mlsql-autosuggest/src/main/java/tech/mlsql/autosuggest/statement/LexerUtils.scala b/external/mlsql-autosuggest/src/main/java/tech/mlsql/autosuggest/statement/LexerUtils.scala index aa6853d64..735bcd2e4 100644 --- a/external/mlsql-autosuggest/src/main/java/tech/mlsql/autosuggest/statement/LexerUtils.scala +++ b/external/mlsql-autosuggest/src/main/java/tech/mlsql/autosuggest/statement/LexerUtils.scala @@ -4,7 +4,7 @@ import org.antlr.v4.runtime.Token import org.antlr.v4.runtime.misc.Interval import org.apache.commons.lang3.StringUtils import streaming.dsl.parser.DSLSQLLexer -import tech.mlsql.autosuggest.dsl.{MLSQLTokenTypeWrapper, TokenTypeWrapper} +import tech.mlsql.autosuggest.dsl.{DSLWrapper, MLSQLTokenTypeWrapper, TokenTypeWrapper} import tech.mlsql.autosuggest.{AutoSuggestContext, TokenPos, TokenPosType} import scala.collection.JavaConverters._ @@ -63,29 +63,36 @@ object LexerUtils { if (tokens.isEmpty) { return TokenPos(-1, TokenPosType.NEXT, -1) } - val _lastToken: Token = tokens.last + // Whether to enter the flag value of the last word on the line above the record cursor + var notEndCodeFlag: Boolean = false + var _lastToken: Token = tokens.last + // Determine if there is code after the line where the cursor is located + if(_lastToken.getLine > lineNum){ + notEndCodeFlag = true + } var _lastTokenIndex = 0 - var _lastLineHeadToken: Token = _lastToken - var _lastLineHeadTokenNum: Int = -1 - var _lastLineHeadTokenIndex = 0 val oneLineTokens = tokens.zipWithIndex.filter { case (token, index) => - _lastTokenIndex = index - if (_lastLineHeadTokenNum != token.getLine) { - _lastLineHeadTokenIndex = index - _lastLineHeadToken = token - _lastLineHeadTokenNum = token.getLine + /* A block of code that records the last word of the line before the cursor */ + if(token.getLine < lineNum && notEndCodeFlag){ + _lastToken = token + _lastTokenIndex = index + } + if(!notEndCodeFlag){ + _lastTokenIndex = index } token.getLine == lineNum } - val firstToken = oneLineTokens.headOption match { - case Some(head) => head - case None => (_lastLineHeadToken, _lastLineHeadTokenIndex) - } val lastToken = oneLineTokens.lastOption match { case Some(last) => last case None => (_lastToken, _lastTokenIndex) } - + if(oneLineTokens.isEmpty && lastToken._1.getType == DSLWrapper.SEMICOLON) { + return TokenPos(-1, TokenPosType.NEXT, -1) + } + val firstToken = oneLineTokens.headOption match { + case Some(head) => head + case None => (_lastToken, _lastTokenIndex) + } if (colNum < firstToken._1.getCharPositionInLine) { return TokenPos(firstToken._2 - 1, TokenPosType.NEXT, 0) } @@ -141,29 +148,36 @@ object LexerUtils { if (tokens.isEmpty) { return TokenPos(-1, TokenPosType.NEXT, -1) } - val _lastToken: Token = tokens.last + // Whether to enter the flag value of the last word on the line above the record cursor + var notEndCodeFlag: Boolean = false + var _lastToken: Token = tokens.last + // Determine if there is code after the line where the cursor is located + if(_lastToken.getLine > lineNum){ + notEndCodeFlag = true + } var _lastTokenIndex = 0 - var _lastLineHeadToken: Token = _lastToken - var _lastLineHeadTokenNum: Int = -1 - var _lastLineHeadTokenIndex = 0 val oneLineTokens = tokens.zipWithIndex.filter { case (token, index) => - _lastTokenIndex = index - if (_lastLineHeadTokenNum != token.getLine) { - _lastLineHeadTokenIndex = index - _lastLineHeadToken = token - _lastLineHeadTokenNum = token.getLine + /* A block of code that records the last word of the line before the cursor */ + if(token.getLine < lineNum && notEndCodeFlag){ + _lastToken = token + _lastTokenIndex = index + } + if(!notEndCodeFlag){ + _lastTokenIndex = index } token.getLine == lineNum } - val firstToken = oneLineTokens.headOption match { - case Some(head) => head - case None => (_lastLineHeadToken, _lastLineHeadTokenIndex) - } val lastToken = oneLineTokens.lastOption match { case Some(last) => last case None => (_lastToken, _lastTokenIndex) } - + if(oneLineTokens.isEmpty && lastToken._1.getType == DSLWrapper.SEMICOLON) { + return TokenPos(-1, TokenPosType.NEXT, -1) + } + val firstToken = oneLineTokens.headOption match { + case Some(head) => head + case None => (_lastToken, _lastTokenIndex) + } if (colNum < firstToken._1.getCharPositionInLine) { return TokenPos(firstToken._2 - 1, TokenPosType.NEXT, 0) } @@ -179,7 +193,7 @@ object LexerUtils { ) { return TokenPos(lastToken._2, TokenPosType.CURRENT, colNum - lastToken._1.getCharPositionInLine) } - oneLineTokens.map { case (token, index) => + val poses = oneLineTokens.map { case (token, index) => val start = token.getCharPositionInLine val end = token.getCharPositionInLine + token.getText.size //紧邻一个token的后面，没有空格,一般情况下是当做前一个token的一部分，用户还没写完，但是如果 @@ -199,7 +213,12 @@ object LexerUtils { } - }.filterNot(_.pos == -2).head + }.filterNot(_.pos == -2) + // If the result after the filter is empty, get the head directly to get the NPE + if (poses.isEmpty) { + return TokenPos(-1, TokenPosType.NEXT, -1) + } + poses.head } diff --git a/external/mlsql-autosuggest/src/test/java/com/intigua/antlr4/autosuggest/LexerUtilsTest.scala b/external/mlsql-autosuggest/src/test/java/com/intigua/antlr4/autosuggest/LexerUtilsTest.scala index 66760f580..398bc8499 100644 --- a/external/mlsql-autosuggest/src/test/java/com/intigua/antlr4/autosuggest/LexerUtilsTest.scala +++ b/external/mlsql-autosuggest/src/test/java/com/intigua/antlr4/autosuggest/LexerUtilsTest.scala @@ -42,5 +42,39 @@ class LexerUtilsTest extends BaseTest { context.buildFromString("load csv.") assert(LexerUtils.toTokenPos(context.rawTokens, 1, 9) == TokenPos(2, TokenPosType.NEXT, 0)) } - + test("select a,b,c from table1 as table1;select aa,bb,cc from table2 as table2;\\n \\n \\n select from table1 t1 left join table2 t2 on t1.a = t2."){ + val sql =""" + |select a,b,c from table1 as table1; + |select aa,bb,cc from table2 as table2; + | + | + | + |select from table1 t1 left join table2 t2 on t1.a = t2. + |""".stripMargin + val items = context.buildFromString(sql).suggest(4, 0) + assert(items.map(_.name) == List("load", "select", "include","register","run","train","predict","save","set")) + } + test("select a,b,c from table1 as table1;select \\n \\n \\n select from table1 t1 left join table2 t2 on t1.a = t2."){ + val sql =""" + |select a,b,c from table1 as table1; + |select + | + | + | + |select from table1 t1 left join table2 t2 on t1.a = t2. + |""".stripMargin + val items = context.buildFromString(sql).suggest(4, 0) + assert(items.map(_.name) == List("load", "select", "include","register","run","train","predict","save","set")) + } + test("select a,b,c from table1 as table1;select aa,bb,cc from table2 as table2;select from table1 t1 left join table2 t2 on t1.a = t2. \\n"){ + val sql =""" + |select a,b,c from table1 as table1; + |select aa,bb,cc from table2 as table2; + |select from table1 t1 left join table2 t2 on t1.a = t2. + | + | + |""".stripMargin + val items = context.buildFromString(sql).suggest(5, 0) + assert(items.map(_.name) == List("table1", "table2", "aa", "bb", "cc", "a", "b", "c", "count", "split")) + } }