Skip to content

Commit

Permalink
Update LexerUtils.scala
Browse files Browse the repository at this point in the history 
Optimized the logic of LexerUtils.scala for blank line judgment.
When the last word on the previous line of the cursor is ";", the complex judgment logic that follows will no longer be executed.

Update LexerUtilsTest

Added unit tests for corresponding optimizations。
And modified an error in the loginfo of the AutoSuggestContext.scala file
  • Loading branch information
@kaliGo-Li
kaliGo-Li committed Mar 28, 2022
1 parent d8230cf commit f28df64
Show file tree
Hide file tree
Showing 3 changed files with 86 additions and 33 deletions.
2 changes: 1 addition & 1 deletion external/mlsql-autosuggest/src/main/java/tech/mlsql/autosuggest/AutoSuggestContext.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,7 @@ class AutoSuggestContext(val session: SparkSession,
private[autosuggest] def _suggest(tokenPos: TokenPos): List[SuggestItem] = {
assert(_rawColumnNum != 0 || _rawLineNum != 0, "lineNum and columnNum should be set")
if (isInDebugMode) {
logInfo("Global Pos::" + tokenPos.str + s"::${rawTokens(tokenPos.pos)}")
logInfo("Global Pos::" + tokenPos.str + s"::${if(tokenPos.pos == -1) null else rawTokens(tokenPos.pos)}")
}
if (tokenPos.pos == -1) {
return firstWords
Expand Down
81 changes: 50 additions & 31 deletions external/mlsql-autosuggest/src/main/java/tech/mlsql/autosuggest/statement/LexerUtils.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import org.antlr.v4.runtime.Token
import org.antlr.v4.runtime.misc.Interval
import org.apache.commons.lang3.StringUtils
import streaming.dsl.parser.DSLSQLLexer
import tech.mlsql.autosuggest.dsl.{MLSQLTokenTypeWrapper, TokenTypeWrapper}
import tech.mlsql.autosuggest.dsl.{DSLWrapper, MLSQLTokenTypeWrapper, TokenTypeWrapper}
import tech.mlsql.autosuggest.{AutoSuggestContext, TokenPos, TokenPosType}

import scala.collection.JavaConverters._
Expand Down Expand Up @@ -63,29 +63,36 @@ object LexerUtils {
if (tokens.isEmpty) {
return TokenPos(-1, TokenPosType.NEXT, -1)
}
val _lastToken: Token = tokens.last
// Whether to enter the flag value of the last word on the line above the record cursor
var notEndCodeFlag: Boolean = false
var _lastToken: Token = tokens.last
// Determine if there is code after the line where the cursor is located
if(_lastToken.getLine > lineNum){
notEndCodeFlag = true
}
var _lastTokenIndex = 0
var _lastLineHeadToken: Token = _lastToken
var _lastLineHeadTokenNum: Int = -1
var _lastLineHeadTokenIndex = 0
val oneLineTokens = tokens.zipWithIndex.filter { case (token, index) =>
_lastTokenIndex = index
if (_lastLineHeadTokenNum != token.getLine) {
_lastLineHeadTokenIndex = index
_lastLineHeadToken = token
_lastLineHeadTokenNum = token.getLine
/* A block of code that records the last word of the line before the cursor */
if(token.getLine < lineNum && notEndCodeFlag){
_lastToken = token
_lastTokenIndex = index
}
if(!notEndCodeFlag){
_lastTokenIndex = index
}
token.getLine == lineNum
}
val firstToken = oneLineTokens.headOption match {
case Some(head) => head
case None => (_lastLineHeadToken, _lastLineHeadTokenIndex)
}
val lastToken = oneLineTokens.lastOption match {
case Some(last) => last
case None => (_lastToken, _lastTokenIndex)
}

if(oneLineTokens.isEmpty && lastToken._1.getType == DSLWrapper.SEMICOLON) {
return TokenPos(-1, TokenPosType.NEXT, -1)
}
val firstToken = oneLineTokens.headOption match {
case Some(head) => head
case None => (_lastToken, _lastTokenIndex)
}
if (colNum < firstToken._1.getCharPositionInLine) {
return TokenPos(firstToken._2 - 1, TokenPosType.NEXT, 0)
}
Expand Down Expand Up @@ -141,29 +148,36 @@ object LexerUtils {
if (tokens.isEmpty) {
return TokenPos(-1, TokenPosType.NEXT, -1)
}
val _lastToken: Token = tokens.last
// Whether to enter the flag value of the last word on the line above the record cursor
var notEndCodeFlag: Boolean = false
var _lastToken: Token = tokens.last
// Determine if there is code after the line where the cursor is located
if(_lastToken.getLine > lineNum){
notEndCodeFlag = true
}
var _lastTokenIndex = 0
var _lastLineHeadToken: Token = _lastToken
var _lastLineHeadTokenNum: Int = -1
var _lastLineHeadTokenIndex = 0
val oneLineTokens = tokens.zipWithIndex.filter { case (token, index) =>
_lastTokenIndex = index
if (_lastLineHeadTokenNum != token.getLine) {
_lastLineHeadTokenIndex = index
_lastLineHeadToken = token
_lastLineHeadTokenNum = token.getLine
/* A block of code that records the last word of the line before the cursor */
if(token.getLine < lineNum && notEndCodeFlag){
_lastToken = token
_lastTokenIndex = index
}
if(!notEndCodeFlag){
_lastTokenIndex = index
}
token.getLine == lineNum
}
val firstToken = oneLineTokens.headOption match {
case Some(head) => head
case None => (_lastLineHeadToken, _lastLineHeadTokenIndex)
}
val lastToken = oneLineTokens.lastOption match {
case Some(last) => last
case None => (_lastToken, _lastTokenIndex)
}

if(oneLineTokens.isEmpty && lastToken._1.getType == DSLWrapper.SEMICOLON) {
return TokenPos(-1, TokenPosType.NEXT, -1)
}
val firstToken = oneLineTokens.headOption match {
case Some(head) => head
case None => (_lastToken, _lastTokenIndex)
}
if (colNum < firstToken._1.getCharPositionInLine) {
return TokenPos(firstToken._2 - 1, TokenPosType.NEXT, 0)
}
Expand All @@ -179,7 +193,7 @@ object LexerUtils {
) {
return TokenPos(lastToken._2, TokenPosType.CURRENT, colNum - lastToken._1.getCharPositionInLine)
}
oneLineTokens.map { case (token, index) =>
val poses = oneLineTokens.map { case (token, index) =>
val start = token.getCharPositionInLine
val end = token.getCharPositionInLine + token.getText.size
//紧邻一个token的后面，没有空格,一般情况下是当做前一个token的一部分，用户还没写完，但是如果
Expand All @@ -199,7 +213,12 @@ object LexerUtils {
}


}.filterNot(_.pos == -2).head
}.filterNot(_.pos == -2)
// If the result after the filter is empty, get the head directly to get the NPE
if (poses.isEmpty) {
return TokenPos(-1, TokenPosType.NEXT, -1)
}
poses.head
}


Expand Down
36 changes: 35 additions & 1 deletion external/mlsql-autosuggest/src/test/java/com/intigua/antlr4/autosuggest/LexerUtilsTest.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,5 +42,39 @@ class LexerUtilsTest extends BaseTest {
context.buildFromString("load csv.")
assert(LexerUtils.toTokenPos(context.rawTokens, 1, 9) == TokenPos(2, TokenPosType.NEXT, 0))
}

test("select a,b,c from table1 as table1;select aa,bb,cc from table2 as table2;\\n \\n \\n select from table1 t1 left join table2 t2 on t1.a = t2."){
val sql ="""
|select a,b,c from table1 as table1;
|select aa,bb,cc from table2 as table2;
|
|
|
|select from table1 t1 left join table2 t2 on t1.a = t2.
|""".stripMargin
val items = context.buildFromString(sql).suggest(4, 0)
assert(items.map(_.name) == List("load", "select", "include","register","run","train","predict","save","set"))
}
test("select a,b,c from table1 as table1;select \\n \\n \\n select from table1 t1 left join table2 t2 on t1.a = t2."){
val sql ="""
|select a,b,c from table1 as table1;
|select
|
|
|
|select from table1 t1 left join table2 t2 on t1.a = t2.
|""".stripMargin
val items = context.buildFromString(sql).suggest(4, 0)
assert(items.map(_.name) == List("load", "select", "include","register","run","train","predict","save","set"))
}
test("select a,b,c from table1 as table1;select aa,bb,cc from table2 as table2;select from table1 t1 left join table2 t2 on t1.a = t2. \\n"){
val sql ="""
|select a,b,c from table1 as table1;
|select aa,bb,cc from table2 as table2;
|select from table1 t1 left join table2 t2 on t1.a = t2.
|
|
|""".stripMargin
val items = context.buildFromString(sql).suggest(5, 0)
assert(items.map(_.name) == List("table1", "table2", "aa", "bb", "cc", "a", "b", "c", "count", "split"))
}
}

0 comments on commit f28df64

Please sign in to comment.