Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

executor: fix load data panic if the data is broken at escape character #30868

Merged
merged 11 commits into from
Jan 18, 2022
Merged

executor: fix load data panic if the data is broken at escape character #30868

Changes from 2 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
15676d6
add log
xiongjiwei Dec 20, 2021
38bd4d1
log
xiongjiwei Dec 21, 2021
c57a8a2
refactor
xiongjiwei Dec 22, 2021
9824ee8
refactor
xiongjiwei Dec 22, 2021
300d8f1
log
xiongjiwei Dec 22, 2021
39b4a1b
remove log
xiongjiwei Jan 12, 2022
be51571
add test
xiongjiwei Jan 12, 2022
30379c3
linter
xiongjiwei Jan 12, 2022
29eac69
address comments
xiongjiwei Jan 14, 2022
aa23b1b
add comments
xiongjiwei Jan 18, 2022
f0073e5
Merge branch 'master' into load-data-log
ti-chi-bot Jan 18, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 27 additions & 3 deletions executor/load_data.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -407,6 +407,9 @@ func (e *LoadDataInfo) getValidData(prevData, curData []byte) ([]byte, []byte) {

func (e *LoadDataInfo) isInQuoter(bs []byte) bool {
inQuoter := false
if e.FieldsInfo.Enclosed == byte(0) {
return false
}
for i := 0; i < len(bs); i++ {
switch bs[i] {
case e.FieldsInfo.Enclosed:
Expand Down Expand Up @@ -461,7 +464,7 @@ func (e *LoadDataInfo) IndexOfTerminator(bs []byte, inQuoter bool) int {
atFieldStart := true
loop:
for i := 0; i < len(bs); i++ {
if atFieldStart && bs[i] == e.FieldsInfo.Enclosed {
if atFieldStart && e.FieldsInfo.Enclosed != byte(0) &&bs[i] == e.FieldsInfo.Enclosed {
inQuoter = !inQuoter
atFieldStart = false
continue
Expand Down Expand Up @@ -536,7 +539,7 @@ func (e *LoadDataInfo) getLine(prevData, curData []byte, ignore bool) ([]byte, [
if ignore {
endIdx = strings.Index(string(hack.String(curData[startingLen:])), e.LinesInfo.Terminated)
} else {
endIdx = e.IndexOfTerminator(curData[startingLen:], inquotor)
endIdx = e.IndexOfTerminator(curData[startingLen:], false)
}
if endIdx != -1 {
nextDataIdx := startingLen + endIdx + terminatedLen
Expand All @@ -557,14 +560,35 @@ func (e *LoadDataInfo) getLine(prevData, curData []byte, ignore bool) ([]byte, [
if ignore {
endIdx = strings.Index(string(hack.String(prevData[startingLen:])), e.LinesInfo.Terminated)
} else {
endIdx = e.IndexOfTerminator(prevData[startingLen:], inquotor)
endIdx = e.IndexOfTerminator(prevData[startingLen:], false)
}
if endIdx >= prevLen {
return prevData[startingLen : startingLen+endIdx], curData[nextDataIdx:], true
}

// terminated symbol in the middle of prevData and curData
lineLen := startingLen + endIdx + terminatedLen
defer func() {
r := recover()
if r != nil {
logutil.BgLogger().Info("load data",
zap.String("Field Terminated", e.FieldsInfo.Terminated),
zap.String("Field Enclosed", string(e.FieldsInfo.Enclosed)),
zap.Bool("Field OptEnclosed", e.FieldsInfo.OptEnclosed),
zap.String("Line Starting", e.LinesInfo.Starting),
zap.String("Line Terminated", e.LinesInfo.Terminated),
zap.Int("endIdx", endIdx),
zap.Int("prevLen", prevLen),
zap.Int("lineLen", lineLen),
zap.Int("nextDataIdx", nextDataIdx),
zap.Bool("inquotor", inquotor),
zap.Bool("ignore", ignore),
zap.Bool("prevData-contains-00bytes", bytes.Contains(prevData, []byte{e.FieldsInfo.Enclosed})),
zap.Bool("curData-contains-00bytes", bytes.Contains(curData, []byte{e.FieldsInfo.Enclosed})),
)
panic(r)
}
}()
return prevData[startingLen : startingLen+endIdx], curData[lineLen-prevLen:], true
}

Expand Down