Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

expression: Improve the compatibility of str_to_date (#25386) #25767

Merged
merged 3 commits into from
Jun 28, 2021
Merged

expression: Improve the compatibility of str_to_date (#25386) #25767

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5ac3d52
cherry pick #25386 to release-5.0
JaySon-Huang Jun 25, 2021
2a33192
Fix lint
JaySon-Huang Jun 25, 2021
004a46c
Merge branch 'release-5.0' into release-5.0-8434069c5f63
ti-chi-bot Jun 28, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 44 additions & 26 deletions expression/builtin_time_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1397,41 +1397,52 @@ func (s *testEvaluatorSuite) TestUTCDate(c *C) {
}

func (s *testEvaluatorSuite) TestStrToDate(c *C) {
// If you want to add test cases for `strToDate` but not the builtin function,
// adding cases in `types.format_test.go` `TestStrToDate` maybe more clear and easier
tests := []struct {
Date string
Format string
Success bool
Kind byte
Expect time.Time
}{
{"10/28/2011 9:46:29 pm", "%m/%d/%Y %l:%i:%s %p", true, time.Date(2011, 10, 28, 21, 46, 29, 0, time.Local)},
{"10/28/2011 9:46:29 Pm", "%m/%d/%Y %l:%i:%s %p", true, time.Date(2011, 10, 28, 21, 46, 29, 0, time.Local)},
{"2011/10/28 9:46:29 am", "%Y/%m/%d %l:%i:%s %p", true, time.Date(2011, 10, 28, 9, 46, 29, 0, time.Local)},
{"20161122165022", `%Y%m%d%H%i%s`, true, time.Date(2016, 11, 22, 16, 50, 22, 0, time.Local)},
{"2016 11 22 16 50 22", `%Y%m%d%H%i%s`, true, time.Date(2016, 11, 22, 16, 50, 22, 0, time.Local)},
{"16-50-22 2016 11 22", `%H-%i-%s%Y%m%d`, true, time.Date(2016, 11, 22, 16, 50, 22, 0, time.Local)},
{"16-50 2016 11 22", `%H-%i-%s%Y%m%d`, false, time.Time{}},
{"15-01-2001 1:59:58.999", "%d-%m-%Y %I:%i:%s.%f", true, time.Date(2001, 1, 15, 1, 59, 58, 999000000, time.Local)},
{"15-01-2001 1:59:58.1", "%d-%m-%Y %H:%i:%s.%f", true, time.Date(2001, 1, 15, 1, 59, 58, 100000000, time.Local)},
{"15-01-2001 1:59:58.", "%d-%m-%Y %H:%i:%s.%f", true, time.Date(2001, 1, 15, 1, 59, 58, 000000000, time.Local)},
{"15-01-2001 1:9:8.999", "%d-%m-%Y %H:%i:%s.%f", true, time.Date(2001, 1, 15, 1, 9, 8, 999000000, time.Local)},
{"15-01-2001 1:9:8.999", "%d-%m-%Y %H:%i:%S.%f", true, time.Date(2001, 1, 15, 1, 9, 8, 999000000, time.Local)},
{"2003-01-02 10:11:12 PM", "%Y-%m-%d %H:%i:%S %p", false, time.Time{}},
{"10:20:10AM", "%H:%i:%S%p", false, time.Time{}},
{"10/28/2011 9:46:29 pm", "%m/%d/%Y %l:%i:%s %p", true, types.KindMysqlTime, time.Date(2011, 10, 28, 21, 46, 29, 0, time.Local)},
{"10/28/2011 9:46:29 Pm", "%m/%d/%Y %l:%i:%s %p", true, types.KindMysqlTime, time.Date(2011, 10, 28, 21, 46, 29, 0, time.Local)},
{"2011/10/28 9:46:29 am", "%Y/%m/%d %l:%i:%s %p", true, types.KindMysqlTime, time.Date(2011, 10, 28, 9, 46, 29, 0, time.Local)},
{"20161122165022", `%Y%m%d%H%i%s`, true, types.KindMysqlTime, time.Date(2016, 11, 22, 16, 50, 22, 0, time.Local)},
{"2016 11 22 16 50 22", `%Y%m%d%H%i%s`, true, types.KindMysqlTime, time.Date(2016, 11, 22, 16, 50, 22, 0, time.Local)},
{"16-50-22 2016 11 22", `%H-%i-%s%Y%m%d`, true, types.KindMysqlTime, time.Date(2016, 11, 22, 16, 50, 22, 0, time.Local)},
{"16-50 2016 11 22", `%H-%i-%s%Y%m%d`, false, types.KindMysqlTime, time.Time{}},
{"15-01-2001 1:59:58.999", "%d-%m-%Y %I:%i:%s.%f", true, types.KindMysqlTime, time.Date(2001, 1, 15, 1, 59, 58, 999000000, time.Local)},
{"15-01-2001 1:59:58.1", "%d-%m-%Y %H:%i:%s.%f", true, types.KindMysqlTime, time.Date(2001, 1, 15, 1, 59, 58, 100000000, time.Local)},
{"15-01-2001 1:59:58.", "%d-%m-%Y %H:%i:%s.%f", true, types.KindMysqlTime, time.Date(2001, 1, 15, 1, 59, 58, 000000000, time.Local)},
{"15-01-2001 1:9:8.999", "%d-%m-%Y %H:%i:%s.%f", true, types.KindMysqlTime, time.Date(2001, 1, 15, 1, 9, 8, 999000000, time.Local)},
{"15-01-2001 1:9:8.999", "%d-%m-%Y %H:%i:%S.%f", true, types.KindMysqlTime, time.Date(2001, 1, 15, 1, 9, 8, 999000000, time.Local)},
{"2003-01-02 10:11:12 PM", "%Y-%m-%d %H:%i:%S %p", false, types.KindMysqlTime, time.Time{}},
{"10:20:10AM", "%H:%i:%S%p", false, types.KindMysqlTime, time.Time{}},
// test %@(skip alpha), %#(skip number), %.(skip punct)
{"2020-10-10ABCD", "%Y-%m-%d%@", true, time.Date(2020, 10, 10, 0, 0, 0, 0, time.Local)},
{"2020-10-101234", "%Y-%m-%d%#", true, time.Date(2020, 10, 10, 0, 0, 0, 0, time.Local)},
{"2020-10-10....", "%Y-%m-%d%.", true, time.Date(2020, 10, 10, 0, 0, 0, 0, time.Local)},
{"2020-10-10.1", "%Y-%m-%d%.%#%@", true, time.Date(2020, 10, 10, 0, 0, 0, 0, time.Local)},
{"abcd2020-10-10.1", "%@%Y-%m-%d%.%#%@", true, time.Date(2020, 10, 10, 0, 0, 0, 0, time.Local)},
{"abcd-2020-10-10.1", "%@-%Y-%m-%d%.%#%@", true, time.Date(2020, 10, 10, 0, 0, 0, 0, time.Local)},
{"2020-10-10", "%Y-%m-%d%@", true, time.Date(2020, 10, 10, 0, 0, 0, 0, time.Local)},
{"2020-10-10abcde123abcdef", "%Y-%m-%d%@%#", true, time.Date(2020, 10, 10, 0, 0, 0, 0, time.Local)},
{"2020-10-10ABCD", "%Y-%m-%d%@", true, types.KindMysqlTime, time.Date(2020, 10, 10, 0, 0, 0, 0, time.Local)},
{"2020-10-101234", "%Y-%m-%d%#", true, types.KindMysqlTime, time.Date(2020, 10, 10, 0, 0, 0, 0, time.Local)},
{"2020-10-10....", "%Y-%m-%d%.", true, types.KindMysqlTime, time.Date(2020, 10, 10, 0, 0, 0, 0, time.Local)},
{"2020-10-10.1", "%Y-%m-%d%.%#%@", true, types.KindMysqlTime, time.Date(2020, 10, 10, 0, 0, 0, 0, time.Local)},
{"abcd2020-10-10.1", "%@%Y-%m-%d%.%#%@", true, types.KindMysqlTime, time.Date(2020, 10, 10, 0, 0, 0, 0, time.Local)},
{"abcd-2020-10-10.1", "%@-%Y-%m-%d%.%#%@", true, types.KindMysqlTime, time.Date(2020, 10, 10, 0, 0, 0, 0, time.Local)},
{"2020-10-10", "%Y-%m-%d%@", true, types.KindMysqlTime, time.Date(2020, 10, 10, 0, 0, 0, 0, time.Local)},
{"2020-10-10abcde123abcdef", "%Y-%m-%d%@%#", true, types.KindMysqlTime, time.Date(2020, 10, 10, 0, 0, 0, 0, time.Local)},
// some input for '%r'
{"12:3:56pm 13/05/2019", "%r %d/%c/%Y", true, types.KindMysqlTime, time.Date(2019, 5, 13, 12, 3, 56, 0, time.Local)},
{"11:13:56 am", "%r", true, types.KindMysqlDuration, time.Date(0, 0, 0, 11, 13, 56, 0, time.Local)},
// some input for '%T'
{"12:13:56 13/05/2019", "%T %d/%c/%Y", true, types.KindMysqlTime, time.Date(2019, 5, 13, 12, 13, 56, 0, time.Local)},
{"19:3:56 13/05/2019", "%T %d/%c/%Y", true, types.KindMysqlTime, time.Date(2019, 5, 13, 19, 3, 56, 0, time.Local)},
{"21:13:24", "%T", true, types.KindMysqlDuration, time.Date(0, 0, 0, 21, 13, 24, 0, time.Local)},
}

fc := funcs[ast.StrToDate]
for _, test := range tests {
date := types.NewStringDatum(test.Date)
format := types.NewStringDatum(test.Format)
c.Logf("input: %s, format: %s", test.Date, test.Format)
f, err := fc.getFunction(s.ctx, s.datumsToConstants([]types.Datum{date, format}))
c.Assert(err, IsNil)
result, err := evalBuiltinFunc(f, chunk.Row{})
Expand All @@ -1441,10 +1452,17 @@ func (s *testEvaluatorSuite) TestStrToDate(c *C) {
c.Assert(result.IsNull(), IsTrue)
continue
}
c.Assert(result.Kind(), Equals, types.KindMysqlTime)
value := result.GetMysqlTime()
t1, _ := value.GoTime(time.Local)
c.Assert(t1, Equals, test.Expect)
c.Assert(result.Kind(), Equals, test.Kind)
switch test.Kind {
case types.KindMysqlTime:
value := result.GetMysqlTime()
t1, _ := value.GoTime(time.Local)
c.Assert(t1, Equals, test.Expect)
case types.KindMysqlDuration:
value := result.GetMysqlDuration()
timeExpect := test.Expect.Sub(time.Date(0, 0, 0, 0, 0, 0, 0, time.Local))
c.Assert(value.Duration, Equals, timeExpect)
}
}
}

Expand Down
54 changes: 49 additions & 5 deletions types/format_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,30 +118,74 @@ func (s *testTimeSuite) TestStrToDate(c *C) {
{`70/10/22`, `%Y/%m/%d`, types.FromDate(1970, 10, 22, 0, 0, 0, 0)},
{`18/10/22`, `%Y/%m/%d`, types.FromDate(2018, 10, 22, 0, 0, 0, 0)},
{`100/10/22`, `%Y/%m/%d`, types.FromDate(100, 10, 22, 0, 0, 0, 0)},
//'%b'/'%M' should be case insensitive
{"31/may/2016 12:34:56.1234", "%d/%b/%Y %H:%i:%S.%f", types.FromDate(2016, 5, 31, 12, 34, 56, 123400)},
{"30/april/2016 12:34:56.", "%d/%M/%Y %H:%i:%s.%f", types.FromDate(2016, 4, 30, 12, 34, 56, 0)},
{"31/mAy/2016 12:34:56.1234", "%d/%b/%Y %H:%i:%S.%f", types.FromDate(2016, 5, 31, 12, 34, 56, 123400)},
{"30/apRil/2016 12:34:56.", "%d/%M/%Y %H:%i:%s.%f", types.FromDate(2016, 4, 30, 12, 34, 56, 0)},
// '%r'
{" 04 :13:56 AM13/05/2019", "%r %d/%c/%Y", types.FromDate(2019, 5, 13, 4, 13, 56, 0)}, //
{"12: 13:56 AM 13/05/2019", "%r%d/%c/%Y", types.FromDate(2019, 5, 13, 0, 13, 56, 0)}, //
{"12:13 :56 pm 13/05/2019", "%r %d/%c/%Y", types.FromDate(2019, 5, 13, 12, 13, 56, 0)}, //
{"12:3: 56pm 13/05/2019", "%r %d/%c/%Y", types.FromDate(2019, 5, 13, 12, 3, 56, 0)}, //
{"11:13:56", "%r", types.FromDate(0, 0, 0, 11, 13, 56, 0)}, // EOF before parsing "AM"/"PM"
{"11:13", "%r", types.FromDate(0, 0, 0, 11, 13, 0, 0)}, // EOF after hh:mm
{"11:", "%r", types.FromDate(0, 0, 0, 11, 0, 0, 0)}, // EOF after hh:
{"11", "%r", types.FromDate(0, 0, 0, 11, 0, 0, 0)}, // EOF after hh:
{"12", "%r", types.FromDate(0, 0, 0, 0, 0, 0, 0)}, // EOF after hh:, and hh=12 -> 0
// '%T'
{" 4 :13:56 13/05/2019", "%T %d/%c/%Y", types.FromDate(2019, 5, 13, 4, 13, 56, 0)},
{"23: 13:56 13/05/2019", "%T%d/%c/%Y", types.FromDate(2019, 5, 13, 23, 13, 56, 0)},
{"12:13 :56 13/05/2019", "%T %d/%c/%Y", types.FromDate(2019, 5, 13, 12, 13, 56, 0)},
{"19:3: 56 13/05/2019", "%T %d/%c/%Y", types.FromDate(2019, 5, 13, 19, 3, 56, 0)},
{"21:13", "%T", types.FromDate(0, 0, 0, 21, 13, 0, 0)}, // EOF after hh:mm
{"21:", "%T", types.FromDate(0, 0, 0, 21, 0, 0, 0)}, // EOF after hh:
// More patterns than input string
{" 2/Jun", "%d/%b/%Y", types.FromDate(0, 6, 2, 0, 0, 0, 0)},
{" liter", "lit era l", types.ZeroCoreTime},
// Feb 29 in leap-year
{"29/Feb/2020 12:34:56.", "%d/%b/%Y %H:%i:%s.%f", types.FromDate(2020, 2, 29, 12, 34, 56, 0)},
// When `AllowInvalidDate` is true, check only that the month is in the range from 1 to 12 and the day is in the range from 1 to 31
{"31/April/2016 12:34:56.", "%d/%M/%Y %H:%i:%s.%f", types.FromDate(2016, 4, 31, 12, 34, 56, 0)}, // April 31th
{"29/Feb/2021 12:34:56.", "%d/%b/%Y %H:%i:%s.%f", types.FromDate(2021, 2, 29, 12, 34, 56, 0)}, // Feb 29 in non-leap-year
{"30/Feb/2016 12:34:56.1234", "%d/%b/%Y %H:%i:%S.%f", types.FromDate(2016, 2, 30, 12, 34, 56, 123400)}, // Feb 30th
}
for i, tt := range tests {
sc.AllowInvalidDate = true
var t types.Time
c.Assert(t.StrToDate(sc, tt.input, tt.format), IsTrue, Commentf("no.%d failed", i))
c.Assert(t.CoreTime(), Equals, tt.expect, Commentf("no.%d failed", i))
c.Assert(t.StrToDate(sc, tt.input, tt.format), IsTrue, Commentf("no.%d failed input=%s format=%s", i, tt.input, tt.format))
c.Assert(t.CoreTime(), Equals, tt.expect, Commentf("no.%d failed input=%s format=%s", i, tt.input, tt.format))
}

errTests := []struct {
input string
format string
}{
{`04/31/2004`, `%m/%d/%Y`},
// invalid days when `AllowInvalidDate` is false
{`04/31/2004`, `%m/%d/%Y`}, // not exists in the real world
{"29/Feb/2021 12:34:56.", "%d/%b/%Y %H:%i:%s.%f"}, // Feb 29 in non-leap-year

{`a09:30:17`, `%h:%i:%s`}, // format mismatch
{`12:43:24`, `%r`}, // no PM or AM followed
{`12:43:24 a`, `%r`}, // followed by incomplete 'AM'/'PM'
{`23:60:12`, `%T`}, // invalid minute
{`18`, `%l`},
{`00:21:22 AM`, `%h:%i:%s %p`},
{`100/10/22`, `%y/%m/%d`},
{"2010-11-12 11 am", `%Y-%m-%d %H %p`},
{"2010-11-12 13 am", `%Y-%m-%d %h %p`},
{"2010-11-12 0 am", `%Y-%m-%d %h %p`},
// MySQL accept `SEPTEMB` as `SEPTEMBER`, but we don't want this "feature" in TiDB
// unless we have to.
{"15 SEPTEMB 2001", "%d %M %Y"},
// '%r'
{"13:13:56 AM13/5/2019", "%r"}, // hh = 13 with am is invalid
{"00:13:56 AM13/05/2019", "%r"}, // hh = 0 with am is invalid
{"00:13:56 pM13/05/2019", "%r"}, // hh = 0 with pm is invalid
{"11:13:56a", "%r"}, // EOF while parsing "AM"/"PM"
}
for i, tt := range errTests {
sc.AllowInvalidDate = false
var t types.Time
c.Assert(t.StrToDate(sc, tt.input, tt.format), IsFalse, Commentf("no.%d failed", i))
c.Assert(t.StrToDate(sc, tt.input, tt.format), IsFalse, Commentf("no.%d failed input=%s format=%s", i, tt.input, tt.format))
}
}
Loading