Merge branch 'master' into vmg/gogoproto

Signed-off-by: Vicent Marti <vmg@strn.cat>

.github/workflows/check_make_sizegen.yml

@@ -0,0 +1,35 @@
+name: check_make_sizegen
+on: [push, pull_request]
+jobs:
+
+  build:
+    name: Check Make Sizegen
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Set up Go
+        uses: actions/setup-go@v1
+        with:
+          go-version: 1.15
+
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Get dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget
+          sudo service mysql stop
+          sudo service etcd stop
+          sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+          sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+          go mod download
+
+      - name: Run make minimaltools
+        run: |
+          make minimaltools
+
+      - name: check_make_sizegen
+        run: |
+          tools/check_make_sizegen.sh
+

go/cache/ristretto/cache.go

@@ -337,6 +337,10 @@ loop:
 	for {
 		select {
 		case i := <-c.setBuf:
+			if i.wg != nil {
+				i.wg.Done()
+				continue
+			}
 			if i.flag != itemUpdate {
 				// In itemUpdate, the value is already set in the store.  So, no need to call
 				// onEvict here.

go/vt/servenv/buildinfo.go

@@ -23,6 +23,13 @@ import (
 	"strconv"
 	"time"
 
+	"vitess.io/vitess/go/vt/proto/vtrpc"
+	"vitess.io/vitess/go/vt/vterrors"
+
+	"vitess.io/vitess/go/vt/log"
+
+	"vitess.io/vitess/go/vt/sqlparser"
+
 	"vitess.io/vitess/go/stats"
 )
 
@@ -104,7 +111,13 @@ func init() {
 		goArch:             runtime.GOARCH,
 		version:            versionName,
 	}
-
+	var convVersion string
+	convVersion, err = convertMySQLVersionToCommentVersion(AppVersion.MySQLVersion())
+	if err != nil {
+		log.Error(err)
+	} else {
+		sqlparser.MySQLVersion = convVersion
+	}
 	stats.NewString("BuildHost").Set(AppVersion.buildHost)
 	stats.NewString("BuildUser").Set(AppVersion.buildUser)
 	stats.NewGauge("BuildTimestamp", "build timestamp").Set(AppVersion.buildTime)
@@ -126,3 +139,41 @@ func init() {
 	}
 	stats.NewGaugesWithMultiLabels("BuildInformation", "build information exposed via label", buildLabels).Set(buildValues, 1)
 }
+
+// convertMySQLVersionToCommentVersion converts the MySQL version into comment version format.
+func convertMySQLVersionToCommentVersion(version string) (string, error) {
+	var res = make([]int, 3)
+	idx := 0
+	val := ""
+	for _, c := range version {
+		if c <= '9' && c >= '0' {
+			val += string(c)
+		} else if c == '.' {
+			v, err := strconv.Atoi(val)
+			if err != nil {
+				return "", err
+			}
+			val = ""
+			res[idx] = v
+			idx++
+			if idx == 3 {
+				break
+			}
+		} else {
+			break
+		}
+	}
+	if val != "" {
+		v, err := strconv.Atoi(val)
+		if err != nil {
+			return "", err
+		}
+		res[idx] = v
+		idx++
+	}
+	if idx == 0 {
+		return "", vterrors.Errorf(vtrpc.Code_INVALID_ARGUMENT, "MySQL version not correctly setup - %s.", version)
+	}
+
+	return fmt.Sprintf("%01d%02d%02d", res[0], res[1], res[2]), nil
+}

go/vt/servenv/buildinfo_test.go

@@ -20,6 +20,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/require"
+
 	"github.com/stretchr/testify/assert"
 )
 
@@ -50,3 +52,41 @@ func TestVersionString(t *testing.T) {
 	MySQLServerVersion = &newVersion
 	assert.Equal(t, newVersion, v.MySQLVersion())
 }
+
+func TestConvertMySQLVersion(t *testing.T) {
+	testcases := []struct {
+		version        string
+		commentVersion string
+		error          string
+	}{{
+		version:        "5.7.9",
+		commentVersion: "50709",
+	}, {
+		version:        "0008.08.9",
+		commentVersion: "80809",
+	}, {
+		version:        "5.7.9, Vitess - 10.0.1",
+		commentVersion: "50709",
+	}, {
+		version:        "8.1 Vitess - 10.0.1",
+		commentVersion: "80100",
+	}, {
+		version: "Vitess - 10.0.1",
+		error:   "MySQL version not correctly setup - Vitess - 10.0.1.",
+	}, {
+		version:        "5.7.9.22",
+		commentVersion: "50709",
+	}}
+
+	for _, tcase := range testcases {
+		t.Run(tcase.version, func(t *testing.T) {
+			output, err := convertMySQLVersionToCommentVersion(tcase.version)
+			if tcase.error != "" {
+				require.EqualError(t, err, tcase.error)
+			} else {
+				require.NoError(t, err)
+				require.Equal(t, tcase.commentVersion, output)
+			}
+		})
+	}
+}

go/vt/sqlparser/comments.go

@@ -181,6 +181,9 @@ func ExtractMysqlComment(sql string) (string, string) {
 	if endOfVersionIndex < 0 {
 		return "", ""
 	}
+	if endOfVersionIndex < 5 {
+		endOfVersionIndex = 0
+	}
 	version := sql[0:endOfVersionIndex]
 	innerSQL := strings.TrimFunc(sql[endOfVersionIndex:], unicode.IsSpace)
 

go/vt/sqlparser/parse_test.go

@@ -1757,6 +1757,9 @@ var (
 		input:      "create database test_db character set * unparsable",
 		output:     "create database test_db",
 		partialDDL: true,
+	}, {
+		input:  "CREATE DATABASE /*!32312 IF NOT EXISTS*/ `mysql` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci */ /*!80016 DEFAULT ENCRYPTION='N' */;",
+		output: "create database if not exists mysql default character set utf8mb4 collate utf8mb4_0900_ai_ci",
 	}, {
 		input: "drop database test_db",
 	}, {

go/vt/sqlparser/parser.go

@@ -35,6 +35,9 @@ var parserPool = sync.Pool{}
 // zeroParser is a zero-initialized parser to help reinitialize the parser for pooling.
 var zeroParser = *(yyNewParser().(*yyParserImpl))
 
+// MySQLVersion is the version of MySQL that the parser would emulate
+var MySQLVersion string = "50709"
+
 // yyParsePooled is a wrapper around yyParse that pools the parser objects. There isn't a
 // particularly good reason to use yyParse directly, since it immediately discards its parser.  What
 // would be ideal down the line is to actually pool the stacks themselves rather than the parser

go/vt/sqlparser/token.go

@@ -1031,8 +1031,14 @@ func (tkn *Tokenizer) scanMySQLSpecificComment() (int, []byte) {
 		}
 		tkn.consumeNext(buffer)
 	}
-	_, sql := ExtractMysqlComment(buffer.String())
-	tkn.specialComment = NewStringTokenizer(sql)
+
+	commentVersion, sql := ExtractMysqlComment(buffer.String())
+
+	if MySQLVersion >= commentVersion {
+		// Only add the special comment to the tokenizer if the version of MySQL is higher or equal to the comment version
+		tkn.specialComment = NewStringTokenizer(sql)
+	}
+
 	return tkn.Scan()
 }
 

go/vt/sqlparser/token_test.go

@@ -19,6 +19,8 @@ package sqlparser
 import (
 	"fmt"
 	"testing"
+
+	"github.com/stretchr/testify/require"
 )
 
 func TestLiteralID(t *testing.T) {
@@ -75,9 +77,8 @@ func TestLiteralID(t *testing.T) {
 	for _, tcase := range testcases {
 		tkn := NewStringTokenizer(tcase.in)
 		id, out := tkn.Scan()
-		if tcase.id != id || string(out) != tcase.out {
-			t.Errorf("Scan(%s): %d, %s, want %d, %s", tcase.in, id, out, tcase.id, tcase.out)
-		}
+		require.Equal(t, tcase.id, id)
+		require.Equal(t, tcase.out, string(out))
 	}
 }
 
@@ -146,10 +147,11 @@ func TestString(t *testing.T) {
 	}}
 
 	for _, tcase := range testcases {
-		id, got := NewStringTokenizer(tcase.in).Scan()
-		if tcase.id != id || string(got) != tcase.want {
-			t.Errorf("Scan(%q) = (%s, %q), want (%s, %q)", tcase.in, tokenName(id), got, tokenName(tcase.id), tcase.want)
-		}
+		t.Run(tcase.in, func(t *testing.T) {
+			id, got := NewStringTokenizer(tcase.in).Scan()
+			require.Equal(t, tcase.id, id, "Scan(%q) = (%s), want (%s)", tcase.in, tokenName(id), tokenName(tcase.id))
+			require.Equal(t, tcase.want, string(got))
+		})
 	}
 }
 
@@ -205,3 +207,61 @@ func TestSplitStatement(t *testing.T) {
 		}
 	}
 }
+
+func TestVersion(t *testing.T) {
+	testcases := []struct {
+		version string
+		in      string
+		id      []int
+	}{{
+		version: "50709",
+		in:      "/*!80102 SELECT*/ FROM IN EXISTS",
+		id:      []int{FROM, IN, EXISTS, 0},
+	}, {
+		version: "80101",
+		in:      "/*!80102 SELECT*/ FROM IN EXISTS",
+		id:      []int{FROM, IN, EXISTS, 0},
+	}, {
+		version: "80201",
+		in:      "/*!80102 SELECT*/ FROM IN EXISTS",
+		id:      []int{SELECT, FROM, IN, EXISTS, 0},
+	}, {
+		version: "80102",
+		in:      "/*!80102 SELECT*/ FROM IN EXISTS",
+		id:      []int{SELECT, FROM, IN, EXISTS, 0},
+	}}
+
+	for _, tcase := range testcases {
+		t.Run(tcase.version+"_"+tcase.in, func(t *testing.T) {
+			MySQLVersion = tcase.version
+			tok := NewStringTokenizer(tcase.in)
+			for _, expectedID := range tcase.id {
+				id, _ := tok.Scan()
+				require.Equal(t, expectedID, id)
+			}
+		})
+	}
+}
+
+func TestExtractMySQLComment(t *testing.T) {
+	testcases := []struct {
+		comment string
+		version string
+	}{{
+		comment: "/*!50108 SELECT * FROM */",
+		version: "50108",
+	}, {
+		comment: "/*!5018 SELECT * FROM */",
+		version: "",
+	}, {
+		comment: "/*!SELECT * FROM */",
+		version: "",
+	}}
+
+	for _, tcase := range testcases {
+		t.Run(tcase.version, func(t *testing.T) {
+			output, _ := ExtractMysqlComment(tcase.comment)
+			require.Equal(t, tcase.version, output)
+		})
+	}
+}