Cherry-pick of three PRs (vitessio#915)

* fix: check for nil earlier Cherry-pick of vitessio#10887 Signed-off-by: Andres Taylor <andres@planetscale.com> * Update how table uses are reported (vitessio#10598) * refactor: move out plan into its own file * refactor: wrap primitive in a PlanResult struct * test: cleanup tests to move to using assert package * feat: add a tablesUsed field to logstats and use it in the output * feat: wireup the tables used from the planning phase to log stats * test: make sure to keep escapes in json strings * feat: add tables used field to the plan output * feat: add tables used to more plan types * refactor: move logstats to it's own package * chore: updated to newest version of cmp * test: refactor * feat: simplify logstats printing * feat: add tables used for INSERT queries * feat: add ks for dual queries * comment: add deprecation warning to logstats fields * docs: add deprecation warning Co-authored-by: Manan Gupta <manan@planetscale.com> Signed-off-by: Andres Taylor <andres@planetscale.com> * New explain format: VTEXPLAIN (vitessio#10556) * feat: added query logging to safe session * feat: added [explain format=vtexplain] parsing and AST * feat: add planning of vtexplain queries * feat: keep the logging even in the precense of autocommits * feat: use table result instead of warnings, and move the logging to scattercon * feat: log commit to vtexplain logger and added e2e test * feat: vindex query to mark vindexExec in session * test: run only for gen4 planner * feat: vtexplain logs with bindvars replaces, and begin being counted with the real query * feat: add ACTUALLY_RUN_QUERIES as a comment directive to vtexplain and block dml queries if not specified * test: make sure we add the actually_runs_queries directive to inserts we want to inspect * docs: added info to release notes * typo: fix error message * feat: ignore cases on comment directives * fix: CommentDirective now hides internals so that the lower casing of keys can be preserved everywhere * test: improve assertion * chore: clean up accidental change * refactor: clean up so more responsibility for vtexplain ends up in the engine primitive * fix: no need to Unquote multiple times * fix: rename comment directive Co-authored-by: Harshit Gangal <harshit@planetscale.com> Co-authored-by: Manan Gupta <manan@planetscale.com> Signed-off-by: Andres Taylor <andres@planetscale.com> * CI: add private repo access, delete unnecessary test Signed-off-by: deepthi <deepthi@planetscale.com> Co-authored-by: Manan Gupta <manan@planetscale.com> Co-authored-by: Harshit Gangal <harshit@planetscale.com> Co-authored-by: deepthi <deepthi@planetscale.com>
planetscale · Aug 8, 2022 · 10d01b5 · 10d01b5
1 parent 648bef4
commit 10d01b5
Show file tree

Hide file tree

Showing 107 changed files with 18,012 additions and 8,077 deletions.
diff --git a/.github/workflows/cluster_initial_sharding_multi.yml b/.github/workflows/cluster_initial_sharding_multi.yml
diff --git a/.github/workflows/docker_test_cluster_10.yml b/.github/workflows/docker_test_cluster_10.yml
@@ -1,31 +1,30 @@
 name: docker_test_cluster_10
-on: [push, pull_request]
+on: [push]
 jobs:
 
   build:
     name: Docker Test Cluster 10
     runs-on: ubuntu-latest
 
     steps:
+      - name: Configure git private repo access
+        env:
+          GITHUB_TOKEN: ${{ secrets.PLANETSCALE_ACTIONS_BOT_TOKEN }}
+        run: |
+          git config --global --add url."https://${GITHUB_TOKEN}@github.com/".insteadOf "https://github.com/"
 
-    - name: Set up Go
-      uses: actions/setup-go@v2
-      with:
-        go-version: 1.18.4
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.18.4
 
-    - name: Tune the OS
-      run: |
-        echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range
+      - name: Tune the OS
+        run: |
+          echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range
 
-    # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185
-    - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file
-      run: |
-        echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
-    # DON'T FORGET TO REMOVE CODE ABOVE WHEN ISSUE IS ADRESSED!
+      - name: Check out code
+        uses: actions/checkout@v2
 
-    - name: Check out code
-      uses: actions/checkout@v2
-
-    - name: Run tests which require docker - 1
-      run: |
-        go run test.go -docker=true --follow -shard 10
+      - name: Run tests which require docker - 1
+        run: |
+          go run test.go -docker=true --follow -shard 10
diff --git a/.github/workflows/docker_test_cluster_25.yml b/.github/workflows/docker_test_cluster_25.yml
@@ -1,31 +1,30 @@
 name: docker_test_cluster_25
-on: [push, pull_request]
+on: [push]
 jobs:
 
   build:
     name: Docker Test Cluster 25
     runs-on: ubuntu-latest
 
     steps:
+      - name: Configure git private repo access
+        env:
+          GITHUB_TOKEN: ${{ secrets.PLANETSCALE_ACTIONS_BOT_TOKEN }}
+        run: |
+          git config --global --add url."https://${GITHUB_TOKEN}@github.com/".insteadOf "https://github.com/"
 
-    - name: Set up Go
-      uses: actions/setup-go@v2
-      with:
-        go-version: 1.18.4
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.18.4
 
-    - name: Tune the OS
-      run: |
-        echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range
+      - name: Tune the OS
+        run: |
+          echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range
 
-    # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185
-    - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file
-      run: |
-        echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
-    # DON'T FORGET TO REMOVE CODE ABOVE WHEN ISSUE IS ADRESSED!
+      - name: Check out code
+        uses: actions/checkout@v2
 
-    - name: Check out code
-      uses: actions/checkout@v2
-
-    - name: Run tests which require docker - 2
-      run: |
-        go run test.go -docker=true --follow -shard 25
+      - name: Run tests which require docker - 2
+        run: |
+          go run test.go -docker=true --follow -shard 25
diff --git a/.github/workflows/ensure_bootstrap_updated.yml b/.github/workflows/ensure_bootstrap_updated.yml
@@ -1,32 +1,31 @@
 name: ensure_bootstrap_version
-on: [pull_request]
+on: [push]
 jobs:
 
   build:
     name: Check Bootstrap Updated
     runs-on: ubuntu-latest
     steps:
+      - name: Configure git private repo access
+        env:
+          GITHUB_TOKEN: ${{ secrets.PLANETSCALE_ACTIONS_BOT_TOKEN }}
+        run: |
+          git config --global --add url."https://${GITHUB_TOKEN}@github.com/".insteadOf "https://github.com/"
 
-    - name: Set up Go
-      uses: actions/setup-go@v2
-      with:
-        go-version: 1.18.4
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.18.4
 
-    - name: Tune the OS
-      run: |
-        echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range
+      - name: Tune the OS
+        run: |
+          echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range
 
-    # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185
-    - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file
-      run: |
-        echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
-    # DON'T FORGET TO REMOVE CODE ABOVE WHEN ISSUE IS ADRESSED!
+      - name: Check out code
+        uses: actions/checkout@v2
 
-    - name: Check out code
-      uses: actions/checkout@v2
-
-    - name: run ensure_bootstrap_version
-      run: |
-        make ensure_bootstrap_version
-        git status
-        test -z "$(git diff-index --name-only HEAD --)" || exit 1
+      - name: run ensure_bootstrap_version
+        run: |
+          make ensure_bootstrap_version
+          git status
+          test -z "$(git diff-index --name-only HEAD --)" || exit 1
diff --git a/go/test/endtoend/vtgate/queries/vtexplain/main_test.go b/go/test/endtoend/vtgate/queries/vtexplain/main_test.go
@@ -0,0 +1,155 @@
+/*
+Copyright 2022 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package vtexplain
+
+import (
+	"context"
+	_ "embed"
+	"flag"
+	"os"
+	"testing"
+
+	"vitess.io/vitess/go/vt/vtgate/planbuilder"
+
+	"github.com/stretchr/testify/require"
+
+	"vitess.io/vitess/go/mysql"
+	"vitess.io/vitess/go/test/endtoend/cluster"
+	"vitess.io/vitess/go/test/endtoend/utils"
+)
+
+var (
+	clusterInstance *cluster.LocalProcessCluster
+	vtParams        mysql.ConnParams
+	shardedKs       = "ks"
+
+	shardedKsShards = []string{"-40", "40-80", "80-c0", "c0-"}
+	Cell            = "test"
+	//go:embed schema.sql
+	shardedSchemaSQL string
+
+	//go:embed vschema.json
+	shardedVSchema string
+)
+
+func TestMain(m *testing.M) {
+	defer cluster.PanicHandler(nil)
+	flag.Parse()
+
+	exitCode := func() int {
+		clusterInstance = cluster.NewCluster(Cell, "localhost")
+		defer clusterInstance.Teardown()
+
+		// Start topo server
+		err := clusterInstance.StartTopo()
+		if err != nil {
+			return 1
+		}
+
+		// Start keyspace
+		sKs := &cluster.Keyspace{
+			Name:      shardedKs,
+			SchemaSQL: shardedSchemaSQL,
+			VSchema:   shardedVSchema,
+		}
+
+		err = clusterInstance.StartKeyspace(*sKs, shardedKsShards, 0, false)
+		if err != nil {
+			return 1
+		}
+
+		// Start vtgate
+		clusterInstance.VtGatePlannerVersion = planbuilder.Gen4
+		err = clusterInstance.StartVtgate()
+		if err != nil {
+			return 1
+		}
+
+		vtParams = mysql.ConnParams{
+			Host: clusterInstance.Hostname,
+			Port: clusterInstance.VtgateMySQLPort,
+		}
+
+		return m.Run()
+	}()
+	os.Exit(exitCode)
+}
+
+func TestVtGateVtExplain(t *testing.T) {
+	conn, err := mysql.Connect(context.Background(), &vtParams)
+	require.NoError(t, err)
+	defer conn.Close()
+
+	utils.AssertContainsError(t, conn,
+		`explain format=vtexplain insert into user (id,lookup,lookup_unique) values (4,'apa','foo'),(5,'apa','bar'),(6,'monkey','nobar')`,
+		"vtexplain will actually run queries")
+
+	expected := `[[INT32(0) VARCHAR("ks") VARCHAR("-40") VARCHAR("begin")]` +
+		` [INT32(0) VARCHAR("ks") VARCHAR("-40") VARCHAR("insert into lookup(lookup, id, keyspace_id) values ('apa', 1, '\x16k@\xb4J\xbaK\xd6'), ('apa', 2, '\x06\xe7\xea\\\"Βp\x8f') on duplicate key update lookup = values(lookup), id = values(id), keyspace_id = values(keyspace_id)")]` +
+		` [INT32(1) VARCHAR("ks") VARCHAR("40-80") VARCHAR("begin")]` +
+		` [INT32(1) VARCHAR("ks") VARCHAR("40-80") VARCHAR("insert into lookup(lookup, id, keyspace_id) values ('monkey', 3, 'N\xb1\x90ɢ\xfa\x16\x9c') on duplicate key update lookup = values(lookup), id = values(id), keyspace_id = values(keyspace_id)")]` +
+		` [INT32(2) VARCHAR("ks") VARCHAR("-40") VARCHAR("commit")]` +
+		` [INT32(3) VARCHAR("ks") VARCHAR("40-80") VARCHAR("commit")]` +
+		` [INT32(4) VARCHAR("ks") VARCHAR("40-80") VARCHAR("begin")]` +
+		` [INT32(4) VARCHAR("ks") VARCHAR("40-80") VARCHAR("insert into lookup_unique(lookup_unique, keyspace_id) values ('monkey', 'N\xb1\x90ɢ\xfa\x16\x9c')")]` +
+		` [INT32(5) VARCHAR("ks") VARCHAR("-40") VARCHAR("begin")]` +
+		` [INT32(5) VARCHAR("ks") VARCHAR("-40") VARCHAR("insert into lookup_unique(lookup_unique, keyspace_id) values ('apa', '\x16k@\xb4J\xbaK\xd6'), ('bandar', '\x06\xe7\xea\\\"Βp\x8f')")]` +
+		` [INT32(6) VARCHAR("ks") VARCHAR("40-80") VARCHAR("commit")]` +
+		` [INT32(7) VARCHAR("ks") VARCHAR("-40") VARCHAR("commit")]` +
+		` [INT32(8) VARCHAR("ks") VARCHAR("40-80") VARCHAR("begin")]` +
+		` [INT32(8) VARCHAR("ks") VARCHAR("40-80") VARCHAR("insert into ` + "`user`" + `(id, lookup, lookup_unique) values (3, 'monkey', 'monkey')")]` +
+		` [INT32(9) VARCHAR("ks") VARCHAR("-40") VARCHAR("begin")]` +
+		` [INT32(9) VARCHAR("ks") VARCHAR("-40") VARCHAR("insert into ` + "`user`" + `(id, lookup, lookup_unique) values (1, 'apa', 'apa'), (2, 'apa', 'bandar')")]]`
+	utils.AssertMatchesNoOrder(t, conn, `explain /*vt+ EXECUTE_DML_QUERIES */ format=vtexplain insert into user (id,lookup,lookup_unique) values (1,'apa','apa'),(2,'apa','bandar'),(3,'monkey','monkey')`, expected)
+
+	expected = `[[INT32(0) VARCHAR("ks") VARCHAR("-40") VARCHAR("select lookup, keyspace_id from lookup where lookup in ('apa')")]` +
+		` [INT32(1) VARCHAR("ks") VARCHAR("-40") VARCHAR("select id from ` + "`user`" + ` where lookup = 'apa'")]]`
+	for _, mode := range []string{"oltp", "olap"} {
+		t.Run(mode, func(t *testing.T) {
+			utils.Exec(t, conn, "set workload = "+mode)
+			utils.AssertMatches(t, conn, `explain format=vtexplain select id from user where lookup = "apa"`, expected)
+		})
+	}
+
+	// transaction explicitly started to no commit in the end.
+	utils.Exec(t, conn, "begin")
+	expected = `[[INT32(0) VARCHAR("ks") VARCHAR("-40") VARCHAR("begin")]` +
+		` [INT32(0) VARCHAR("ks") VARCHAR("-40") VARCHAR("insert into lookup(lookup, id, keyspace_id) values ('apa', 4, '\xd2\xfd\x88g\xd5\\r-\xfe'), ('apa', 5, 'p\xbb\x02<\x81\f\xa8z') on duplicate key update lookup = values(lookup), id = values(id), keyspace_id = values(keyspace_id)")]` +
+		` [INT32(1) VARCHAR("ks") VARCHAR("40-80") VARCHAR("begin")]` +
+		` [INT32(1) VARCHAR("ks") VARCHAR("40-80") VARCHAR("insert into lookup(lookup, id, keyspace_id) values ('monkey', 6, '\xf0\x98H\\n\xc4ľq') on duplicate key update lookup = values(lookup), id = values(id), keyspace_id = values(keyspace_id)")]` +
+		` [INT32(2) VARCHAR("ks") VARCHAR("-40") VARCHAR("commit")]` +
+		` [INT32(3) VARCHAR("ks") VARCHAR("40-80") VARCHAR("commit")]` +
+		` [INT32(4) VARCHAR("ks") VARCHAR("-40") VARCHAR("begin")]` +
+		` [INT32(4) VARCHAR("ks") VARCHAR("-40") VARCHAR("insert into lookup_unique(lookup_unique, keyspace_id) values ('foo', '\xd2\xfd\x88g\xd5\\r-\xfe')")]` +
+		` [INT32(5) VARCHAR("ks") VARCHAR("80-c0") VARCHAR("begin")]` +
+		` [INT32(5) VARCHAR("ks") VARCHAR("80-c0") VARCHAR("insert into lookup_unique(lookup_unique, keyspace_id) values ('bar', 'p\xbb\x02<\x81\f\xa8z')")]` +
+		` [INT32(6) VARCHAR("ks") VARCHAR("c0-") VARCHAR("begin")]` +
+		` [INT32(6) VARCHAR("ks") VARCHAR("c0-") VARCHAR("insert into lookup_unique(lookup_unique, keyspace_id) values ('nobar', '\xf0\x98H\\n\xc4ľq')")]` +
+		` [INT32(7) VARCHAR("ks") VARCHAR("-40") VARCHAR("commit")]` +
+		` [INT32(8) VARCHAR("ks") VARCHAR("80-c0") VARCHAR("commit")]` +
+		` [INT32(9) VARCHAR("ks") VARCHAR("c0-") VARCHAR("commit")]` +
+		` [INT32(10) VARCHAR("ks") VARCHAR("40-80") VARCHAR("begin")]` +
+		` [INT32(10) VARCHAR("ks") VARCHAR("40-80") VARCHAR("insert into ` + "`user`" + `(id, lookup, lookup_unique) values (5, 'apa', 'bar')")]` +
+		` [INT32(11) VARCHAR("ks") VARCHAR("c0-") VARCHAR("begin")]` +
+		` [INT32(11) VARCHAR("ks") VARCHAR("c0-") VARCHAR("insert into ` + "`user`" + `(id, lookup, lookup_unique) values (4, 'apa', 'foo'), (6, 'monkey', 'nobar')")]]`
+
+	utils.AssertMatchesNoOrder(t, conn,
+		`explain /*vt+ EXECUTE_DML_QUERIES */ format=vtexplain insert into user (id,lookup,lookup_unique) values (4,'apa','foo'),(5,'apa','bar'),(6,'monkey','nobar')`,
+		expected)
+
+	utils.Exec(t, conn, "rollback")
+}
diff --git a/go/test/endtoend/vtgate/queries/vtexplain/schema.sql b/go/test/endtoend/vtgate/queries/vtexplain/schema.sql
@@ -0,0 +1,22 @@
+create table user
+(
+    id            bigint,
+    lookup        varchar(128),
+    lookup_unique varchar(128),
+    primary key (id)
+) Engine = InnoDB;
+
+create table lookup
+(
+    lookup      varchar(128),
+    id          bigint,
+    keyspace_id varbinary(100),
+    primary key (id)
+) Engine = InnoDB;
+
+create table lookup_unique
+(
+    lookup_unique varchar(128),
+    keyspace_id   varbinary(100),
+    primary key (lookup_unique)
+) Engine = InnoDB;