diff --git a/.github/workflows/check_make_sizegen.yml b/.github/workflows/check_make_sizegen.yml new file mode 100644 index 00000000000..b8eef13acf4 --- /dev/null +++ b/.github/workflows/check_make_sizegen.yml @@ -0,0 +1,35 @@ +name: check_make_sizegen +on: [push, pull_request] +jobs: + + build: + name: Check Make Sizegen + runs-on: ubuntu-latest + steps: + + - name: Set up Go + uses: actions/setup-go@v1 + with: + go-version: 1.15 + + - name: Check out code + uses: actions/checkout@v2 + + - name: Get dependencies + run: | + sudo apt-get update + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + - name: Run make minimaltools + run: | + make minimaltools + + - name: check_make_sizegen + run: | + tools/check_make_sizegen.sh + diff --git a/go/cache/ristretto/cache.go b/go/cache/ristretto/cache.go index 2d2368f1427..bb39a3ff359 100644 --- a/go/cache/ristretto/cache.go +++ b/go/cache/ristretto/cache.go @@ -337,6 +337,10 @@ loop: for { select { case i := <-c.setBuf: + if i.wg != nil { + i.wg.Done() + continue + } if i.flag != itemUpdate { // In itemUpdate, the value is already set in the store. So, no need to call // onEvict here. diff --git a/go/vt/proto/vtadmin/vtadmin.pb.go b/go/vt/proto/vtadmin/vtadmin.pb.go index 4e371cb1ed6..2073e7c0a6c 100644 --- a/go/vt/proto/vtadmin/vtadmin.pb.go +++ b/go/vt/proto/vtadmin/vtadmin.pb.go @@ -971,6 +971,116 @@ func (m *GetTabletsResponse) GetTablets() []*Tablet { return nil } +type VTExplainRequest struct { + Cluster string `protobuf:"bytes,1,opt,name=cluster,proto3" json:"cluster,omitempty"` + Keyspace string `protobuf:"bytes,2,opt,name=keyspace,proto3" json:"keyspace,omitempty"` + Sql string `protobuf:"bytes,3,opt,name=sql,proto3" json:"sql,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *VTExplainRequest) Reset() { *m = VTExplainRequest{} } +func (m *VTExplainRequest) String() string { return proto.CompactTextString(m) } +func (*VTExplainRequest) ProtoMessage() {} +func (*VTExplainRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_609739e22a0a50b3, []int{17} +} +func (m *VTExplainRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *VTExplainRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_VTExplainRequest.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *VTExplainRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_VTExplainRequest.Merge(m, src) +} +func (m *VTExplainRequest) XXX_Size() int { + return m.Size() +} +func (m *VTExplainRequest) XXX_DiscardUnknown() { + xxx_messageInfo_VTExplainRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_VTExplainRequest proto.InternalMessageInfo + +func (m *VTExplainRequest) GetCluster() string { + if m != nil { + return m.Cluster + } + return "" +} + +func (m *VTExplainRequest) GetKeyspace() string { + if m != nil { + return m.Keyspace + } + return "" +} + +func (m *VTExplainRequest) GetSql() string { + if m != nil { + return m.Sql + } + return "" +} + +type VTExplainResponse struct { + Response string `protobuf:"bytes,1,opt,name=response,proto3" json:"response,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *VTExplainResponse) Reset() { *m = VTExplainResponse{} } +func (m *VTExplainResponse) String() string { return proto.CompactTextString(m) } +func (*VTExplainResponse) ProtoMessage() {} +func (*VTExplainResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_609739e22a0a50b3, []int{18} +} +func (m *VTExplainResponse) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *VTExplainResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_VTExplainResponse.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *VTExplainResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_VTExplainResponse.Merge(m, src) +} +func (m *VTExplainResponse) XXX_Size() int { + return m.Size() +} +func (m *VTExplainResponse) XXX_DiscardUnknown() { + xxx_messageInfo_VTExplainResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_VTExplainResponse proto.InternalMessageInfo + +func (m *VTExplainResponse) GetResponse() string { + if m != nil { + return m.Response + } + return "" +} + func init() { proto.RegisterEnum("vtadmin.Tablet_ServingState", Tablet_ServingState_name, Tablet_ServingState_value) proto.RegisterType((*Cluster)(nil), "vtadmin.Cluster") @@ -991,62 +1101,68 @@ func init() { proto.RegisterType((*GetTabletRequest)(nil), "vtadmin.GetTabletRequest") proto.RegisterType((*GetTabletsRequest)(nil), "vtadmin.GetTabletsRequest") proto.RegisterType((*GetTabletsResponse)(nil), "vtadmin.GetTabletsResponse") + proto.RegisterType((*VTExplainRequest)(nil), "vtadmin.VTExplainRequest") + proto.RegisterType((*VTExplainResponse)(nil), "vtadmin.VTExplainResponse") } func init() { proto.RegisterFile("vtadmin.proto", fileDescriptor_609739e22a0a50b3) } var fileDescriptor_609739e22a0a50b3 = []byte{ - // 791 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x55, 0xcd, 0x6e, 0x13, 0x3b, - 0x14, 0xce, 0x24, 0xcd, 0xdf, 0x99, 0xde, 0x26, 0x75, 0x2b, 0xdd, 0xdc, 0x69, 0x9b, 0x1b, 0x59, - 0xf7, 0xa2, 0x80, 0x20, 0x91, 0x02, 0x54, 0x94, 0x0d, 0x2a, 0xa5, 0x8a, 0x4a, 0x45, 0x82, 0x9c, - 0x10, 0x24, 0x36, 0xd5, 0x34, 0x63, 0xd2, 0x51, 0x93, 0x99, 0x10, 0xbb, 0x91, 0xfa, 0x22, 0x88, - 0x2d, 0xef, 0xc0, 0x43, 0xb0, 0x41, 0xe2, 0x11, 0x50, 0x59, 0xf2, 0x12, 0x68, 0x6c, 0x8f, 0x33, - 0x93, 0x94, 0xfe, 0xec, 0xec, 0xf3, 0xf3, 0x9d, 0xef, 0x3b, 0xe7, 0x58, 0x86, 0xbf, 0xa6, 0xdc, - 0x76, 0x46, 0xae, 0x57, 0x1b, 0x4f, 0x7c, 0xee, 0xa3, 0xac, 0xba, 0x5a, 0x7f, 0x73, 0xfb, 0x78, - 0x48, 0xf9, 0xc8, 0xf6, 0xec, 0x01, 0x9d, 0x38, 0x36, 0xb7, 0x65, 0x84, 0xb5, 0xc2, 0xfd, 0xb1, - 0x1f, 0xb9, 0x17, 0xa6, 0xbc, 0xcf, 0x87, 0x33, 0x03, 0x7e, 0x00, 0xd9, 0xbd, 0xe1, 0x19, 0xe3, - 0x74, 0x82, 0x56, 0x20, 0xe9, 0x3a, 0x25, 0xa3, 0x62, 0x54, 0xf3, 0x24, 0xe9, 0x3a, 0x08, 0xc1, - 0x92, 0x67, 0x8f, 0x68, 0x29, 0x29, 0x2c, 0xe2, 0x8c, 0x7f, 0x19, 0x90, 0x3b, 0xa4, 0xe7, 0x6c, - 0x6c, 0xf7, 0x29, 0xba, 0x07, 0xd9, 0xbe, 0xcc, 0x15, 0x59, 0x66, 0xa3, 0x58, 0x0b, 0xf9, 0x29, - 0x4c, 0x12, 0x06, 0xa0, 0x3a, 0xe4, 0x4e, 0x55, 0x9e, 0x00, 0x34, 0x1b, 0x6b, 0xb5, 0x19, 0x97, - 0x10, 0x92, 0xe8, 0x20, 0xf4, 0x18, 0x32, 0xec, 0xc4, 0x9e, 0x38, 0xac, 0x94, 0xaa, 0xa4, 0xaa, - 0x66, 0x63, 0x4b, 0x63, 0x87, 0xc1, 0xb5, 0x8e, 0xf0, 0xef, 0x7b, 0x7c, 0x72, 0x4e, 0x54, 0xb0, - 0x75, 0x08, 0x66, 0xc4, 0x8c, 0x8a, 0x90, 0x3a, 0xa5, 0xe7, 0x4a, 0x54, 0x70, 0x44, 0x77, 0x20, - 0x3d, 0xb5, 0x87, 0x67, 0x21, 0x8b, 0x62, 0x84, 0x85, 0x48, 0x24, 0xd2, 0xfd, 0x34, 0xf9, 0xc4, - 0xc0, 0x9f, 0x0d, 0xc8, 0x74, 0xfa, 0x27, 0x74, 0x64, 0xdf, 0x4a, 0xab, 0x35, 0xa7, 0x35, 0x1f, - 0x91, 0xd5, 0x86, 0x55, 0x31, 0xab, 0x23, 0x87, 0xbe, 0x77, 0x3d, 0x97, 0xbb, 0xbe, 0x17, 0x2a, - 0xc4, 0xb5, 0xc5, 0x29, 0x76, 0x03, 0xcb, 0x0b, 0x1d, 0x4a, 0x8a, 0x3c, 0x6e, 0x60, 0xf8, 0x9b, - 0x01, 0x19, 0x11, 0xc5, 0x6f, 0xc5, 0xb1, 0x0a, 0x19, 0x59, 0x4d, 0xf7, 0x41, 0x6f, 0x8a, 0x44, - 0x23, 0xca, 0x8f, 0x1a, 0x90, 0x66, 0xdc, 0xe6, 0xb4, 0x94, 0xaa, 0x18, 0xd5, 0x95, 0xc6, 0xa6, - 0xc6, 0x94, 0x71, 0xb5, 0x0e, 0x9d, 0x4c, 0x5d, 0x6f, 0xd0, 0x09, 0x62, 0x88, 0x0c, 0xc5, 0x3b, - 0xb0, 0x1c, 0x35, 0x23, 0x13, 0xb2, 0x6f, 0x5a, 0x87, 0xad, 0xf6, 0xdb, 0x56, 0x31, 0x11, 0x5c, - 0x3a, 0xfb, 0xa4, 0x77, 0xd0, 0x6a, 0x16, 0x0d, 0x54, 0x00, 0xb3, 0xd5, 0xee, 0x1e, 0x85, 0x86, - 0x24, 0x7e, 0x0d, 0x99, 0x9e, 0x98, 0x48, 0xd0, 0xc6, 0x13, 0x9f, 0x71, 0xb1, 0x83, 0x72, 0x80, - 0xfa, 0x1e, 0x95, 0x9a, 0xbc, 0x46, 0x2a, 0xfe, 0x68, 0x40, 0xa6, 0xd7, 0x6d, 0x06, 0x3c, 0xae, - 0x82, 0x44, 0xb0, 0x34, 0xf6, 0xfd, 0x61, 0xb8, 0xee, 0xc1, 0x39, 0xb0, 0xf5, 0xe9, 0x70, 0x28, - 0xa4, 0xe7, 0x89, 0x38, 0x47, 0x4b, 0x2f, 0x5d, 0xd7, 0xe5, 0x4d, 0xc8, 0x87, 0x93, 0x67, 0xa5, - 0x74, 0x25, 0x55, 0xcd, 0x93, 0x99, 0x01, 0xaf, 0x03, 0x6a, 0x52, 0xae, 0x92, 0x18, 0xa1, 0x1f, - 0xce, 0x28, 0xe3, 0x78, 0x0f, 0xd6, 0x62, 0x56, 0x36, 0xf6, 0x3d, 0x46, 0xd1, 0x7d, 0xc8, 0x29, - 0x54, 0x56, 0x32, 0xc4, 0xbe, 0x2c, 0xd6, 0xd5, 0x11, 0xb8, 0x01, 0x85, 0x26, 0xe5, 0x81, 0xe6, - 0x10, 0x17, 0xfd, 0x0b, 0xa6, 0x72, 0x1f, 0xb9, 0x8e, 0xc4, 0xc8, 0x13, 0x50, 0xa6, 0x03, 0x87, - 0xe1, 0x1d, 0x28, 0xce, 0x72, 0x54, 0xd5, 0xff, 0x21, 0x3d, 0x08, 0x0c, 0xaa, 0x64, 0x41, 0x97, - 0x94, 0x0d, 0x25, 0xd2, 0x8b, 0xb7, 0x05, 0xe7, 0xf0, 0x61, 0xde, 0xbc, 0x64, 0x13, 0xd6, 0xe3, - 0x79, 0xaa, 0x6c, 0x3d, 0xda, 0x37, 0x59, 0x7a, 0x75, 0xe1, 0xfd, 0x47, 0x5b, 0xf9, 0x08, 0x56, - 0x9b, 0x94, 0xcb, 0xb7, 0x7a, 0xf3, 0xf2, 0xcf, 0xc4, 0x00, 0x74, 0x96, 0x2a, 0x7e, 0x17, 0xb2, - 0x4c, 0x9a, 0x16, 0x54, 0xcb, 0x50, 0x12, 0xfa, 0x71, 0x5b, 0xb4, 0x4c, 0x3d, 0x18, 0x55, 0xf5, - 0xaa, 0x1d, 0x9b, 0x63, 0x94, 0x5c, 0x60, 0x24, 0x75, 0x48, 0xc0, 0xdb, 0xea, 0xd0, 0x59, 0x33, - 0x1d, 0xf2, 0x09, 0x2f, 0xea, 0x50, 0x8c, 0x43, 0x7f, 0xe3, 0x4b, 0x0a, 0xb2, 0xbd, 0xee, 0x6e, - 0xe0, 0x43, 0x2f, 0xc1, 0x8c, 0xec, 0x1f, 0xda, 0xd0, 0x49, 0x8b, 0xbb, 0x6a, 0x6d, 0x5e, 0xee, - 0x94, 0x04, 0x70, 0x02, 0xed, 0x42, 0x2e, 0x5c, 0x29, 0x54, 0x8a, 0xc6, 0x46, 0x37, 0xd3, 0xfa, - 0xe7, 0x12, 0x8f, 0x86, 0x78, 0x05, 0xcb, 0xd1, 0x15, 0x41, 0xb1, 0x92, 0xf3, 0x1b, 0x67, 0x6d, - 0xfd, 0xc1, 0xab, 0xe1, 0x9a, 0x00, 0xb3, 0x91, 0x23, 0x2b, 0x1a, 0x1e, 0xdf, 0x1e, 0x6b, 0xe3, - 0x52, 0x9f, 0x06, 0xda, 0x81, 0xbc, 0xee, 0x39, 0x8a, 0x29, 0x88, 0xad, 0x83, 0x35, 0xdf, 0x74, - 0xcd, 0x41, 0x8d, 0x2b, 0xce, 0x21, 0x3e, 0xf9, 0x38, 0x87, 0xb9, 0xf9, 0xe2, 0xc4, 0xf3, 0xed, - 0xaf, 0x17, 0x65, 0xe3, 0xfb, 0x45, 0xd9, 0xf8, 0x71, 0x51, 0x36, 0x3e, 0xfd, 0x2c, 0x27, 0xde, - 0xfd, 0x37, 0x75, 0x39, 0x65, 0xac, 0xe6, 0xfa, 0x75, 0x79, 0xaa, 0x0f, 0xfc, 0xfa, 0x94, 0xd7, - 0xc5, 0x67, 0x5f, 0x57, 0x60, 0xc7, 0x19, 0x71, 0x7d, 0xf8, 0x3b, 0x00, 0x00, 0xff, 0xff, 0x60, - 0xa7, 0x26, 0xf2, 0x4f, 0x08, 0x00, 0x00, + // 851 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x56, 0xdd, 0x6e, 0xe3, 0x44, + 0x14, 0xae, 0x93, 0x36, 0x89, 0x8f, 0x97, 0xd6, 0x99, 0x5d, 0x09, 0xe3, 0xed, 0x86, 0x68, 0x04, + 0x28, 0x20, 0x88, 0xa5, 0x00, 0x2b, 0xca, 0x0d, 0x5a, 0x76, 0xab, 0x68, 0xa9, 0x48, 0x90, 0x13, + 0x82, 0xb4, 0x37, 0x95, 0x37, 0x1e, 0x52, 0x6b, 0x1d, 0x3b, 0x9b, 0x99, 0x46, 0xf4, 0x45, 0x10, + 0xb7, 0xbc, 0x0d, 0x37, 0x48, 0x3c, 0x02, 0x2a, 0x97, 0xbc, 0x01, 0x57, 0xc8, 0xf3, 0x97, 0x71, + 0x92, 0xdd, 0xb6, 0x77, 0x33, 0xe7, 0xe7, 0x3b, 0xe7, 0x3b, 0xf3, 0x1d, 0xcb, 0xf0, 0xce, 0x8a, + 0x45, 0xf1, 0x3c, 0xc9, 0xba, 0x8b, 0x65, 0xce, 0x72, 0x54, 0x97, 0x57, 0xff, 0x5d, 0x16, 0xbd, + 0x4c, 0x09, 0x9b, 0x47, 0x59, 0x34, 0x23, 0xcb, 0x38, 0x62, 0x91, 0x88, 0xf0, 0x0f, 0x59, 0xbe, + 0xc8, 0x8d, 0xfb, 0xd1, 0x8a, 0x4d, 0x59, 0xba, 0x36, 0xe0, 0xcf, 0xa0, 0xfe, 0x34, 0xbd, 0xa4, + 0x8c, 0x2c, 0xd1, 0x21, 0x54, 0x92, 0xd8, 0xb3, 0xda, 0x56, 0xc7, 0x0e, 0x2b, 0x49, 0x8c, 0x10, + 0xec, 0x67, 0xd1, 0x9c, 0x78, 0x15, 0x6e, 0xe1, 0x67, 0xfc, 0xaf, 0x05, 0x8d, 0x33, 0x72, 0x45, + 0x17, 0xd1, 0x94, 0xa0, 0x4f, 0xa0, 0x3e, 0x15, 0xb9, 0x3c, 0xcb, 0xe9, 0xb9, 0x5d, 0xd5, 0x9f, + 0xc4, 0x0c, 0x55, 0x00, 0x0a, 0xa0, 0xf1, 0x4a, 0xe6, 0x71, 0x40, 0xa7, 0x77, 0xbf, 0xbb, 0xee, + 0x45, 0x41, 0x86, 0x3a, 0x08, 0x7d, 0x09, 0x35, 0x7a, 0x11, 0x2d, 0x63, 0xea, 0x55, 0xdb, 0xd5, + 0x8e, 0xd3, 0x7b, 0xa4, 0xb1, 0x55, 0x70, 0x77, 0xc4, 0xfd, 0xa7, 0x19, 0x5b, 0x5e, 0x85, 0x32, + 0xd8, 0x3f, 0x03, 0xc7, 0x30, 0x23, 0x17, 0xaa, 0xaf, 0xc8, 0x95, 0x24, 0x55, 0x1c, 0xd1, 0x47, + 0x70, 0xb0, 0x8a, 0xd2, 0x4b, 0xd5, 0x85, 0x6b, 0x74, 0xc1, 0x13, 0x43, 0xe1, 0xfe, 0xba, 0xf2, + 0x95, 0x85, 0x7f, 0xb7, 0xa0, 0x36, 0x9a, 0x5e, 0x90, 0x79, 0x74, 0x27, 0xae, 0xfe, 0x06, 0x57, + 0xdb, 0xa0, 0x35, 0x84, 0x26, 0x7f, 0xab, 0xf3, 0x98, 0xfc, 0x9c, 0x64, 0x09, 0x4b, 0xf2, 0x4c, + 0x31, 0xc4, 0xdd, 0xed, 0x57, 0x1c, 0x17, 0x96, 0x67, 0x3a, 0x34, 0x74, 0x59, 0xd9, 0x40, 0xf1, + 0x9f, 0x16, 0xd4, 0x78, 0x14, 0xbb, 0x53, 0x8f, 0x1d, 0xa8, 0x89, 0x6a, 0x7a, 0x0e, 0x5a, 0x29, + 0x02, 0x2d, 0x94, 0x7e, 0xd4, 0x83, 0x03, 0xca, 0x22, 0x46, 0xbc, 0x6a, 0xdb, 0xea, 0x1c, 0xf6, + 0x8e, 0x35, 0xa6, 0x88, 0xeb, 0x8e, 0xc8, 0x72, 0x95, 0x64, 0xb3, 0x51, 0x11, 0x13, 0x8a, 0x50, + 0x7c, 0x02, 0xf7, 0x4c, 0x33, 0x72, 0xa0, 0xfe, 0xe3, 0xe0, 0x6c, 0x30, 0xfc, 0x69, 0xe0, 0xee, + 0x15, 0x97, 0xd1, 0x69, 0x38, 0x79, 0x3e, 0xe8, 0xbb, 0x16, 0x3a, 0x02, 0x67, 0x30, 0x1c, 0x9f, + 0x2b, 0x43, 0x05, 0xff, 0x00, 0xb5, 0x09, 0x7f, 0x91, 0x62, 0x8c, 0x17, 0x39, 0x65, 0x5c, 0x83, + 0xe2, 0x01, 0xf5, 0xdd, 0xa4, 0x5a, 0xb9, 0x81, 0x2a, 0xfe, 0xd5, 0x82, 0xda, 0x64, 0xdc, 0x2f, + 0xfa, 0x78, 0x1b, 0x24, 0x82, 0xfd, 0x45, 0x9e, 0xa7, 0x4a, 0xee, 0xc5, 0xb9, 0xb0, 0x4d, 0x49, + 0x9a, 0x72, 0xea, 0x76, 0xc8, 0xcf, 0x66, 0xe9, 0xfd, 0x9b, 0xa6, 0x7c, 0x0c, 0xb6, 0x7a, 0x79, + 0xea, 0x1d, 0xb4, 0xab, 0x1d, 0x3b, 0x5c, 0x1b, 0xf0, 0x03, 0x40, 0x7d, 0xc2, 0x64, 0x12, 0x0d, + 0xc9, 0xeb, 0x4b, 0x42, 0x19, 0x7e, 0x0a, 0xf7, 0x4b, 0x56, 0xba, 0xc8, 0x33, 0x4a, 0xd0, 0xa7, + 0xd0, 0x90, 0xa8, 0xd4, 0xb3, 0xb8, 0x5e, 0xb6, 0xeb, 0xea, 0x08, 0xdc, 0x83, 0xa3, 0x3e, 0x61, + 0x05, 0x67, 0x85, 0x8b, 0xde, 0x07, 0x47, 0xba, 0xcf, 0x93, 0x58, 0x60, 0xd8, 0x21, 0x48, 0xd3, + 0xf3, 0x98, 0xe2, 0x13, 0x70, 0xd7, 0x39, 0xb2, 0xea, 0x87, 0x70, 0x30, 0x2b, 0x0c, 0xb2, 0xe4, + 0x91, 0x2e, 0x29, 0x06, 0x1a, 0x0a, 0x2f, 0x7e, 0xcc, 0x7b, 0x56, 0x8b, 0x79, 0xfb, 0x92, 0x7d, + 0x78, 0x50, 0xce, 0x93, 0x65, 0x03, 0x73, 0x6e, 0xa2, 0x74, 0x73, 0x6b, 0xff, 0xcd, 0x51, 0x7e, + 0x01, 0xcd, 0x3e, 0x61, 0x62, 0x57, 0x6f, 0x5f, 0xfe, 0x1b, 0xfe, 0x00, 0x3a, 0x4b, 0x16, 0xff, + 0x18, 0xea, 0x54, 0x98, 0xb6, 0x58, 0x8b, 0xd0, 0x50, 0xf9, 0xf1, 0x90, 0x8f, 0x4c, 0x2e, 0x8c, + 0xac, 0xfa, 0x36, 0x8d, 0x6d, 0x74, 0x54, 0xd9, 0xea, 0x48, 0xf0, 0x10, 0x80, 0x77, 0xe5, 0xa1, + 0xb3, 0xd6, 0x3c, 0xc4, 0x0a, 0x6f, 0xf3, 0x90, 0x1d, 0x2b, 0x3f, 0x7e, 0x01, 0xee, 0x64, 0x7c, + 0xfa, 0xcb, 0x22, 0x8d, 0x92, 0x4c, 0x55, 0xf5, 0xca, 0x5f, 0x13, 0xfb, 0x76, 0xdf, 0x37, 0x17, + 0xaa, 0xf4, 0xb5, 0x5a, 0x98, 0xe2, 0x88, 0x03, 0x68, 0x1a, 0xd8, 0xb2, 0x37, 0x1f, 0x1a, 0x4b, + 0x79, 0x56, 0x43, 0x52, 0xf7, 0xde, 0x7f, 0x55, 0xa8, 0x4f, 0xc6, 0x4f, 0x8a, 0x46, 0xd1, 0x77, + 0xe0, 0x18, 0xcb, 0x80, 0x1e, 0x6a, 0x06, 0xdb, 0x8b, 0xe3, 0x1f, 0xef, 0x76, 0x0a, 0x54, 0xbc, + 0x87, 0x9e, 0x40, 0x43, 0xe9, 0x1b, 0x79, 0x66, 0xac, 0xb9, 0x26, 0xfe, 0x7b, 0x3b, 0x3c, 0x1a, + 0xe2, 0x7b, 0xb8, 0x67, 0xea, 0x15, 0x95, 0x4a, 0x6e, 0xca, 0xdf, 0x7f, 0xf4, 0x06, 0xaf, 0x86, + 0xeb, 0x03, 0xac, 0xf5, 0x87, 0x7c, 0x33, 0xbc, 0x2c, 0x65, 0xff, 0xe1, 0x4e, 0x9f, 0x06, 0x3a, + 0x01, 0x5b, 0x0b, 0x00, 0x95, 0x18, 0x94, 0xb4, 0xe9, 0x6f, 0x2a, 0x40, 0xf7, 0x20, 0xb5, 0x53, + 0xee, 0xa1, 0x2c, 0xc3, 0x72, 0x0f, 0x1b, 0x62, 0xc3, 0x7b, 0xe8, 0x19, 0xd8, 0xfa, 0x9d, 0x8d, + 0x1e, 0x36, 0x75, 0xe5, 0xfb, 0xbb, 0x5c, 0x0a, 0xe5, 0xdb, 0xc7, 0x7f, 0x5c, 0xb7, 0xac, 0xbf, + 0xae, 0x5b, 0xd6, 0xdf, 0xd7, 0x2d, 0xeb, 0xb7, 0x7f, 0x5a, 0x7b, 0x2f, 0x3e, 0x58, 0x25, 0x8c, + 0x50, 0xda, 0x4d, 0xf2, 0x40, 0x9c, 0x82, 0x59, 0x1e, 0xac, 0x58, 0xc0, 0xff, 0x5f, 0x02, 0x89, + 0xf5, 0xb2, 0xc6, 0xaf, 0x9f, 0xff, 0x1f, 0x00, 0x00, 0xff, 0xff, 0x19, 0xb9, 0x4d, 0x1f, 0x22, + 0x09, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -1074,6 +1190,8 @@ type VTAdminClient interface { GetTablet(ctx context.Context, in *GetTabletRequest, opts ...grpc.CallOption) (*Tablet, error) // GetTablets returns all tablets across all the specified clusters. GetTablets(ctx context.Context, in *GetTabletsRequest, opts ...grpc.CallOption) (*GetTabletsResponse, error) + // VTExplain provides information on how Vitess plans to execute a particular query. + VTExplain(ctx context.Context, in *VTExplainRequest, opts ...grpc.CallOption) (*VTExplainResponse, error) } type vTAdminClient struct { @@ -1138,6 +1256,15 @@ func (c *vTAdminClient) GetTablets(ctx context.Context, in *GetTabletsRequest, o return out, nil } +func (c *vTAdminClient) VTExplain(ctx context.Context, in *VTExplainRequest, opts ...grpc.CallOption) (*VTExplainResponse, error) { + out := new(VTExplainResponse) + err := c.cc.Invoke(ctx, "/vtadmin.VTAdmin/VTExplain", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + // VTAdminServer is the server API for VTAdmin service. type VTAdminServer interface { // GetClusters returns all configured clusters. @@ -1153,6 +1280,8 @@ type VTAdminServer interface { GetTablet(context.Context, *GetTabletRequest) (*Tablet, error) // GetTablets returns all tablets across all the specified clusters. GetTablets(context.Context, *GetTabletsRequest) (*GetTabletsResponse, error) + // VTExplain provides information on how Vitess plans to execute a particular query. + VTExplain(context.Context, *VTExplainRequest) (*VTExplainResponse, error) } // UnimplementedVTAdminServer can be embedded to have forward compatible implementations. @@ -1177,6 +1306,9 @@ func (*UnimplementedVTAdminServer) GetTablet(ctx context.Context, req *GetTablet func (*UnimplementedVTAdminServer) GetTablets(ctx context.Context, req *GetTabletsRequest) (*GetTabletsResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method GetTablets not implemented") } +func (*UnimplementedVTAdminServer) VTExplain(ctx context.Context, req *VTExplainRequest) (*VTExplainResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method VTExplain not implemented") +} func RegisterVTAdminServer(s *grpc.Server, srv VTAdminServer) { s.RegisterService(&_VTAdmin_serviceDesc, srv) @@ -1290,6 +1422,24 @@ func _VTAdmin_GetTablets_Handler(srv interface{}, ctx context.Context, dec func( return interceptor(ctx, in, info, handler) } +func _VTAdmin_VTExplain_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(VTExplainRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VTAdminServer).VTExplain(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/vtadmin.VTAdmin/VTExplain", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VTAdminServer).VTExplain(ctx, req.(*VTExplainRequest)) + } + return interceptor(ctx, in, info, handler) +} + var _VTAdmin_serviceDesc = grpc.ServiceDesc{ ServiceName: "vtadmin.VTAdmin", HandlerType: (*VTAdminServer)(nil), @@ -1318,6 +1468,10 @@ var _VTAdmin_serviceDesc = grpc.ServiceDesc{ MethodName: "GetTablets", Handler: _VTAdmin_GetTablets_Handler, }, + { + MethodName: "VTExplain", + Handler: _VTAdmin_VTExplain_Handler, + }, }, Streams: []grpc.StreamDesc{}, Metadata: "vtadmin.proto", @@ -2091,6 +2245,88 @@ func (m *GetTabletsResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *VTExplainRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *VTExplainRequest) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *VTExplainRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.Sql) > 0 { + i -= len(m.Sql) + copy(dAtA[i:], m.Sql) + i = encodeVarintVtadmin(dAtA, i, uint64(len(m.Sql))) + i-- + dAtA[i] = 0x1a + } + if len(m.Keyspace) > 0 { + i -= len(m.Keyspace) + copy(dAtA[i:], m.Keyspace) + i = encodeVarintVtadmin(dAtA, i, uint64(len(m.Keyspace))) + i-- + dAtA[i] = 0x12 + } + if len(m.Cluster) > 0 { + i -= len(m.Cluster) + copy(dAtA[i:], m.Cluster) + i = encodeVarintVtadmin(dAtA, i, uint64(len(m.Cluster))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *VTExplainResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *VTExplainResponse) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *VTExplainResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.Response) > 0 { + i -= len(m.Response) + copy(dAtA[i:], m.Response) + i = encodeVarintVtadmin(dAtA, i, uint64(len(m.Response))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + func encodeVarintVtadmin(dAtA []byte, offset int, v uint64) int { offset -= sovVtadmin(v) base := offset @@ -2454,6 +2690,46 @@ func (m *GetTabletsResponse) Size() (n int) { return n } +func (m *VTExplainRequest) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Cluster) + if l > 0 { + n += 1 + l + sovVtadmin(uint64(l)) + } + l = len(m.Keyspace) + if l > 0 { + n += 1 + l + sovVtadmin(uint64(l)) + } + l = len(m.Sql) + if l > 0 { + n += 1 + l + sovVtadmin(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *VTExplainResponse) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Response) + if l > 0 { + n += 1 + l + sovVtadmin(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + func sovVtadmin(x uint64) (n int) { return (math_bits.Len64(x|1) + 6) / 7 } @@ -4430,6 +4706,242 @@ func (m *GetTabletsResponse) Unmarshal(dAtA []byte) error { } return nil } +func (m *VTExplainRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowVtadmin + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: VTExplainRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: VTExplainRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Cluster", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowVtadmin + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthVtadmin + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthVtadmin + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Cluster = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Keyspace", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowVtadmin + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthVtadmin + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthVtadmin + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Keyspace = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Sql", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowVtadmin + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthVtadmin + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthVtadmin + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Sql = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipVtadmin(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthVtadmin + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthVtadmin + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *VTExplainResponse) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowVtadmin + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: VTExplainResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: VTExplainResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Response", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowVtadmin + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthVtadmin + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthVtadmin + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Response = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipVtadmin(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthVtadmin + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthVtadmin + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func skipVtadmin(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 diff --git a/go/vt/servenv/buildinfo.go b/go/vt/servenv/buildinfo.go index d73c3e07c57..449d226ac15 100644 --- a/go/vt/servenv/buildinfo.go +++ b/go/vt/servenv/buildinfo.go @@ -23,6 +23,13 @@ import ( "strconv" "time" + "vitess.io/vitess/go/vt/proto/vtrpc" + "vitess.io/vitess/go/vt/vterrors" + + "vitess.io/vitess/go/vt/log" + + "vitess.io/vitess/go/vt/sqlparser" + "vitess.io/vitess/go/stats" ) @@ -104,7 +111,13 @@ func init() { goArch: runtime.GOARCH, version: versionName, } - + var convVersion string + convVersion, err = convertMySQLVersionToCommentVersion(AppVersion.MySQLVersion()) + if err != nil { + log.Error(err) + } else { + sqlparser.MySQLVersion = convVersion + } stats.NewString("BuildHost").Set(AppVersion.buildHost) stats.NewString("BuildUser").Set(AppVersion.buildUser) stats.NewGauge("BuildTimestamp", "build timestamp").Set(AppVersion.buildTime) @@ -126,3 +139,41 @@ func init() { } stats.NewGaugesWithMultiLabels("BuildInformation", "build information exposed via label", buildLabels).Set(buildValues, 1) } + +// convertMySQLVersionToCommentVersion converts the MySQL version into comment version format. +func convertMySQLVersionToCommentVersion(version string) (string, error) { + var res = make([]int, 3) + idx := 0 + val := "" + for _, c := range version { + if c <= '9' && c >= '0' { + val += string(c) + } else if c == '.' { + v, err := strconv.Atoi(val) + if err != nil { + return "", err + } + val = "" + res[idx] = v + idx++ + if idx == 3 { + break + } + } else { + break + } + } + if val != "" { + v, err := strconv.Atoi(val) + if err != nil { + return "", err + } + res[idx] = v + idx++ + } + if idx == 0 { + return "", vterrors.Errorf(vtrpc.Code_INVALID_ARGUMENT, "MySQL version not correctly setup - %s.", version) + } + + return fmt.Sprintf("%01d%02d%02d", res[0], res[1], res[2]), nil +} diff --git a/go/vt/servenv/buildinfo_test.go b/go/vt/servenv/buildinfo_test.go index 63ba5596df7..1517f4abf65 100644 --- a/go/vt/servenv/buildinfo_test.go +++ b/go/vt/servenv/buildinfo_test.go @@ -20,6 +20,8 @@ import ( "testing" "time" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/assert" ) @@ -50,3 +52,41 @@ func TestVersionString(t *testing.T) { MySQLServerVersion = &newVersion assert.Equal(t, newVersion, v.MySQLVersion()) } + +func TestConvertMySQLVersion(t *testing.T) { + testcases := []struct { + version string + commentVersion string + error string + }{{ + version: "5.7.9", + commentVersion: "50709", + }, { + version: "0008.08.9", + commentVersion: "80809", + }, { + version: "5.7.9, Vitess - 10.0.1", + commentVersion: "50709", + }, { + version: "8.1 Vitess - 10.0.1", + commentVersion: "80100", + }, { + version: "Vitess - 10.0.1", + error: "MySQL version not correctly setup - Vitess - 10.0.1.", + }, { + version: "5.7.9.22", + commentVersion: "50709", + }} + + for _, tcase := range testcases { + t.Run(tcase.version, func(t *testing.T) { + output, err := convertMySQLVersionToCommentVersion(tcase.version) + if tcase.error != "" { + require.EqualError(t, err, tcase.error) + } else { + require.NoError(t, err) + require.Equal(t, tcase.commentVersion, output) + } + }) + } +} diff --git a/go/vt/sqlparser/comments.go b/go/vt/sqlparser/comments.go index f4bc122b95c..f64c4fd5e7e 100644 --- a/go/vt/sqlparser/comments.go +++ b/go/vt/sqlparser/comments.go @@ -181,6 +181,9 @@ func ExtractMysqlComment(sql string) (string, string) { if endOfVersionIndex < 0 { return "", "" } + if endOfVersionIndex < 5 { + endOfVersionIndex = 0 + } version := sql[0:endOfVersionIndex] innerSQL := strings.TrimFunc(sql[endOfVersionIndex:], unicode.IsSpace) diff --git a/go/vt/sqlparser/parse_test.go b/go/vt/sqlparser/parse_test.go index d4c1553927d..1266bad3a92 100644 --- a/go/vt/sqlparser/parse_test.go +++ b/go/vt/sqlparser/parse_test.go @@ -1757,6 +1757,9 @@ var ( input: "create database test_db character set * unparsable", output: "create database test_db", partialDDL: true, + }, { + input: "CREATE DATABASE /*!32312 IF NOT EXISTS*/ `mysql` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci */ /*!80016 DEFAULT ENCRYPTION='N' */;", + output: "create database if not exists mysql default character set utf8mb4 collate utf8mb4_0900_ai_ci", }, { input: "drop database test_db", }, { diff --git a/go/vt/sqlparser/parser.go b/go/vt/sqlparser/parser.go index 08df99efb65..8298cba6f1c 100644 --- a/go/vt/sqlparser/parser.go +++ b/go/vt/sqlparser/parser.go @@ -35,6 +35,9 @@ var parserPool = sync.Pool{} // zeroParser is a zero-initialized parser to help reinitialize the parser for pooling. var zeroParser = *(yyNewParser().(*yyParserImpl)) +// MySQLVersion is the version of MySQL that the parser would emulate +var MySQLVersion string = "50709" + // yyParsePooled is a wrapper around yyParse that pools the parser objects. There isn't a // particularly good reason to use yyParse directly, since it immediately discards its parser. What // would be ideal down the line is to actually pool the stacks themselves rather than the parser diff --git a/go/vt/sqlparser/token.go b/go/vt/sqlparser/token.go index 6b24dacd78e..46382591429 100644 --- a/go/vt/sqlparser/token.go +++ b/go/vt/sqlparser/token.go @@ -1031,8 +1031,14 @@ func (tkn *Tokenizer) scanMySQLSpecificComment() (int, []byte) { } tkn.consumeNext(buffer) } - _, sql := ExtractMysqlComment(buffer.String()) - tkn.specialComment = NewStringTokenizer(sql) + + commentVersion, sql := ExtractMysqlComment(buffer.String()) + + if MySQLVersion >= commentVersion { + // Only add the special comment to the tokenizer if the version of MySQL is higher or equal to the comment version + tkn.specialComment = NewStringTokenizer(sql) + } + return tkn.Scan() } diff --git a/go/vt/sqlparser/token_test.go b/go/vt/sqlparser/token_test.go index cac25328e00..6b80ec27c6c 100644 --- a/go/vt/sqlparser/token_test.go +++ b/go/vt/sqlparser/token_test.go @@ -19,6 +19,8 @@ package sqlparser import ( "fmt" "testing" + + "github.com/stretchr/testify/require" ) func TestLiteralID(t *testing.T) { @@ -75,9 +77,8 @@ func TestLiteralID(t *testing.T) { for _, tcase := range testcases { tkn := NewStringTokenizer(tcase.in) id, out := tkn.Scan() - if tcase.id != id || string(out) != tcase.out { - t.Errorf("Scan(%s): %d, %s, want %d, %s", tcase.in, id, out, tcase.id, tcase.out) - } + require.Equal(t, tcase.id, id) + require.Equal(t, tcase.out, string(out)) } } @@ -146,10 +147,11 @@ func TestString(t *testing.T) { }} for _, tcase := range testcases { - id, got := NewStringTokenizer(tcase.in).Scan() - if tcase.id != id || string(got) != tcase.want { - t.Errorf("Scan(%q) = (%s, %q), want (%s, %q)", tcase.in, tokenName(id), got, tokenName(tcase.id), tcase.want) - } + t.Run(tcase.in, func(t *testing.T) { + id, got := NewStringTokenizer(tcase.in).Scan() + require.Equal(t, tcase.id, id, "Scan(%q) = (%s), want (%s)", tcase.in, tokenName(id), tokenName(tcase.id)) + require.Equal(t, tcase.want, string(got)) + }) } } @@ -205,3 +207,61 @@ func TestSplitStatement(t *testing.T) { } } } + +func TestVersion(t *testing.T) { + testcases := []struct { + version string + in string + id []int + }{{ + version: "50709", + in: "/*!80102 SELECT*/ FROM IN EXISTS", + id: []int{FROM, IN, EXISTS, 0}, + }, { + version: "80101", + in: "/*!80102 SELECT*/ FROM IN EXISTS", + id: []int{FROM, IN, EXISTS, 0}, + }, { + version: "80201", + in: "/*!80102 SELECT*/ FROM IN EXISTS", + id: []int{SELECT, FROM, IN, EXISTS, 0}, + }, { + version: "80102", + in: "/*!80102 SELECT*/ FROM IN EXISTS", + id: []int{SELECT, FROM, IN, EXISTS, 0}, + }} + + for _, tcase := range testcases { + t.Run(tcase.version+"_"+tcase.in, func(t *testing.T) { + MySQLVersion = tcase.version + tok := NewStringTokenizer(tcase.in) + for _, expectedID := range tcase.id { + id, _ := tok.Scan() + require.Equal(t, expectedID, id) + } + }) + } +} + +func TestExtractMySQLComment(t *testing.T) { + testcases := []struct { + comment string + version string + }{{ + comment: "/*!50108 SELECT * FROM */", + version: "50108", + }, { + comment: "/*!5018 SELECT * FROM */", + version: "", + }, { + comment: "/*!SELECT * FROM */", + version: "", + }} + + for _, tcase := range testcases { + t.Run(tcase.version, func(t *testing.T) { + output, _ := ExtractMysqlComment(tcase.comment) + require.Equal(t, tcase.version, output) + }) + } +} diff --git a/go/vt/vtadmin/api.go b/go/vt/vtadmin/api.go index cb755ae5c6c..bb45cc03dd3 100644 --- a/go/vt/vtadmin/api.go +++ b/go/vt/vtadmin/api.go @@ -18,7 +18,10 @@ package vtadmin import ( "context" + "encoding/json" + "fmt" "net/http" + "strings" "sync" "github.com/gorilla/handlers" @@ -26,13 +29,16 @@ import ( "vitess.io/vitess/go/trace" "vitess.io/vitess/go/vt/concurrency" + "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/vtadmin/cluster" "vitess.io/vitess/go/vt/vtadmin/grpcserver" vtadminhttp "vitess.io/vitess/go/vt/vtadmin/http" vthandlers "vitess.io/vitess/go/vt/vtadmin/http/handlers" "vitess.io/vitess/go/vt/vtadmin/sort" "vitess.io/vitess/go/vt/vterrors" + "vitess.io/vitess/go/vt/vtexplain" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" vtadminpb "vitess.io/vitess/go/vt/proto/vtadmin" vtctldatapb "vitess.io/vitess/go/vt/proto/vtctldata" vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" @@ -83,6 +89,7 @@ func NewAPI(clusters []*cluster.Cluster, opts grpcserver.Options, httpOpts vtadm router.HandleFunc("/schemas", httpAPI.Adapt(vtadminhttp.GetSchemas)).Name("API.GetSchemas") router.HandleFunc("/tablets", httpAPI.Adapt(vtadminhttp.GetTablets)).Name("API.GetTablets") router.HandleFunc("/tablet/{tablet}", httpAPI.Adapt(vtadminhttp.GetTablet)).Name("API.GetTablet") + router.HandleFunc("/vtexplain", httpAPI.Adapt(vtadminhttp.VTExplain)).Name("API.VTExplain") // Middlewares are executed in order of addition. Our ordering (all // middlewares being optional) is: @@ -534,6 +541,47 @@ func (api *API) getTablets(ctx context.Context, c *cluster.Cluster) ([]*vtadminp return ParseTablets(rows, c) } +// findTablet returns the first tablet in a given cluster that satisfies the filter function. +func (api *API) findTablet(ctx context.Context, cluster *cluster.Cluster, filter func(*vtadminpb.Tablet) bool) (*vtadminpb.Tablet, error) { + tablets, err := api.findTablets(ctx, cluster, filter, 1) + if err != nil { + return nil, err + } + + if len(tablets) != 1 { + return nil, ErrNoTablet + } + + return tablets[0], nil +} + +// findTablets returns the first N tablets in the given cluster that satisfy +// the filter function. If N = -1, then all matching tablets are returned. +// Ordering is not guaranteed, and callers should write their filter functions accordingly. +func (api *API) findTablets(ctx context.Context, cluster *cluster.Cluster, filter func(*vtadminpb.Tablet) bool, n int) ([]*vtadminpb.Tablet, error) { + tablets, err := api.getTablets(ctx, cluster) + if err != nil { + return nil, err + } + + if n == -1 { + n = len(tablets) + } + + results := make([]*vtadminpb.Tablet, 0, n) + for _, t := range tablets { + if len(results) >= n { + break + } + + if filter(t) { + results = append(results, t) + } + } + + return results, nil +} + func (api *API) getClustersForRequest(ids []string) ([]*cluster.Cluster, []string) { if len(ids) == 0 { clusterIDs := make([]string, 0, len(api.clusters)) @@ -555,3 +603,152 @@ func (api *API) getClustersForRequest(ids []string) ([]*cluster.Cluster, []strin return clusters, ids } + +// VTExplain is part of the vtadminpb.VTAdminServer interface. +func (api *API) VTExplain(ctx context.Context, req *vtadminpb.VTExplainRequest) (*vtadminpb.VTExplainResponse, error) { + span, ctx := trace.NewSpan(ctx, "API.VTExplain") + defer span.Finish() + + if req.Cluster == "" { + return nil, fmt.Errorf("%w: cluster ID is required", ErrInvalidRequest) + } + + if req.Keyspace == "" { + return nil, fmt.Errorf("%w: keyspace name is required", ErrInvalidRequest) + } + + if req.Sql == "" { + return nil, fmt.Errorf("%w: SQL query is required", ErrInvalidRequest) + } + + c, ok := api.clusterMap[req.Cluster] + if !ok { + return nil, ErrUnsupportedCluster + } + + tablet, err := api.findTablet(ctx, c, func(t *vtadminpb.Tablet) bool { + return t.Tablet.Keyspace == req.Keyspace && topo.IsInServingGraph(t.Tablet.Type) && t.Tablet.Type != topodatapb.TabletType_MASTER && t.State == vtadminpb.Tablet_SERVING + }) + + if err != nil { + return nil, err + } + + if err := c.Vtctld.Dial(ctx); err != nil { + return nil, err + } + + var ( + wg sync.WaitGroup + er concurrency.AllErrorRecorder + + // Writes to these three variables are, in the strictest sense, unsafe. + // However, there is one goroutine responsible for writing each of these + // values (so, no concurrent writes), and reads are blocked on the call to + // wg.Wait(), so we guarantee that all writes have finished before attempting + // to read anything. + srvVSchema string + schema string + shardMap string + ) + + wg.Add(3) + + // GetSchema + go func(c *cluster.Cluster) { + defer wg.Done() + + res, err := c.Vtctld.GetSchema(ctx, &vtctldatapb.GetSchemaRequest{ + TabletAlias: tablet.Tablet.Alias, + }) + + if err != nil { + er.RecordError(err) + return + } + + schemas := make([]string, len(res.Schema.TableDefinitions)) + for i, td := range res.Schema.TableDefinitions { + schemas[i] = td.Schema + } + + schema = strings.Join(schemas, ";") + }(c) + + // GetSrvVSchema + go func(c *cluster.Cluster) { + defer wg.Done() + + res, err := c.Vtctld.GetSrvVSchema(ctx, &vtctldatapb.GetSrvVSchemaRequest{ + Cell: tablet.Tablet.Alias.Cell, + }) + + if err != nil { + er.RecordError(err) + return + } + + ksvs, ok := res.SrvVSchema.Keyspaces[req.Keyspace] + if !ok { + er.RecordError(fmt.Errorf("%w: keyspace %s", ErrNoSrvVSchema, req.Keyspace)) + return + } + + ksvsb, err := json.Marshal(&ksvs) + if err != nil { + er.RecordError(err) + return + } + + srvVSchema = fmt.Sprintf(`{"%s": %s}`, req.Keyspace, string(ksvsb)) + }(c) + + // FindAllShardsInKeyspace + go func(c *cluster.Cluster) { + defer wg.Done() + + ksm, err := c.Vtctld.FindAllShardsInKeyspace(ctx, &vtctldatapb.FindAllShardsInKeyspaceRequest{ + Keyspace: req.Keyspace, + }) + + if err != nil { + er.RecordError(err) + return + } + + vtsm := make(map[string]*topodatapb.Shard) + for _, s := range ksm.Shards { + vtsm[s.Name] = s.Shard + } + + vtsb, err := json.Marshal(&vtsm) + if err != nil { + er.RecordError(err) + return + } + + shardMap = fmt.Sprintf(`{"%s": %s}`, req.Keyspace, string(vtsb)) + }(c) + + wg.Wait() + + if er.HasErrors() { + return nil, er.Error() + } + + opts := &vtexplain.Options{ReplicationMode: "ROW"} + + if err := vtexplain.Init(srvVSchema, schema, shardMap, opts); err != nil { + return nil, err + } + + plans, err := vtexplain.Run(req.Sql) + if err != nil { + return nil, err + } + + response := vtexplain.ExplainsAsText(plans) + return &vtadminpb.VTExplainResponse{ + Response: response, + }, nil +} diff --git a/go/vt/vtadmin/api_test.go b/go/vt/vtadmin/api_test.go index 4a73860dd2b..688aa917fa2 100644 --- a/go/vt/vtadmin/api_test.go +++ b/go/vt/vtadmin/api_test.go @@ -19,6 +19,7 @@ package vtadmin import ( "context" "database/sql" + "errors" "fmt" "testing" @@ -47,6 +48,7 @@ import ( querypb "vitess.io/vitess/go/vt/proto/query" tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata" topodatapb "vitess.io/vitess/go/vt/proto/topodata" + vschemapb "vitess.io/vitess/go/vt/proto/vschema" vtadminpb "vitess.io/vitess/go/vt/proto/vtadmin" vtctldatapb "vitess.io/vitess/go/vt/proto/vtctldata" vtctlservicepb "vitess.io/vitess/go/vt/proto/vtctlservice" @@ -1351,6 +1353,282 @@ func TestGetTablet(t *testing.T) { } } +func TestVTExplain(t *testing.T) { + tests := []struct { + name string + keyspaces []*vtctldatapb.Keyspace + shards []*vtctldatapb.Shard + srvVSchema *vschemapb.SrvVSchema + tabletSchemas map[string]*tabletmanagerdatapb.SchemaDefinition + tablets []*vtadminpb.Tablet + req *vtadminpb.VTExplainRequest + expectedError error + }{ + { + name: "runs VTExplain given a valid request in a valid topology", + keyspaces: []*vtctldatapb.Keyspace{ + { + Name: "commerce", + Keyspace: &topodatapb.Keyspace{}, + }, + }, + shards: []*vtctldatapb.Shard{ + { + Name: "-", + Keyspace: "commerce", + }, + }, + srvVSchema: &vschemapb.SrvVSchema{ + Keyspaces: map[string]*vschemapb.Keyspace{ + "commerce": { + Sharded: false, + Tables: map[string]*vschemapb.Table{ + "customers": {}, + }, + }, + }, + RoutingRules: &vschemapb.RoutingRules{ + Rules: []*vschemapb.RoutingRule{}, + }, + }, + tabletSchemas: map[string]*tabletmanagerdatapb.SchemaDefinition{ + "c0_cell1-0000000100": { + DatabaseSchema: "CREATE DATABASE commerce", + TableDefinitions: []*tabletmanagerdatapb.TableDefinition{ + { + Name: "t1", + Schema: `CREATE TABLE customers (id int(11) not null,PRIMARY KEY (id));`, + Type: "BASE", + Columns: []string{"id"}, + DataLength: 100, + RowCount: 50, + Fields: []*querypb.Field{ + { + Name: "id", + Type: querypb.Type_INT32, + }, + }, + }, + }, + }, + }, + tablets: []*vtadminpb.Tablet{ + { + Cluster: &vtadminpb.Cluster{ + Id: "c0", + Name: "cluster0", + }, + State: vtadminpb.Tablet_SERVING, + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Uid: 100, + Cell: "c0_cell1", + }, + Hostname: "tablet-cell1-a", + Keyspace: "commerce", + Shard: "-", + Type: topodatapb.TabletType_REPLICA, + }, + }, + }, + req: &vtadminpb.VTExplainRequest{ + Cluster: "c0", + Keyspace: "commerce", + Sql: "select * from customers", + }, + }, + { + name: "returns an error if no appropriate tablet found in keyspace", + keyspaces: []*vtctldatapb.Keyspace{ + { + Name: "commerce", + Keyspace: &topodatapb.Keyspace{}, + }, + }, + shards: []*vtctldatapb.Shard{ + { + Name: "-", + Keyspace: "commerce", + }, + }, + srvVSchema: &vschemapb.SrvVSchema{ + Keyspaces: map[string]*vschemapb.Keyspace{ + "commerce": { + Sharded: false, + Tables: map[string]*vschemapb.Table{ + "customers": {}, + }, + }, + }, + RoutingRules: &vschemapb.RoutingRules{ + Rules: []*vschemapb.RoutingRule{}, + }, + }, + tabletSchemas: map[string]*tabletmanagerdatapb.SchemaDefinition{ + "c0_cell1-0000000102": { + DatabaseSchema: "CREATE DATABASE commerce", + TableDefinitions: []*tabletmanagerdatapb.TableDefinition{ + { + Name: "t1", + Schema: `CREATE TABLE customers (id int(11) not null,PRIMARY KEY (id));`, + Type: "BASE", + Columns: []string{"id"}, + DataLength: 100, + RowCount: 50, + Fields: []*querypb.Field{ + { + Name: "id", + Type: querypb.Type_INT32, + }, + }, + }, + }, + }, + }, + tablets: []*vtadminpb.Tablet{ + { + Cluster: &vtadminpb.Cluster{ + Id: "c0", + Name: "cluster0", + }, + State: vtadminpb.Tablet_SERVING, + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Uid: 100, + Cell: "c0_cell1", + }, + Hostname: "tablet-cell1-a", + Keyspace: "commerce", + Shard: "-", + Type: topodatapb.TabletType_MASTER, + }, + }, + { + Cluster: &vtadminpb.Cluster{ + Id: "c0", + Name: "cluster0", + }, + State: vtadminpb.Tablet_SERVING, + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Uid: 101, + Cell: "c0_cell1", + }, + Hostname: "tablet-cell1-b", + Keyspace: "commerce", + Shard: "-", + Type: topodatapb.TabletType_DRAINED, + }, + }, + { + Cluster: &vtadminpb.Cluster{ + Id: "c0", + Name: "cluster0", + }, + State: vtadminpb.Tablet_NOT_SERVING, + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Uid: 102, + Cell: "c0_cell1", + }, + Hostname: "tablet-cell1-c", + Keyspace: "commerce", + Shard: "-", + Type: topodatapb.TabletType_REPLICA, + }, + }, + }, + req: &vtadminpb.VTExplainRequest{ + Cluster: "c0", + Keyspace: "commerce", + Sql: "select * from customers", + }, + expectedError: ErrNoTablet, + }, + { + name: "returns an error if cluster unspecified in request", + req: &vtadminpb.VTExplainRequest{ + Keyspace: "commerce", + Sql: "select * from customers", + }, + expectedError: ErrInvalidRequest, + }, + { + name: "returns an error if keyspace unspecified in request", + req: &vtadminpb.VTExplainRequest{ + Cluster: "c0", + Sql: "select * from customers", + }, + expectedError: ErrInvalidRequest, + }, + { + name: "returns an error if SQL unspecified in request", + req: &vtadminpb.VTExplainRequest{ + Cluster: "c0", + Keyspace: "commerce", + }, + expectedError: ErrInvalidRequest, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + toposerver := memorytopo.NewServer("c0_cell1") + + tmc := testutil.TabletManagerClient{ + GetSchemaResults: map[string]struct { + Schema *tabletmanagerdatapb.SchemaDefinition + Error error + }{}, + } + + vtctldserver := testutil.NewVtctldServerWithTabletManagerClient(t, toposerver, &tmc, func(ts *topo.Server) vtctlservicepb.VtctldServer { + return grpcvtctldserver.NewVtctldServer(ts) + }) + + testutil.WithTestServer(t, vtctldserver, func(t *testing.T, vtctldClient vtctldclient.VtctldClient) { + + toposerver.UpdateSrvVSchema(context.Background(), "c0_cell1", tt.srvVSchema) + testutil.AddKeyspaces(context.Background(), t, toposerver, tt.keyspaces...) + testutil.AddShards(context.Background(), t, toposerver, tt.shards...) + + for _, tablet := range tt.tablets { + testutil.AddTablet(context.Background(), t, toposerver, tablet.Tablet, nil) + + // Adds each SchemaDefinition to the fake TabletManagerClient, or nil + // if there are no schemas for that tablet. (All tablet aliases must + // exist in the map. Otherwise, TabletManagerClient will return an error when + // looking up the schema with tablet alias that doesn't exist.) + alias := topoproto.TabletAliasString(tablet.Tablet.Alias) + tmc.GetSchemaResults[alias] = struct { + Schema *tabletmanagerdatapb.SchemaDefinition + Error error + }{ + Schema: tt.tabletSchemas[alias], + Error: nil, + } + } + + c := buildCluster(0, vtctldClient, tt.tablets, nil) + clusters := []*cluster.Cluster{c} + + api := NewAPI(clusters, grpcserver.Options{}, http.Options{}) + resp, err := api.VTExplain(context.Background(), tt.req) + + if tt.expectedError != nil { + assert.True(t, errors.Is(err, tt.expectedError), "expected error type %w does not match actual error type %w", err, tt.expectedError) + } else { + require.NoError(t, err) + + // We don't particularly care to test the contents of the VTExplain response, + // just that it exists. + assert.NotEmpty(t, resp.Response) + } + }) + }) + } +} + type dbcfg struct { shouldErr bool } diff --git a/go/vt/vtadmin/errors.go b/go/vt/vtadmin/errors.go index 60356e9bde9..efec58e2de1 100644 --- a/go/vt/vtadmin/errors.go +++ b/go/vt/vtadmin/errors.go @@ -22,9 +22,14 @@ var ( // ErrAmbiguousTablet occurs when more than one tablet is found for a given // set of filter criteria. ErrAmbiguousTablet = errors.New("multiple tablets found") + // ErrInvalidRequest occurs when a request is invalid for any reason. + // For example, if mandatory parameters are undefined. + ErrInvalidRequest = errors.New("Invalid request") // ErrNoTablet occurs when a tablet cannot be found for a given set of // filter criteria. ErrNoTablet = errors.New("no such tablet") // ErrUnsupportedCluster occurs when a cluster parameter is invalid. ErrUnsupportedCluster = errors.New("unsupported cluster(s)") + // ErrNoSrvVSchema occurs when no SrvVSchema is found for a given keyspace. + ErrNoSrvVSchema = errors.New("SrvVSchema not found") ) diff --git a/go/vt/vtadmin/http/vtexplain.go b/go/vt/vtadmin/http/vtexplain.go new file mode 100644 index 00000000000..53a70f3b899 --- /dev/null +++ b/go/vt/vtadmin/http/vtexplain.go @@ -0,0 +1,34 @@ +/* +Copyright 2021 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package http + +import ( + "context" + + vtadminpb "vitess.io/vitess/go/vt/proto/vtadmin" +) + +// VTExplain implements the http wrapper for /vtexplain?cluster=&keyspace=&sql= +func VTExplain(ctx context.Context, r Request, api *API) *JSONResponse { + query := r.URL.Query() + res, err := api.server.VTExplain(ctx, &vtadminpb.VTExplainRequest{ + Cluster: query.Get("cluster"), + Keyspace: query.Get("keyspace"), + Sql: query.Get("sql"), + }) + return NewJSONResponse(res, err) +} diff --git a/go/vt/vtgate/scatter_conn.go b/go/vt/vtgate/scatter_conn.go index f2910ed31fc..12565c6af09 100644 --- a/go/vt/vtgate/scatter_conn.go +++ b/go/vt/vtgate/scatter_conn.go @@ -266,7 +266,7 @@ func (stc *ScatterConn) ExecuteMultiShard( return qr, allErrors.GetErrors() } -var errRegx = regexp.MustCompile("transaction ([a-z0-9:]+) ended") +var errRegx = regexp.MustCompile("transaction ([a-z0-9:]+) (?:ended|not found)") func checkAndResetShardSession(info *shardActionInfo, err error, session *SafeSession) bool { if info.reservedID != 0 && info.transactionID == 0 && wasConnectionClosed(err) { diff --git a/go/vt/vtgate/scatter_conn_test.go b/go/vt/vtgate/scatter_conn_test.go index 28baeb77db7..49b85b1123a 100644 --- a/go/vt/vtgate/scatter_conn_test.go +++ b/go/vt/vtgate/scatter_conn_test.go @@ -295,12 +295,64 @@ func TestReservedConnFail(t *testing.T) { session := NewSafeSession(&vtgatepb.Session{InTransaction: false, InReservedConn: true}) destinations := []key.Destination{key.DestinationShard("0")} + executeOnShards(t, res, keyspace, sc, session, destinations) assert.Equal(t, 1, len(session.ShardSessions)) oldRId := session.Session.ShardSessions[0].ReservedId + sbc0.EphemeralShardErr = mysql.NewSQLError(mysql.CRServerGone, mysql.SSUnknownSQLState, "lost connection") _ = executeOnShardsReturnsErr(t, res, keyspace, sc, session, destinations) assert.Equal(t, 3, len(sbc0.Queries), "1 for the successful run, one for the failed attempt, and one for the retry") require.Equal(t, 1, len(session.ShardSessions)) assert.NotEqual(t, oldRId, session.Session.ShardSessions[0].ReservedId, "should have recreated a reserved connection since the last connection was lost") + oldRId = session.Session.ShardSessions[0].ReservedId + + sbc0.Queries = nil + sbc0.EphemeralShardErr = mysql.NewSQLError(mysql.ERQueryInterrupted, mysql.SSUnknownSQLState, "transaction 123 not found") + _ = executeOnShardsReturnsErr(t, res, keyspace, sc, session, destinations) + assert.Equal(t, 2, len(sbc0.Queries), "one for the failed attempt, and one for the retry") + require.Equal(t, 1, len(session.ShardSessions)) + assert.NotEqual(t, oldRId, session.Session.ShardSessions[0].ReservedId, "should have recreated a reserved connection since the last connection was lost") + oldRId = session.Session.ShardSessions[0].ReservedId + + sbc0.Queries = nil + sbc0.EphemeralShardErr = mysql.NewSQLError(mysql.ERQueryInterrupted, mysql.SSUnknownSQLState, "transaction 123 ended at 2020-01-20") + _ = executeOnShardsReturnsErr(t, res, keyspace, sc, session, destinations) + assert.Equal(t, 2, len(sbc0.Queries), "one for the failed attempt, and one for the retry") + require.Equal(t, 1, len(session.ShardSessions)) + assert.NotEqual(t, oldRId, session.Session.ShardSessions[0].ReservedId, "should have recreated a reserved connection since the last connection was lost") +} + +func TestIsConnClosed(t *testing.T) { + var testCases = []struct { + name string + err error + conClosed bool + }{{ + "server gone", + mysql.NewSQLError(mysql.CRServerGone, mysql.SSServerShutdown, ""), + true, + }, { + "connection lost", + mysql.NewSQLError(mysql.CRServerLost, mysql.SSServerShutdown, ""), + true, + }, { + "tx ended", + mysql.NewSQLError(mysql.ERQueryInterrupted, mysql.SSUnknownSQLState, "transaction 111 ended at ..."), + true, + }, { + "tx not found", + mysql.NewSQLError(mysql.ERQueryInterrupted, mysql.SSUnknownSQLState, "transaction 111 not found ..."), + true, + }, { + "tx not found missing tx id", + mysql.NewSQLError(mysql.ERQueryInterrupted, mysql.SSUnknownSQLState, "transaction not found"), + false, + }} + + for _, tCase := range testCases { + t.Run(tCase.name, func(t *testing.T) { + assert.Equal(t, tCase.conClosed, wasConnectionClosed(tCase.err)) + }) + } } diff --git a/go/vt/vttablet/endtoend/framework/testcase.go b/go/vt/vttablet/endtoend/framework/testcase.go index c025aebbac1..fc093a76966 100644 --- a/go/vt/vttablet/endtoend/framework/testcase.go +++ b/go/vt/vttablet/endtoend/framework/testcase.go @@ -108,6 +108,9 @@ func (tc *TestCase) Test(name string, client *QueryClient) error { name = tc.Name } + // wait for all previous test cases to have been settled in cache + client.server.QueryPlanCacheWait() + catcher := NewQueryCatcher() defer catcher.Close() diff --git a/go/vt/vttablet/endtoend/queries_test.go b/go/vt/vttablet/endtoend/queries_test.go index 4e3bf4ff6bb..6e2017fa02f 100644 --- a/go/vt/vttablet/endtoend/queries_test.go +++ b/go/vt/vttablet/endtoend/queries_test.go @@ -18,7 +18,6 @@ package endtoend import ( "testing" - "time" "github.com/stretchr/testify/require" @@ -1785,9 +1784,6 @@ func TestQueries(t *testing.T) { }, } - // Wait for the vtgate caches to flush - time.Sleep(1 * time.Second) - for _, tcase := range testCases { if err := tcase.Test("", client); err != nil { t.Error(err) diff --git a/go/vt/vttablet/tabletserver/query_executor_test.go b/go/vt/vttablet/tabletserver/query_executor_test.go index bc2f3ea4e51..95dd523adc5 100644 --- a/go/vt/vttablet/tabletserver/query_executor_test.go +++ b/go/vt/vttablet/tabletserver/query_executor_test.go @@ -301,6 +301,9 @@ func TestQueryExecutorPlans(t *testing.T) { assert.Equal(t, tcase.planWant, qre.logStats.PlanType, tcase.input) assert.Equal(t, tcase.logWant, qre.logStats.RewrittenSQL(), tcase.input) + // Wait for the existing query to be processed by the cache + tsv.QueryPlanCacheWait() + // Test inside a transaction. target := tsv.sm.Target() txid, alias, err := tsv.Begin(ctx, &target, nil) diff --git a/go/vt/vttablet/tabletserver/tabletserver.go b/go/vt/vttablet/tabletserver/tabletserver.go index 1d2a3627d86..40085bdb766 100644 --- a/go/vt/vttablet/tabletserver/tabletserver.go +++ b/go/vt/vttablet/tabletserver/tabletserver.go @@ -1742,6 +1742,11 @@ func (tsv *TabletServer) QueryPlanCacheLen() int { return tsv.qe.QueryPlanCacheLen() } +// QueryPlanCacheWait waits until the query plan cache has processed all recent queries +func (tsv *TabletServer) QueryPlanCacheWait() { + tsv.qe.plans.Wait() +} + // SetMaxResultSize changes the max result size to the specified value. func (tsv *TabletServer) SetMaxResultSize(val int) { tsv.qe.maxResultSize.Set(int64(val)) diff --git a/proto/vtadmin.proto b/proto/vtadmin.proto index efb4644a350..12cdb940a37 100644 --- a/proto/vtadmin.proto +++ b/proto/vtadmin.proto @@ -43,6 +43,8 @@ service VTAdmin { rpc GetTablet(GetTabletRequest) returns (Tablet) {}; // GetTablets returns all tablets across all the specified clusters. rpc GetTablets(GetTabletsRequest) returns (GetTabletsResponse) {}; + // VTExplain provides information on how Vitess plans to execute a particular query. + rpc VTExplain(VTExplainRequest) returns (VTExplainResponse) {}; } /* Data types */ @@ -151,3 +153,13 @@ message GetTabletsRequest { message GetTabletsResponse { repeated Tablet tablets = 1; } + +message VTExplainRequest { + string cluster = 1; + string keyspace = 2; + string sql = 3; +} + +message VTExplainResponse { + string response = 1; +} diff --git a/tools/check_make_sizegen.sh b/tools/check_make_sizegen.sh new file mode 100755 index 00000000000..edcff23a5e3 --- /dev/null +++ b/tools/check_make_sizegen.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# +# Validate that the current version of the generated cache_size files match the output +# generated by sizegen. +# +# This is used in Travis to verify that the currently committed version was +# generated with the proper cache_size files. + +source build.env + +TMP="/tmp/cached_size.$$.go" +ALL_FILES=$(find . -name "cached_size.go") + +set +e + +for SRC in $ALL_FILES +do + TMP="/tmp/"$(echo "$SRC" | sed 's/\//_/g' | sed "s/cached_size.go/cached_size_$$.go/g") + mv "$SRC" "$TMP" +done + +make sizegen + +STATUS=0 + +for SRC in $ALL_FILES +do + TMP="/tmp/"$(echo "$SRC" | sed 's/\//_/g' | sed "s/cached_size.go/cached_size_$$.go/g") + + if [ ! -f "$SRC" ]; then + mv "$TMP" "$SRC" + continue + fi + + if ! diff -q "$SRC" "$TMP" > /dev/null ; then + echo "ERROR: Regenerated file for $SRC does not match the current version:" + diff -u "$SRC" "$TMP" + + echo + echo "Please re-run 'make sizegen' to generate." + STATUS=1 + fi + mv "$TMP" "$SRC" +done + +exit $STATUS + diff --git a/web/orchestrator/public/js/orchestrator.js b/web/orchestrator/public/js/orchestrator.js index 62316744a36..b00f11f8869 100644 --- a/web/orchestrator/public/js/orchestrator.js +++ b/web/orchestrator/public/js/orchestrator.js @@ -76,6 +76,19 @@ function isCompactDisplay() { return ($.cookie("compact-display") == "true"); } +// origin: https://vanillajstoolkit.com/ +/** + * Sanitize and encode all HTML in a user-submitted string + * https://portswigger.net/web-security/cross-site-scripting/preventing + * @param {String} str The user-submitted string + * @return {String} str The sanitized string + */ +function sanitizeHTML (str) { + return str.replace(/[^\w-_. ]/gi, function (c) { + return '&#' + c.charCodeAt(0) + ';'; + }); +} + function anonymizeInstanceId(instanceId) { var tokens = instanceId.split("__"); return "instance-" + md5(tokens[1]).substring(0, 4) + ":" + tokens[2]; @@ -1133,7 +1146,7 @@ $(document).ready(function() { $("[data-nav-page=user-id]").css('display', 'inline-block'); $("[data-nav-page=user-id] a").html(" " + getUserId()); } - var orchestratorMsg = getParameterByName("orchestrator-msg") + var orchestratorMsg = sanitizeHTML(getParameterByName("orchestrator-msg")) if (orchestratorMsg) { addInfo(orchestratorMsg) diff --git a/web/vtadmin/package-lock.json b/web/vtadmin/package-lock.json index cc66d480c86..3e44e8a070c 100644 --- a/web/vtadmin/package-lock.json +++ b/web/vtadmin/package-lock.json @@ -4521,6 +4521,11 @@ } } }, + "compute-scroll-into-view": { + "version": "1.0.16", + "resolved": "https://registry.npmjs.org/compute-scroll-into-view/-/compute-scroll-into-view-1.0.16.tgz", + "integrity": "sha512-a85LHKY81oQnikatZYA90pufpZ6sQx++BoCxOEMsjpZx+ZnaKGQnCyCehTRr/1p9GBIAHTjcU9k71kSYWloLiQ==" + }, "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -5528,6 +5533,24 @@ "resolved": "https://registry.npmjs.org/dotenv-expand/-/dotenv-expand-5.1.0.tgz", "integrity": "sha512-YXQl1DSa4/PQyRfgrv6aoNjhasp/p4qs9FjJ4q4cQk+8m4r6k4ZSiEyytKG8f8W9gi8WsQtIObNmKd+tMzNTmA==" }, + "downshift": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/downshift/-/downshift-6.1.0.tgz", + "integrity": "sha512-MnEJERij+1pTVAsOPsH3q9MJGNIZuu2sT90uxOCEOZYH6sEzkVGtUcTBVDRQkE8y96zpB7uEbRn24aE9VpHnZg==", + "requires": { + "@babel/runtime": "^7.12.5", + "compute-scroll-into-view": "^1.0.16", + "prop-types": "^15.7.2", + "react-is": "^17.0.1" + }, + "dependencies": { + "react-is": { + "version": "17.0.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.1.tgz", + "integrity": "sha512-NAnt2iGDXohE5LI7uBnLnqvLQMtzhkiAOLXTmv+qnF9Ky7xAPcX8Up/xWIhxvLVGJvuLiNc4xQLtuqDRzb4fSA==" + } + } + }, "duplexer": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.2.tgz", diff --git a/web/vtadmin/package.json b/web/vtadmin/package.json index 8982ee2aac9..12ec97abd53 100644 --- a/web/vtadmin/package.json +++ b/web/vtadmin/package.json @@ -15,6 +15,7 @@ "@types/react-dom": "^16.9.10", "@types/react-router-dom": "^5.1.7", "classnames": "^2.2.6", + "downshift": "^6.1.0", "history": "^5.0.0", "lodash-es": "^4.17.20", "node-sass": "^4.14.1", @@ -59,6 +60,11 @@ "last 1 safari version" ] }, + "jest": { + "transformIgnorePatterns": [ + "/!node_modules\\/lodash-es/" + ] + }, "devDependencies": { "@testing-library/jest-dom": "^5.11.9", "@testing-library/react": "^11.2.5", diff --git a/web/vtadmin/src/components/TextInput.module.scss b/web/vtadmin/src/components/TextInput.module.scss index 5c3e67b4448..74a061ae610 100644 --- a/web/vtadmin/src/components/TextInput.module.scss +++ b/web/vtadmin/src/components/TextInput.module.scss @@ -1,5 +1,5 @@ .inputContainer { - display: inline-block; + display: block; position: relative; } diff --git a/web/vtadmin/src/components/TextInput.tsx b/web/vtadmin/src/components/TextInput.tsx index 5b567c79819..4a530cc38df 100644 --- a/web/vtadmin/src/components/TextInput.tsx +++ b/web/vtadmin/src/components/TextInput.tsx @@ -15,10 +15,10 @@ interface Props extends NativeInputProps { className?: string; iconLeft?: Icons; iconRight?: Icons; - size?: 'large' | 'medium'; // We have no need for small inputs right now. + size?: 'large'; } -export const TextInput = ({ className, iconLeft, iconRight, size = 'medium', ...props }: Props) => { +export const TextInput = ({ className, iconLeft, iconRight, size, ...props }: Props) => { const inputClass = cx(style.input, { [style.large]: size === 'large', [style.withIconLeft]: !!iconLeft, diff --git a/web/vtadmin/src/components/inputs/Label.module.scss b/web/vtadmin/src/components/inputs/Label.module.scss new file mode 100644 index 00000000000..517162d3983 --- /dev/null +++ b/web/vtadmin/src/components/inputs/Label.module.scss @@ -0,0 +1,19 @@ +/** + * Copyright 2021 The Vitess Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +.label { + font-weight: 700; + line-height: 3.2rem; +} diff --git a/web/vtadmin/src/components/inputs/Label.tsx b/web/vtadmin/src/components/inputs/Label.tsx new file mode 100644 index 00000000000..5bba38da553 --- /dev/null +++ b/web/vtadmin/src/components/inputs/Label.tsx @@ -0,0 +1,33 @@ +/** + * Copyright 2021 The Vitess Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import * as React from 'react'; + +import style from './Label.module.scss'; + +type NativeLabelProps = React.DetailedHTMLProps, HTMLLabelElement>; + +interface Props extends NativeLabelProps { + label: string; +} + +export const Label: React.FunctionComponent = ({ children, label, ...props }) => { + return ( + + ); +}; diff --git a/web/vtadmin/src/components/inputs/Select.module.scss b/web/vtadmin/src/components/inputs/Select.module.scss new file mode 100644 index 00000000000..d5fd9ca72e2 --- /dev/null +++ b/web/vtadmin/src/components/inputs/Select.module.scss @@ -0,0 +1,151 @@ +/** + * Copyright 2021 The Vitess Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +.container { + display: inline-block; + position: relative; +} + +.toggle { + background: var(--backgroundPrimary); + border: solid 2px var(--colorDisabled); + border-radius: 6px; + box-sizing: border-box; + color: var(--textColorPrimary); + cursor: pointer; + display: block; + font-family: var(--fontFamilyPrimary); + font-size: var(--fontSizeBody); + height: var(--inputHeightMedium); + min-width: 16rem; + padding: 0 40px 0 12px; + position: relative; + text-align: left; + transition: all 0.1s ease-in-out; + white-space: nowrap; +} + +.placeholder .toggle { + color: var(--textColorSecondary); +} + +.open .toggle, +.toggle:active, +.toggle:focus { + border-color: var(--colorPrimary); + outline: none; +} + +.toggle:disabled { + background: var(--backgroundSecondary); + border-color: var(--backgroundSecondaryHighlight); + color: var(--textColorSecondary); + cursor: not-allowed; +} + +.large .toggle { + font-size: var(--fontSizeLarge); + height: var(--inputHeightLarge); + min-width: 24rem; + padding: 0 16px; +} + +.chevron { + height: 20px; + position: absolute; + top: calc(50% - 10px); + right: 4px; +} + +.dropdown { + background: var(--backgroundPrimary); + border: solid 2px var(--colorDisabled); + border-radius: 6px; + box-sizing: border-box; + margin: 4px 0 0 0; + height: min-content; + max-height: 420px; + overflow: auto; + outline: none; + padding: 8px 0; + min-width: 100%; + position: absolute; + z-index: 1000; +} + +.menu { + list-style-type: none; + margin: 0; + outline: none; + padding: 0; +} + +.menu li { + line-height: 32px; + padding: 4px 12px; + + &:hover { + background: var(--backgroundPrimaryHighlight); + cursor: pointer; + } + + &.active { + background: var(--backgroundPrimaryHighlight); + } +} + +.large .menu li { + font-size: var(--fontSizeLarge); + min-width: 24rem; + padding: 8px 16px; +} + +.clear { + background: none; + border: none; + box-sizing: border-box; + color: var(--textColorSecondary); + cursor: pointer; + display: block; + font-family: var(--fontFamilyPrimary); + font-size: var(--fontSizeBody); + min-width: 16rem; + padding: 4px 12px; + position: relative; + text-align: left; + transition: all 0.1s ease-in-out; + white-space: nowrap; + width: 100%; + + &:hover, + &:active, + &:focus { + background: var(--backgroundPrimaryHighlight); + } +} + +.large .clear { + font-size: var(--fontSizeLarge); + padding: 8px 16px; +} + +.emptyContainer { + outline: none; + padding: 8px 12px; +} + +.emptyPlaceholder { + color: var(--textColorSecondary); +} diff --git a/web/vtadmin/src/components/inputs/Select.tsx b/web/vtadmin/src/components/inputs/Select.tsx new file mode 100644 index 00000000000..4572d83612a --- /dev/null +++ b/web/vtadmin/src/components/inputs/Select.tsx @@ -0,0 +1,148 @@ +/** + * Copyright 2021 The Vitess Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import cx from 'classnames'; +import { useSelect, UseSelectStateChange } from 'downshift'; +import * as React from 'react'; + +import { Label } from './Label'; +import style from './Select.module.scss'; +import { Icon, Icons } from '../Icon'; + +interface Props { + disabled?: boolean; + items: T[]; + itemToString?: (item: T | null) => string; + label: string; + onChange: (selectedItem: T | null | undefined) => void; + placeholder: string; + emptyPlaceholder?: string | (() => JSX.Element | string); + renderItem?: (item: T) => JSX.Element | string; + selectedItem: T | null; + size?: 'large'; +} + +/** + * Select performs exactly the same as the native HTML setTheme(t)} - type="radio" - value={t} - /> - {t} - - - ))} - +
+

Theme

+
+ {Object.values(Theme).map((t) => ( +
+ +
+ ))} +
+
-

Icons

-
- {Object.values(Icons).map((i) => ( - - ))} -
+
+

Icons

+
+ {Object.values(Icons).map((i) => ( + + ))} +
+
-

Text Inputs

-
- - - - - - -
+
+

Select

+
+
+ setFormData({ ...formData, selectFruitNameDefault: fruitName })} + placeholder="Choose a fruit name" + selectedItem={formData.selectFruitNameDefault || null} + /> + setFormData({ ...formData, selectFruitDefault: fruit })} + placeholder="Choose a fruit" + renderItem={(fruit) => `${fruit.emoji} ${fruit.name}`} + selectedItem={formData.selectFruitDefault || null} + /> +
+
+ setFormData({ ...formData, selectFruitNameLarge: fruitName })} + placeholder="Choose a fruit name" + size="large" + selectedItem={formData.selectFruitNameLarge || null} + /> +