Skip to content

Commit

Permalink
psdbconnect.v1 prep
Browse files Browse the repository at this point in the history
This reorganized everything and yanks a bunch from the previous
psdb.v1beta1 branch.

Also swaps to buf CLI instead of protoc
  • Loading branch information
mattrobenolt committed Feb 27, 2023
1 parent 77bbe10 commit 06f7cdd
Show file tree
Hide file tree
Showing 17 changed files with 1,603 additions and 188 deletions.
3 changes: 3 additions & 0 deletions .gitmodules
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
[submodule "thirdparty/vitess"]
path = thirdparty/vitess
url = https://github.com/planetscale/vitess-types
164 changes: 77 additions & 87 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,116 +1,102 @@
gomod := github.com/planetscale/psdb

PSDB_PROTO_OUT := types
PSDB_PROTO_ROOT := $(PSDB_PROTO_OUT)/psdb
PSDB_DATA_V1 := $(PSDB_PROTO_ROOT)/data/v1
PSDB_V1ALPHA1 := $(PSDB_PROTO_ROOT)/v1alpha1
PROTO_OUT := types
PSDB_OUT := $(PROTO_OUT)/psdb
PSDBDATA_OUT := $(PSDB_OUT)/data
PSDBCONNECT_OUT := $(PROTO_OUT)/psdbconnect

PROTOC_VERSION=21.3
PSDBDATAV1_OUT := $(PSDBDATA_OUT)/v1
PSDBV1ALPHA1_OUT := $(PSDB_OUT)/v1alpha1
PSDBCONNECTV1_OUT := $(PSDBCONNECT_OUT)/v1

BIN := bin
PROTO_SRC := proto-src
PSDB_SRC := $(PROTO_SRC)/psdb
PSDBDATA_SRC := $(PSDB_SRC)/data
PSDBCONNECT_SRC := $(PROTO_SRC)/psdbconnect

UNAME_OS := $(shell uname -s)
UNAME_ARCH := $(shell uname -m)
PSDBDATAV1_SRC := $(PSDBDATA_SRC)/v1
PSDBV1ALPHA1_SRC := $(PSDB_SRC)/v1alpha1
PSDBCONNECTV1_SRC := $(PSDBCONNECT_SRC)/v1

proto: \
$(PSDB_DATA_V1)/data.pb.go \
$(PSDB_V1ALPHA1)/database.pb.go
BIN := bin

clean: clean-proto clean-bin

clean-proto:
rm -rf $(PSDB_PROTO_OUT)
rm -rf $(PROTO_OUT)

clean-bin:
rm -rf $(BIN)

$(BIN):
mkdir -p $(BIN)

$(PSDB_PROTO_OUT):
mkdir -p $(PSDB_PROTO_OUT)
$(PROTO_OUT):
mkdir -p $(PROTO_OUT)

TOOL_INSTALL := cd tools && env GOBIN=$(PWD)/$(BIN) go install
TOOL_INSTALL := env GOBIN=$(PWD)/$(BIN) go install

$(BIN)/protoc-gen-go: | $(BIN)
$(TOOL_INSTALL) google.golang.org/protobuf/cmd/protoc-gen-go
$(BIN)/protoc-gen-go: Makefile | $(BIN)
$(TOOL_INSTALL) google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.1

$(BIN)/protoc-gen-connect-go: | $(BIN)
$(TOOL_INSTALL) github.com/bufbuild/connect-go/cmd/protoc-gen-connect-go
$(BIN)/protoc-gen-go-vtproto: Makefile | $(BIN)
$(TOOL_INSTALL) github.com/planetscale/vtprotobuf/cmd/protoc-gen-go-vtproto@v0.4.0

$(BIN)/protoc-gen-go-vtproto: | $(BIN)
$(TOOL_INSTALL) github.com/planetscale/vtprotobuf/cmd/protoc-gen-go-vtproto
$(BIN)/protoc-gen-connect-go: Makefile | $(BIN)
$(TOOL_INSTALL) github.com/bufbuild/connect-go/cmd/protoc-gen-connect-go@v1.5.1

$(BIN)/gofumpt: | $(BIN)
$(TOOL_INSTALL) mvdan.cc/gofumpt
$(BIN)/gofumpt: Makefile | $(BIN)
$(TOOL_INSTALL) mvdan.cc/gofumpt@v0.4.0

$(BIN)/staticcheck: | $(BIN)
$(TOOL_INSTALL) honnef.co/go/tools/cmd/staticcheck
$(BIN)/staticcheck: Makefile | $(BIN)
$(TOOL_INSTALL) honnef.co/go/tools/cmd/staticcheck@v0.3.3

$(BIN)/enumcheck: | $(BIN)
$(TOOL_INSTALL) loov.dev/enumcheck
$(BIN)/enumcheck: Makefile | $(BIN)
$(TOOL_INSTALL) loov.dev/enumcheck@v0.0.0-20220314183541-8aa7b787306e

ifeq ($(UNAME_OS),Darwin)
PROTOC_OS := osx
ifeq ($(UNAME_ARCH),arm64)
PROTOC_ARCH := aarch_64
else
PROTOC_ARCH := x86_64
endif
endif
ifeq ($(UNAME_OS),Linux)
PROTOC_OS = linux
ifeq ($(UNAME_ARCH),aarch64)
PROTOC_ARCH := aarch_64
else
PROTOC_ARCH := $(UNAME_ARCH)
endif
endif
$(BIN)/govulncheck: Makefile | $(BIN)
$(TOOL_INSTALL) golang.org/x/vuln/cmd/govulncheck@v0.0.0-20230110180137-6ad3e3d07815

$(BIN)/protoc: | $(BIN)
rm -rf tmp-protoc
mkdir -p tmp-protoc
wget -O tmp-protoc/protoc.zip https://github.com/protocolbuffers/protobuf/releases/download/v$(PROTOC_VERSION)/protoc-$(PROTOC_VERSION)-$(PROTOC_OS)-$(PROTOC_ARCH).zip
unzip -d tmp-protoc tmp-protoc/protoc.zip
mv tmp-protoc/bin/protoc $(BIN)/
rm -rf tmp-protoc

PROTO_TOOLS := $(BIN)/protoc $(BIN)/protoc-gen-go $(BIN)/protoc-gen-connect-go $(BIN)/protoc-gen-go-vtproto
tools: $(PROTO_TOOLS) $(BIN)/gofumpt $(BIN)/staticcheck $(BIN)/enumcheck

$(PSDB_DATA_V1)/data.pb.go: $(PROTO_TOOLS) proto-src/psdb/data/v1/data.proto | $(PSDB_PROTO_OUT)
$(BIN)/protoc \
--plugin=protoc-gen-go=$(BIN)/protoc-gen-go \
--plugin=protoc-gen-go-vtproto=$(BIN)/protoc-gen-go-vtproto \
--go_out=$(PSDB_PROTO_OUT) \
--go-vtproto_out=$(PSDB_PROTO_OUT) \
--go_opt=paths=source_relative \
--go-vtproto_opt=features=marshal+unmarshal+size \
--go-vtproto_opt=paths=source_relative \
-I proto-src \
proto-src/psdb/data/v1/data.proto

$(PSDB_V1ALPHA1)/database.pb.go: $(PROTO_TOOLS) proto-src/psdb/v1alpha1/database.proto | $(PSDB_PROTO_OUT)
$(BIN)/protoc \
--plugin=protoc-gen-go=$(BIN)/protoc-gen-go \
--plugin=protoc-gen-go-vtproto=$(BIN)/protoc-gen-go-vtproto \
--plugin=protoc-gen-connect-go=$(BIN)/protoc-gen-connect-go \
--go_out=$(PSDB_PROTO_OUT) \
--go-vtproto_out=$(PSDB_PROTO_OUT) \
--connect-go_out=$(PSDB_PROTO_OUT) \
--go_opt=paths=source_relative \
--go-vtproto_opt=features=marshal+unmarshal+size \
--go-vtproto_opt=paths=source_relative \
--connect-go_opt=paths=source_relative \
-I proto-src \
proto-src/psdb/v1alpha1/database.proto

fmt: fmt-go
$(BIN)/buf: Makefile | $(BIN)
$(TOOL_INSTALL) github.com/bufbuild/buf/cmd/buf@v1.14.0

$(BIN)/yq: Makefile | $(BIN)
$(TOOL_INSTALL) github.com/mikefarah/yq/v4@v4.30.8

PROTO_TOOLS := $(BIN)/protoc-gen-go $(BIN)/protoc-gen-connect-go $(BIN)/protoc-gen-go-vtproto $(BIN)/buf
tools: $(PROTO_TOOLS) $(BIN)/gofumpt $(BIN)/staticcheck $(BIN)/enumcheck $(BIN)/govulncheck $(BIN)/yq

proto: \
$(PSDBCONNECTV1_OUT)/connect.pb.go \
$(PSDBDATAV1_OUT)/data.pb.go \
$(PSDBV1ALPHA1_OUT)/database.pb.go

$(PSDBCONNECTV1_OUT)/connect.pb.go: $(PROTO_TOOLS) $(PSDBCONNECTV1_SRC)/connect.proto | $(PROTO_OUT)
$(BIN)/buf generate --path $(PSDBCONNECTV1_SRC)/connect.proto

$(PSDBDATAV1_OUT)/data.pb.go: $(PROTO_TOOLS) $(PSDBDATAV1_SRC)/data.proto | $(PROTO_OUT)
$(BIN)/buf generate --path $(PSDBDATAV1_SRC)/data.proto

$(PSDBV1ALPHA1_OUT)/database.pb.go: $(PROTO_TOOLS) $(PSDBV1ALPHA1_SRC)/database.proto | $(PROTO_OUT)
$(BIN)/buf generate --path $(PSDBV1ALPHA1_SRC)/database.proto

fmt: fmt-go fmt-proto

fmt-go: $(BIN)/gofumpt
$(BIN)/gofumpt -l -w .

lint: lint-vet lint-staticcheck lint-enumcheck
fmt-proto: $(BIN)/buf
$(BIN)/buf format -w proto-src

fmt-yaml: $(BIN)/yq
ifeq (, $(shell command -v fd 2>/dev/null))
@echo "!! Maybe install 'fd', it's a lot faster (https://github.com/sharkdp/fd)"
find . -type f \( -name '*.yaml' -o -name '*.yml' \) -exec $(BIN)/yq -iP eval-all . {} \;
else
fd . -t f -e yaml -e yml -x $(BIN)/yq -iP eval-all . {} \;
endif

lint: lint-vet lint-staticcheck lint-enumcheck lint-govulncheck lint-proto

lint-vet:
go vet ./...
Expand All @@ -121,18 +107,22 @@ lint-staticcheck: $(BIN)/staticcheck
lint-enumcheck: $(BIN)/enumcheck
$(BIN)/enumcheck ./...

lint-govulncheck: $(BIN)/govulncheck
$(BIN)/govulncheck ./...

lint-proto: $(BIN)/buf
$(BIN)/buf lint -v proto-src

tests:
go test -v ./...

update:
go get -v -u ./...
go mod tidy
cd tools && go get -v -u ./internal
cd tools && go mod tidy
$(MAKE) clean proto

.PHONY: proto tools update \
clean clean-proto clean-bin \
fmt fmt-go \
lint lint-vet lint-staticcheck lint-enumcheck \
fmt fmt-go fmt-proto fmt-yaml \
lint lint-vet lint-staticcheck lint-enumcheck lint-govulncheck lint-proto \
tests
18 changes: 18 additions & 0 deletions buf.gen.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
version: v1
plugins:
- name: go
path: bin/protoc-gen-go
out: types
opt:
- paths=source_relative
- name: go-vtproto
path: bin/protoc-gen-go-vtproto
out: types
opt:
- paths=source_relative
- features=marshal+unmarshal+size
- name: connect-go
path: bin/protoc-gen-connect-go
out: types
opt:
- paths=source_relative
4 changes: 4 additions & 0 deletions buf.work.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
version: v1
directories:
- proto-src
- thirdparty/vitess/src/vitess/v15
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ go 1.19
require (
github.com/bufbuild/connect-go v1.5.1
github.com/klauspost/connect-compress v1.0.0
github.com/planetscale/vitess-types v0.0.0-20230210165505-c0e8b7520c81
github.com/segmentio/asm v1.2.0
github.com/stretchr/testify v1.7.0
google.golang.org/protobuf v1.28.1
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7y
github.com/klauspost/compress v1.15.15/go.mod h1:ZcK2JAFqKOpnBlxcLsJzYfrS9X1akm9fHZNnD9+Vo/4=
github.com/klauspost/connect-compress v1.0.0 h1:ta/pUrREWnlm9VzmNKDAyn09pQnEsQPta6No9gWmtb4=
github.com/klauspost/connect-compress v1.0.0/go.mod h1:cATACCZtbGMcHcyqT3dqbJh6CDmyUNcE/sE2Qa6CVLw=
github.com/planetscale/vitess-types v0.0.0-20230210165505-c0e8b7520c81 h1:ZZmeb7vzWLDAUl6grl/wbfCQhcDWIUCITW7sCw/FSgQ=
github.com/planetscale/vitess-types v0.0.0-20230210165505-c0e8b7520c81/go.mod h1:i8Ym169j/0PLwPf8+2W6qE0Md2EMcO8vGUSt4NtMxso=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=
Expand Down
15 changes: 15 additions & 0 deletions proto-src/buf.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
version: v1
breaking:
use:
- FILE
lint:
use:
- DEFAULT
except:
- ENUM_VALUE_UPPER_SNAKE_CASE
- ENUM_VALUE_PREFIX
- ENUM_ZERO_VALUE_SUFFIX
- SERVICE_SUFFIX
- RPC_REQUEST_RESPONSE_UNIQUE
- RPC_REQUEST_STANDARD_NAME
- RPC_RESPONSE_STANDARD_NAME
37 changes: 37 additions & 0 deletions proto-src/psdbconnect/v1/connect.proto
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
syntax = "proto3";

package psdbconnect.v1;

import "query/query.proto";
import "vtrpc/vtrpc.proto";

option go_package = "github.com/planetscale/psdb/types/psdbconnect/v1;psdbconnectv1";

//enumcheck:exhaustive
enum TabletType {
replica = 0;
primary = 1;
}

message SyncRequest {
string table_name = 1;
TableCursor cursor = 2;
TabletType tablet_type = 3;
}

message SyncResponse {
repeated query.QueryResult result = 1;
TableCursor cursor = 2;
vtrpc.RPCError error = 3;
}

message TableCursor {
string shard = 1;
string keyspace = 2;
string position = 3;
query.QueryResult last_known_pk = 4;
}

service Connect {
rpc Sync(SyncRequest) returns (stream SyncResponse) {}
}
1 change: 1 addition & 0 deletions thirdparty/vitess
Submodule vitess added at c0e8b7
22 changes: 0 additions & 22 deletions tools/go.mod

This file was deleted.

Loading

0 comments on commit 06f7cdd

Please sign in to comment.