Skip to content

Commit

Permalink
Migrate to memefish (#121)
Browse files Browse the repository at this point in the history
* Migrate to memefish

* Update testdata

* Do go mod tidy

* Update to test comments in DDL

* Update to reduce useless use of gsqlutils
  • Loading branch information
apstndb authored Nov 20, 2024
1 parent 3a330a0 commit 9432313
Show file tree
Hide file tree
Showing 10 changed files with 83 additions and 39 deletions.
10 changes: 8 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@
module github.com/cloudspannerecosystem/wrench

go 1.21
go 1.23.1

toolchain go1.23.2

require (
cloud.google.com/go/spanner v1.64.0
github.com/apstndb/gsqlutils v0.0.0-20241110070457-b8882df29778
github.com/google/uuid v1.6.0
github.com/hashicorp/go-multierror v1.1.1
github.com/spf13/cobra v1.7.0
Expand All @@ -21,6 +24,7 @@ require (
github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.5.0 // indirect
github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/cloudspannerecosystem/memefish v0.0.0-20241106111047-2b2b4b23a1e7 // indirect
github.com/cncf/xds/go v0.0.0-20240318125728-8a4994d93e50 // indirect
github.com/envoyproxy/go-control-plane v0.12.0 // indirect
github.com/envoyproxy/protoc-gen-validate v1.0.4 // indirect
Expand All @@ -34,6 +38,7 @@ require (
github.com/googleapis/gax-go/v2 v2.12.5 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/samber/lo v1.47.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
Expand All @@ -45,12 +50,13 @@ require (
golang.org/x/net v0.26.0 // indirect
golang.org/x/oauth2 v0.21.0 // indirect
golang.org/x/sync v0.7.0 // indirect
golang.org/x/sys v0.21.0 // indirect
golang.org/x/sys v0.26.0 // indirect
golang.org/x/text v0.16.0 // indirect
golang.org/x/time v0.5.0 // indirect
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
google.golang.org/genproto v0.0.0-20240617180043-68d350f18fd4 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240617180043-68d350f18fd4 // indirect
google.golang.org/protobuf v1.34.2 // indirect
spheric.cloud/xiter v0.0.0-20240904151420-c999f37a46b2 // indirect
)
19 changes: 17 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -614,6 +614,8 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym
github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.5.0 h1:oVLqHXhnYtUwM89y9T1fXGaK9wTkXHgNp8/ZNMQzUxE=
github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.5.0/go.mod h1:dppbR7CwXD4pgtV9t3wD1812RaLDcBjtblcDF5f1vI0=
github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk=
github.com/MakeNowJust/heredoc/v2 v2.0.1 h1:rlCHh70XXXv7toz95ajQWOWQnN4WNLt0TdpZYIR/J6A=
github.com/MakeNowJust/heredoc/v2 v2.0.1/go.mod h1:6/2Abh5s+hc3g9nbWLe9ObDIOhaRrqsyY9MWy+4JdRM=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY=
github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk=
Expand All @@ -624,6 +626,8 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd
github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0=
github.com/apache/arrow/go/v11 v11.0.0/go.mod h1:Eg5OsL5H+e299f7u5ssuXsuHQVEGC4xei5aX110hRiI=
github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU=
github.com/apstndb/gsqlutils v0.0.0-20241110070457-b8882df29778 h1:eFzpW6G5NzTytQtSnsEIOfH2aAYGnNN5zmbdh2jQFCY=
github.com/apstndb/gsqlutils v0.0.0-20241110070457-b8882df29778/go.mod h1:iy/NytOejQT/jaXcRaYYHthvkTpgWNZxggLVEecd7C4=
github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
Expand All @@ -638,6 +642,8 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cloudspannerecosystem/memefish v0.0.0-20241106111047-2b2b4b23a1e7 h1:vmS9Nvh7ij5EVnMwvkWsL84xzx5cGM3j/SJlM1zfVLE=
github.com/cloudspannerecosystem/memefish v0.0.0-20241106111047-2b2b4b23a1e7/go.mod h1:iYAaNZfVIn4QYfUmXt+3EeHAok/kqpN/fp/8kgDHjx8=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
Expand Down Expand Up @@ -831,6 +837,8 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
github.com/k0kubun/pp v1.3.1-0.20200204103551-99835366d1cc h1:XLjmW07gT7cG/wb6mavIrvAIWBYaTacPo8UOnxGSspA=
github.com/k0kubun/pp v1.3.1-0.20200204103551-99835366d1cc/go.mod h1:qK2ivXw91omfE1uXcpR5kWbAMZRdDOnGbqWlZ7reRFk=
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
Expand All @@ -846,7 +854,10 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA=
github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA=
github.com/lyft/protoc-gen-star/v2 v2.0.1/go.mod h1:RcCdONR2ScXaYnQC5tUzxzlpA3WVYF7/opLeUgcQs/o=
github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY=
Expand All @@ -873,6 +884,8 @@ github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/f
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w=
github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245/go.mod h1:pQAZKsJ8yyVxGRWYNEm9oFB8ieLgKFnamEyDmSA0BRk=
github.com/samber/lo v1.47.0 h1:z7RynLwP5nbyRscyvcD043DWYoOcYRv3mV8lBeqOCLc=
github.com/samber/lo v1.47.0/go.mod h1:RmDH9Ct32Qy3gduHQuKJ3gW1fMHAnE/fAzQuf6He5cU=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4=
github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
Expand Down Expand Up @@ -1181,8 +1194,8 @@ golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
Expand Down Expand Up @@ -1618,3 +1631,5 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
spheric.cloud/xiter v0.0.0-20240904151420-c999f37a46b2 h1:/2/LZep2TpsdSwYhdMsGRh+OVun1UMYlMEip4EEsgg8=
spheric.cloud/xiter v0.0.0-20240904151420-c999f37a46b2/go.mod h1:i4SlkNfFrn1974zbGZWg8FYXAWLnrS6cYAXtSfmIDhU=
2 changes: 1 addition & 1 deletion pkg/spanner/client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ func TestLoadDDL(t *testing.T) {
t.Fatalf("failed to load ddl: %v", err)
}

wantDDL, err := os.ReadFile("testdata/schema.sql")
wantDDL, err := os.ReadFile("testdata/schema_loaded.sql")
if err != nil {
t.Fatalf("failed to read ddl file: %v", err)
}
Expand Down
42 changes: 42 additions & 0 deletions pkg/spanner/memefish.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
package spanner

import (
"github.com/apstndb/gsqlutils"
"github.com/cloudspannerecosystem/memefish"
)

// Directly use of memefish/gsqlutils is permitted only in this file.
func toStatements(filename string, data []byte) ([]string, error) {
rawStmts, err := memefish.SplitRawStatements(filename, string(data))
if err != nil {
return nil, err
}

// need to strip comments because memefish.SplitRawStatements preserve comments, but UpdateDDL doesn't support comments.
var result []string
for _, rawStmt := range rawStmts {
stripped, err := gsqlutils.SimpleStripComments("", rawStmt.Statement)
if err != nil {
return nil, err
}
result = append(result, stripped)
}
return result, nil
}

func isDML(statement string) bool {
token, err := gsqlutils.FirstNonHintToken("", statement)
if err != nil {
return false
}
return token.IsKeywordLike("INSERT")
}

func isPartitionedDML(statement string) bool {
// It is better than regular expression because PDML can be prefixed by statement hints.
token, err := gsqlutils.FirstNonHintToken("", statement)
if err != nil {
return false
}
return token.IsKeywordLike("UPDATE") || token.IsKeywordLike("DELETE")
}
36 changes: 2 additions & 34 deletions pkg/spanner/migration.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,6 @@ import (
"path/filepath"
"regexp"
"strconv"

"cloud.google.com/go/spanner/spansql"
)

var (
Expand All @@ -38,9 +36,6 @@ var (
migrationFileRegex = regexp.MustCompile(`^([0-9]+)(?:_([a-zA-Z0-9_\-]+))?(\.up)?\.sql$`)

MigrationNameRegex = regexp.MustCompile(`[a-zA-Z0-9_\-]+`)

dmlRegex = regexp.MustCompile("^(INSERT)[\t\n\f\r ].*")
partitionedDmlRegex = regexp.MustCompile("^(UPDATE|DELETE)[\t\n\f\r ].*")
)

const (
Expand Down Expand Up @@ -144,31 +139,11 @@ func LoadMigrations(dir string) (Migrations, error) {
}

func ddlToStatements(filename string, data []byte) ([]string, error) {
ddl, err := spansql.ParseDDL(filename, string(data))
if err != nil {
return nil, err
}

var statements []string
for _, stmt := range ddl.List {
statements = append(statements, stmt.SQL())
}

return statements, nil
return toStatements(filename, data)
}

func dmlToStatements(filename string, data []byte) ([]string, error) {
dml, err := spansql.ParseDML(filename, string(data))
if err != nil {
return nil, err
}

var statements []string
for _, stmt := range dml.List {
statements = append(statements, stmt.SQL())
}

return statements, nil
return toStatements(filename, data)
}

func inspectStatementsKind(statements []string) (statementKind, error) {
Expand Down Expand Up @@ -200,10 +175,3 @@ func inspectStatementsKind(statements []string) (statementKind, error) {
}
}

func isDML(statement string) bool {
return dmlRegex.Match([]byte(statement))
}

func isPartitionedDML(statement string) bool {
return partitionedDmlRegex.Match([]byte(statement))
}
1 change: 1 addition & 0 deletions pkg/spanner/testdata/ddl.sql
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
-- Comments must be permitted
ALTER TABLE Singers ADD COLUMN Foo STRING(MAX);
1 change: 1 addition & 0 deletions pkg/spanner/testdata/migrations/000002_test.sql
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
-- Comments must be ignored
ALTER TABLE Singers ADD COLUMN LastName STRING(MAX);
1 change: 1 addition & 0 deletions pkg/spanner/testdata/migrations/000003.sql
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
-- Comments must be permitted
UPDATE Singers SET LastName = "" WHERE LastName IS NULL;
1 change: 1 addition & 0 deletions pkg/spanner/testdata/schema.sql
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
-- Comments must be permitted
CREATE TABLE SchemaMigrations (
Version INT64 NOT NULL,
Dirty BOOL NOT NULL,
Expand Down
9 changes: 9 additions & 0 deletions pkg/spanner/testdata/schema_loaded.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
CREATE TABLE SchemaMigrations (
Version INT64 NOT NULL,
Dirty BOOL NOT NULL,
) PRIMARY KEY(Version);

CREATE TABLE Singers (
SingerID STRING(36) NOT NULL,
FirstName STRING(1024),
) PRIMARY KEY(SingerID);

0 comments on commit 9432313

Please sign in to comment.