Skip to content

Commit 88157e0

Browse files
serathiusserathius
committed
tests: Reproducing 14890
Signed-off-by: Marek Siarkowicz <siarkowicz@google.com>
1 parent ad5840c commit 88157e0

File tree

7 files changed

+278
-39
lines changed

7 files changed

+278
-39
lines changed

‎.github/workflows/linearizability.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ jobs:
1313
make build
1414
mkdir -p /tmp/linearizability
1515
cat server/etcdserver/raft.fail.go
16-
EXPECT_DEBUG=true GO_TEST_FLAGS='-v --count 60 --failfast --run TestLinearizability' RESULTS_DIR=/tmp/linearizability make test-linearizability
16+
EXPECT_DEBUG=true GO_TEST_FLAGS='-v --count 1 --failfast --run TestLinearizability' RESULTS_DIR=/tmp/linearizability make test-linearizability
1717
- uses: actions/upload-artifact@v2
1818
if: always()
1919
with:

‎client/v2/keys.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,7 @@ type WatcherOptions struct {
147147

148148
type CreateInOrderOptions struct {
149149
// TTL defines a period of time after-which the Node should
150-
// expire and no longer exist. Values <= 0 are ignored. Given
150+
// expire and no longer exist. Elements <= 0 are ignored. Given
151151
// that the zero-value is ignored, TTL cannot be used to set
152152
// a TTL of 0.
153153
TTL time.Duration
@@ -177,7 +177,7 @@ type SetOptions struct {
177177
PrevExist PrevExistType
178178

179179
// TTL defines a period of time after-which the Node should
180-
// expire and no longer exist. Values <= 0 are ignored. Given
180+
// expire and no longer exist. Elements <= 0 are ignored. Given
181181
// that the zero-value is ignored, TTL cannot be used to set
182182
// a TTL of 0.
183183
TTL time.Duration

‎tests/linearizability/append_model.go

+138
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
// Copyright 2022 The etcd Authors
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package linearizability
16+
17+
import (
18+
"encoding/json"
19+
"fmt"
20+
"strings"
21+
22+
"github.com/anishathalye/porcupine"
23+
)
24+
25+
const (
26+
Append Operation = "append"
27+
)
28+
29+
type AppendRequest struct {
30+
Op Operation
31+
Key string
32+
AppendData string
33+
}
34+
35+
type AppendResponse struct {
36+
GetData string
37+
}
38+
39+
type AppendState struct {
40+
Key string
41+
Elements []string
42+
}
43+
44+
var appendModel = porcupine.Model{
45+
Init: func() interface{} { return "{}" },
46+
Step: func(st interface{}, in interface{}, out interface{}) (bool, interface{}) {
47+
var state AppendState
48+
err := json.Unmarshal([]byte(st.(string)), &state)
49+
if err != nil {
50+
panic(err)
51+
}
52+
ok, state := appendModelStep(state, in.(AppendRequest), out.(AppendResponse))
53+
data, err := json.Marshal(state)
54+
if err != nil {
55+
panic(err)
56+
}
57+
return ok, string(data)
58+
},
59+
DescribeOperation: func(in, out interface{}) string {
60+
request := in.(AppendRequest)
61+
response := out.(AppendResponse)
62+
switch request.Op {
63+
case Get:
64+
elements := strings.Split(response.GetData, ",")
65+
return fmt.Sprintf("get(%q) -> %q", request.Key, elements[len(elements)-1])
66+
case Append:
67+
return fmt.Sprintf("append(%q, %q)", request.Key, request.AppendData)
68+
default:
69+
return "<invalid>"
70+
}
71+
},
72+
}
73+
74+
func appendModelStep(state AppendState, request AppendRequest, response AppendResponse) (bool, AppendState) {
75+
if request.Key == "" {
76+
panic("invalid request")
77+
}
78+
if state.Key == "" {
79+
return true, initAppendState(request, response)
80+
}
81+
if state.Key != request.Key {
82+
panic("Multiple keys not supported")
83+
}
84+
switch request.Op {
85+
case Get:
86+
return stepAppendGet(state, request, response)
87+
case Append:
88+
return stepAppend(state, request, response)
89+
default:
90+
panic("Unknown operation")
91+
}
92+
}
93+
94+
func initAppendState(request AppendRequest, response AppendResponse) AppendState {
95+
state := AppendState{
96+
Key: request.Key,
97+
}
98+
switch request.Op {
99+
case Get:
100+
state.Elements = elements(response)
101+
case Append:
102+
state.Elements = []string{request.AppendData}
103+
default:
104+
panic("Unknown operation")
105+
}
106+
return state
107+
}
108+
109+
func stepAppendGet(state AppendState, request AppendRequest, response AppendResponse) (bool, AppendState) {
110+
newElements := elements(response)
111+
if len(newElements) < len(state.Elements) {
112+
return false, state
113+
}
114+
115+
for i := 0; i < len(state.Elements); i++ {
116+
if state.Elements[i] != newElements[i] {
117+
return false, state
118+
}
119+
}
120+
state.Elements = newElements
121+
return true, state
122+
}
123+
124+
func stepAppend(state AppendState, request AppendRequest, response AppendResponse) (bool, AppendState) {
125+
if request.AppendData == "" {
126+
panic("unsupported empty appendData")
127+
}
128+
state.Elements = append(state.Elements, request.AppendData)
129+
return true, state
130+
}
131+
132+
func elements(response AppendResponse) []string {
133+
elements := strings.Split(response.GetData, ",")
134+
if len(elements) == 1 && elements[0] == "" {
135+
elements = []string{}
136+
}
137+
return elements
138+
}

‎tests/linearizability/append_model_test.go

+68
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
// Copyright 2022 The etcd Authors
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package linearizability
16+
17+
import (
18+
"testing"
19+
)
20+
21+
func TestAppendModel(t *testing.T) {
22+
tcs := []struct {
23+
name string
24+
operations []testAppendOperation
25+
}{
26+
{
27+
name: "Append appends",
28+
operations: []testAppendOperation{
29+
{req: AppendRequest{Key: "key", Op: Append, AppendData: "1"}, resp: AppendResponse{}},
30+
{req: AppendRequest{Key: "key", Op: Get}, resp: AppendResponse{GetData: "1"}},
31+
{req: AppendRequest{Key: "key", Op: Append, AppendData: "2"}, resp: AppendResponse{}},
32+
{req: AppendRequest{Key: "key", Op: Get}, resp: AppendResponse{GetData: "1,3"}, failure: true},
33+
{req: AppendRequest{Key: "key", Op: Get}, resp: AppendResponse{GetData: "1,2"}},
34+
},
35+
},
36+
{
37+
name: "Get validates prefix matches",
38+
operations: []testAppendOperation{
39+
{req: AppendRequest{Key: "key", Op: Get}, resp: AppendResponse{GetData: ""}},
40+
{req: AppendRequest{Key: "key", Op: Get}, resp: AppendResponse{GetData: "1"}},
41+
{req: AppendRequest{Key: "key", Op: Get}, resp: AppendResponse{GetData: "2"}, failure: true},
42+
{req: AppendRequest{Key: "key", Op: Append, AppendData: "2"}, resp: AppendResponse{}},
43+
{req: AppendRequest{Key: "key", Op: Get}, resp: AppendResponse{GetData: "1,3"}, failure: true},
44+
{req: AppendRequest{Key: "key", Op: Get}, resp: AppendResponse{GetData: "1,2,3"}},
45+
{req: AppendRequest{Key: "key", Op: Get}, resp: AppendResponse{GetData: "2,3"}, failure: true},
46+
},
47+
},
48+
}
49+
for _, tc := range tcs {
50+
var ok bool
51+
t.Run(tc.name, func(t *testing.T) {
52+
state := appendModel.Init()
53+
for _, op := range tc.operations {
54+
t.Logf("state: %v", state)
55+
ok, state = appendModel.Step(state, op.req, op.resp)
56+
if ok != !op.failure {
57+
t.Errorf("Unexpected operation result, expect: %v, got: %v, operation: %s", !op.failure, ok, appendModel.DescribeOperation(op.req, op.resp))
58+
}
59+
}
60+
})
61+
}
62+
}
63+
64+
type testAppendOperation struct {
65+
req AppendRequest
66+
resp AppendResponse
67+
failure bool
68+
}

‎tests/linearizability/history.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,7 @@ func (h *appendableHistory) appendFailed(request EtcdRequest, start time.Time, e
123123
})
124124
// Operations of single client needs to be sequential.
125125
// As we don't know return time of failed operations, all new writes need to be done with new client id.
126-
h.id = h.idProvider.ClientId()
126+
//h.id = h.idProvider.ClientId()
127127
}
128128

129129
type history struct {

‎tests/linearizability/linearizability_test.go

+67-35
Original file line numberDiff line numberDiff line change
@@ -43,47 +43,34 @@ func TestLinearizability(t *testing.T) {
4343
tcs := []struct {
4444
name string
4545
failpoint Failpoint
46+
traffic Traffic
4647
config e2e.EtcdProcessClusterConfig
4748
}{
4849
{
49-
name: "ClusterOfSize1",
50-
failpoint: RandomFailpoint,
50+
name: "Issue14890",
51+
traffic: AppendOnly,
52+
failpoint: KillFailpoint,
5153
config: *e2e.NewConfig(
52-
e2e.WithClusterSize(1),
53-
e2e.WithGoFailEnabled(true),
54-
e2e.WithCompactionBatchLimit(100), // required for compactBeforeCommitBatch and compactAfterCommitBatch failpoints
55-
),
56-
},
57-
{
58-
name: "ClusterOfSize3",
59-
failpoint: RandomFailpoint,
60-
config: *e2e.NewConfig(
61-
e2e.WithGoFailEnabled(true),
62-
e2e.WithCompactionBatchLimit(100), // required for compactBeforeCommitBatch and compactAfterCommitBatch failpoints
63-
),
64-
},
65-
{
66-
name: "Issue14370",
67-
failpoint: RaftBeforeSavePanic,
68-
config: *e2e.NewConfig(
69-
e2e.WithClusterSize(1),
70-
e2e.WithGoFailEnabled(true),
54+
e2e.WithClusterSize(5),
7155
),
7256
},
7357
}
7458
for _, tc := range tcs {
7559
t.Run(tc.name, func(t *testing.T) {
7660
failpoint := FailpointConfig{
7761
failpoint: tc.failpoint,
78-
count: 1,
62+
count: 60,
7963
retries: 3,
8064
waitBetweenTriggers: waitBetweenFailpointTriggers,
8165
}
8266
traffic := trafficConfig{
83-
minimalQPS: minimalQPS,
84-
maximalQPS: maximalQPS,
85-
clientCount: 8,
86-
traffic: DefaultTraffic,
67+
minimalQPS: 100,
68+
maximalQPS: 1000,
69+
traffic: tc.traffic,
70+
clientCount: 10,
71+
}
72+
if tc.traffic == nil {
73+
tc.traffic = DefaultTraffic
8774
}
8875
testLinearizability(context.Background(), t, tc.config, failpoint, traffic)
8976
})
@@ -193,18 +180,63 @@ func checkOperationsAndPersistResults(t *testing.T, operations []porcupine.Opera
193180
t.Error(err)
194181
}
195182

196-
linearizable, info := porcupine.CheckOperationsVerbose(etcdModel, operations, 0)
197-
if linearizable != porcupine.Ok {
198-
t.Error("Model is not linearizable")
199-
persistMemberDataDir(t, clus, path)
183+
appendOperations := []porcupine.Operation{}
184+
isAppendOnly := true
185+
appendCheck:
186+
for _, op := range operations {
187+
req := op.Input.(EtcdRequest)
188+
resp := op.Output.(EtcdResponse)
189+
switch req.Op {
190+
case Get:
191+
appendOperations = append(appendOperations, porcupine.Operation{
192+
ClientId: op.ClientId,
193+
Input: AppendRequest{Op: Get, Key: req.Key},
194+
Call: op.Call,
195+
Output: AppendResponse{GetData: resp.GetData},
196+
Return: op.Return,
197+
})
198+
case Txn:
199+
if resp.Err != nil || !resp.TxnSucceeded {
200+
continue
201+
}
202+
elements := strings.Split(req.TxnNewData, ",")
203+
appendOperations = append(appendOperations, porcupine.Operation{
204+
ClientId: op.ClientId,
205+
Input: AppendRequest{Op: Append, Key: req.Key, AppendData: elements[len(elements)-1]},
206+
Call: op.Call,
207+
Output: AppendResponse{GetData: resp.GetData},
208+
Return: op.Return,
209+
})
210+
default:
211+
isAppendOnly = false
212+
break appendCheck
213+
}
200214
}
201-
202215
visualizationPath := filepath.Join(path, "history.html")
203-
t.Logf("saving visualization to %q", visualizationPath)
204-
err = porcupine.VisualizePath(etcdModel, info, visualizationPath)
205-
if err != nil {
206-
t.Errorf("Failed to visualize, err: %v", err)
216+
if isAppendOnly {
217+
t.Log("Using append model")
218+
linearizable, info := porcupine.CheckOperationsVerbose(appendModel, appendOperations, 0)
219+
if linearizable != porcupine.Ok {
220+
t.Error("Model is not linearizable")
221+
persistMemberDataDir(t, clus, path)
222+
}
223+
err = porcupine.VisualizePath(appendModel, info, visualizationPath)
224+
if err != nil {
225+
t.Errorf("Failed to visualize, err: %v", err)
226+
}
227+
} else {
228+
t.Error("Using etcd model")
229+
linearizable, info := porcupine.CheckOperationsVerbose(etcdModel, operations, 0)
230+
if linearizable != porcupine.Ok {
231+
t.Error("Model is not linearizable")
232+
persistMemberDataDir(t, clus, path)
233+
}
234+
err = porcupine.VisualizePath(etcdModel, info, visualizationPath)
235+
if err != nil {
236+
t.Errorf("Failed to visualize, err: %v", err)
237+
}
207238
}
239+
t.Logf("saving visualization to %q", visualizationPath)
208240
}
209241

210242
func persistMemberDataDir(t *testing.T, clus *e2e.EtcdProcessCluster, path string) {

‎tests/linearizability/traffic.go

+1
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525

2626
var (
2727
DefaultTraffic Traffic = readWriteSingleKey{key: "key", writes: []opChance{{operation: Put, chance: 90}, {operation: Delete, chance: 5}, {operation: Txn, chance: 5}}}
28+
AppendOnly Traffic = readWriteSingleKey{key: "key", writes: []opChance{{operation: Txn, chance: 100}}}
2829
)
2930

3031
type Traffic interface {

0 commit comments

Comments
 (0)