Skip to content

Commit

Permalink
test: instability simulation tests
Browse files Browse the repository at this point in the history
This adds a new suite of simulation tests to test how `xud` responds to
instability. It simulates crashes immediately after the maker sends
payment as part of a swap. When the maker comes back online, it will
fetch the preimage for the successful payment and use it to settle
the incoming payment from the interrupted swap.

This tests two key cases, one where the maker's payment goes through
while the maker is offline, and another where the payment goes through
only after the maker has come back online. In the latter case, the maker
must detect that the payment is in a pending state and monitor it in
case it goes through.

Closes #1183.
  • Loading branch information
sangaman committed Sep 11, 2019
1 parent 856f14a commit d401b0c
Show file tree
Hide file tree
Showing 6 changed files with 182 additions and 8 deletions.
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
"test:crypto": "mocha --timeout 10000 -r ts-node/register test/crypto/*",
"test:sim": "(npm run compile && cd test/simulation && export PATH=\"$PWD/go/bin:$PATH\" && GOPATH=$PWD/go GO111MODULE=on go test -run=TestIntegration -v)",
"test:sim:security": "(npm run compile && cd test/simulation && export PATH=\"$PWD/go/bin:$PATH\" && GOPATH=$PWD/go GO111MODULE=on go test -run=TestSecurity -v)",
"test:sim:instability": "(npm run compile && cd test/simulation && export PATH=\"$PWD/go/bin:$PATH\" && GOPATH=$PWD/go GO111MODULE=on go test -run=TestInstability -v)",
"test:jest": "jest",
"test:seedutil": "jest seedutil",
"test:jest:watch": "jest --watch",
Expand Down
3 changes: 2 additions & 1 deletion test/simulation/harnessTest.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,11 @@ package main
import (
"context"
"fmt"
"testing"

"github.com/ExchangeUnion/xud-simulation/xudtest"
goerrors "github.com/go-errors/errors"
"github.com/stretchr/testify/require"
"testing"
)

type testCase struct {
Expand Down
139 changes: 139 additions & 0 deletions test/simulation/tests-instability.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,139 @@
package main

import (
"time"

"github.com/ExchangeUnion/xud-simulation/xudrpc"
"github.com/ExchangeUnion/xud-simulation/xudtest"
)

var ltcQuantity int64 = 1000000

// instabilityTestCases are test cases which try to simulate instability
// due to bugs, network outages, or system issues. They test whether xud
// can handle such problems gracefully and prevent loss of funds.
var instabilityTestCases = []*testCase{
{
name: "network initialization", // must be the first test case to be run
test: testNetworkInit,
},
{
name: "maker crashed after send payment",
test: testMakerCrashedAfterSend,
},
{
name: "maker crashed after send payment with delayed settlement",
test: testMakerCrashedAfterSendDelayedSettlement,
},
}

func testMakerCrashedAfterSend(net *xudtest.NetworkHarness, ht *harnessTest) {
var err error
net.Alice, err = net.SetCustomXud(net.Alice, "instability", []string{"BREAKSWAP=MAKER_CRASH_AFTER_SEND"})
ht.assert.NoError(err)
ht.act.init(net.Alice)

// Connect Alice to Bob.
ht.act.connect(net.Alice, net.Bob)
ht.act.verifyConnectivity(net.Alice, net.Bob)

// Save the initial balance.
alicePrevBalance, err := getBalance(ht.ctx, net.Alice)
ht.assert.NoError(err)
alicePrevLtcBalance := alicePrevBalance.ltc.channel.GetBalance()

// Place an order on Alice.
aliceOrderReq := &xudrpc.PlaceOrderRequest{
OrderId: "maker_order_id",
Price: 0.02,
Quantity: uint64(ltcQuantity),
PairId: "LTC/BTC",
Side: xudrpc.OrderSide_BUY,
}
ht.act.placeOrderAndBroadcast(net.Alice, net.Bob, aliceOrderReq)

// Place a matching order on Bob.
bobOrderReq := &xudrpc.PlaceOrderRequest{
OrderId: "taker_order_id",
Price: aliceOrderReq.Price,
Quantity: aliceOrderReq.Quantity,
PairId: aliceOrderReq.PairId,
Side: xudrpc.OrderSide_SELL,
}
_, err = net.Bob.Client.PlaceOrderSync(ht.ctx, bobOrderReq)

<-net.Alice.ProcessExit

net.Alice.Start(nil)

// Brief delay to allow for swap to be recovered consistently
time.Sleep(1 * time.Second)

// Verify that alice received her LTC
aliceBalance, err := getBalance(ht.ctx, net.Alice)
ht.assert.NoError(err)
aliceLtcBalance := aliceBalance.ltc.channel.GetBalance()
ht.assert.Equal(alicePrevLtcBalance+ltcQuantity, aliceLtcBalance, "alice did not receive LTC")
}

func testMakerCrashedAfterSendDelayedSettlement(net *xudtest.NetworkHarness, ht *harnessTest) {
var err error
net.Alice, err = net.SetCustomXud(net.Alice, "instability", []string{"BREAKSWAP=MAKER_CRASH_AFTER_SEND"})
ht.assert.NoError(err)

net.Bob, err = net.SetCustomXud(net.Bob, "instability", []string{"BREAKSWAP=TAKER_DELAY_BEFORE_SETTLE"})
ht.assert.NoError(err)

ht.act.init(net.Alice)
ht.act.init(net.Bob)

// Connect Alice to Bob.
ht.act.connect(net.Alice, net.Bob)
ht.act.verifyConnectivity(net.Alice, net.Bob)

// Save the initial balance.
alicePrevBalance, err := getBalance(ht.ctx, net.Alice)
ht.assert.NoError(err)
alicePrevLtcBalance := alicePrevBalance.ltc.channel.GetBalance()

// Place an order on Alice.
aliceOrderReq := &xudrpc.PlaceOrderRequest{
OrderId: "maker_order_id",
Price: 0.02,
Quantity: uint64(ltcQuantity),
PairId: "LTC/BTC",
Side: xudrpc.OrderSide_BUY,
}
ht.act.placeOrderAndBroadcast(net.Alice, net.Bob, aliceOrderReq)

// Place a matching order on Bob.
bobOrderReq := &xudrpc.PlaceOrderRequest{
OrderId: "taker_order_id",
Price: aliceOrderReq.Price,
Quantity: aliceOrderReq.Quantity,
PairId: aliceOrderReq.PairId,
Side: xudrpc.OrderSide_SELL,
}
go net.Bob.Client.PlaceOrderSync(ht.ctx, bobOrderReq)

<-net.Alice.ProcessExit

net.Alice.Start(nil)

// Verify that alice hasn't claimed her LTC yet. The incoming LTC payment
// cannot be settled until the outgoing BTC payment is settled by bob,
// which is being intentionally delayed.
aliceIntermediateBalance, err := getBalance(ht.ctx, net.Alice)
ht.assert.NoError(err)
aliceIntermediateLtcBalance := aliceIntermediateBalance.ltc.channel.GetBalance()
ht.assert.NotEqual(alicePrevLtcBalance+ltcQuantity, aliceIntermediateLtcBalance)

// Delay to allow for payment to be claimed by bob then recovered by alice
time.Sleep(4 * time.Second)

// Verify that alice received her LTC
aliceBalance, err := getBalance(ht.ctx, net.Alice)
ht.assert.NoError(err)
aliceLtcBalance := aliceBalance.ltc.channel.GetBalance()
ht.assert.Equal(alicePrevLtcBalance+ltcQuantity, aliceLtcBalance, "alice did not receive LTC")
}
35 changes: 34 additions & 1 deletion test/simulation/xud_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,15 @@ package main

import (
"fmt"
"github.com/ExchangeUnion/xud-simulation/xudrpc"
"log"
"os"
"os/exec"
"strings"
"testing"
"time"

"github.com/ExchangeUnion/xud-simulation/xudrpc"

"github.com/ExchangeUnion/xud-simulation/lntest"
"github.com/ExchangeUnion/xud-simulation/xudtest"
ltcchaincfg "github.com/ltcsuite/ltcd/chaincfg"
Expand Down Expand Up @@ -127,6 +128,38 @@ func TestIntegration(t *testing.T) {
ht.assert.NoError(err)
}

func TestInstability(t *testing.T) {
xudNetwork, teardown := launchNetwork(true)
defer teardown()
ht := newHarnessTest(context.Background(), t)
t.Logf("Running %v instability tests", len(instabilityTestCases))

// Open channels from both directions on each chain.
aliceBobBtcChanPoint, err := openBtcChannel(ht.ctx, xudNetwork.LndBtcNetwork, xudNetwork.Alice.LndBtcNode, xudNetwork.Bob.LndBtcNode)
ht.assert.NoError(err)
aliceBobLtcChanPoint, err := openLtcChannel(ht.ctx, xudNetwork.LndLtcNetwork, xudNetwork.Alice.LndLtcNode, xudNetwork.Bob.LndLtcNode)
ht.assert.NoError(err)

for _, testCase := range instabilityTestCases {
success := t.Run(testCase.name, func(t1 *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
ht := newHarnessTest(ctx, t1)
ht.RunTestCase(testCase, xudNetwork)
})

if !success {
break
}
}

// Close all channels before next iteration.
err = closeBtcChannel(ht.ctx, xudNetwork.LndBtcNetwork, xudNetwork.Alice.LndBtcNode, aliceBobBtcChanPoint, false)
ht.assert.NoError(err)
err = closeLtcChannel(ht.ctx, xudNetwork.LndLtcNetwork, xudNetwork.Alice.LndLtcNode, aliceBobLtcChanPoint, false)
ht.assert.NoError(err)
}

func TestSecurity(t *testing.T) {
xudNetwork, teardown := launchNetwork(true)
defer teardown()
Expand Down
8 changes: 4 additions & 4 deletions test/simulation/xudtest/harness.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ func (n *NetworkHarness) SetCustomXud(node *HarnessNode, branch string, envVars

xudPath := filepath.Join(wd, "./temp", branch)
if _, err := os.Stat(xudPath); os.IsNotExist(err) {
log.Printf("custon xud not found at %v, installing...", xudPath)
log.Printf("custom xud not found at %v, installing...", xudPath)
_, err := exec.Command("git", "clone", "-b", branch, "https://github.com/ExchangeUnion/xud", xudPath).Output()
if err != nil {
return nil, fmt.Errorf("custom xud git clone failure: %v", err)
Expand Down Expand Up @@ -95,7 +95,7 @@ func (n *NetworkHarness) SetCustomXud(node *HarnessNode, branch string, envVars
errChan := make(chan error, 1)
go func() {
defer wg.Done()
if err := customNode.start(n.errorChan); err != nil {
if err := customNode.Start(n.errorChan); err != nil {
if err != nil {
errChan <- err
return
Expand All @@ -120,7 +120,7 @@ func (n *NetworkHarness) newNode(name string, xudPath string, noBalanceChecks bo
return node, nil
}

// Start starts this xud node and its corresponding lnd nodes.
// Start starts all xud nodes and their corresponding lnd nodes.
func (n *NetworkHarness) Start() error {
var wg sync.WaitGroup
wg.Add(4)
Expand All @@ -130,7 +130,7 @@ func (n *NetworkHarness) Start() error {
node := _node
go func() {
defer wg.Done()
if err := node.start(n.errorChan); err != nil {
if err := node.Start(n.errorChan); err != nil {
if err != nil {
errChan <- err
return
Expand Down
4 changes: 2 additions & 2 deletions test/simulation/xudtest/node.go
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,7 @@ func (hn *HarnessNode) SetEnvVars(envVars []string) {
}

// Start launches a new running process of xud.
func (hn *HarnessNode) start(errorChan chan<- *XudError) error {
func (hn *HarnessNode) Start(errorChan chan<- *XudError) error {
hn.quit = make(chan struct{})

args := hn.Cfg.genArgs()
Expand Down Expand Up @@ -210,7 +210,7 @@ func (hn *HarnessNode) start(errorChan chan<- *XudError) error {
defer hn.wg.Done()

err := hn.Cmd.Wait()
if err != nil {
if err != nil && errorChan != nil {
errorChan <- &XudError{hn, errors.Errorf("%v: %v\n%v\n", err, errb.String(), out.String())}
}

Expand Down

0 comments on commit d401b0c

Please sign in to comment.