From 0cf4c35ab0e917819cfbba28d261e9528dc3b7a1 Mon Sep 17 00:00:00 2001 From: Artem Barger Date: Wed, 19 Jul 2017 18:14:34 +0300 Subject: [PATCH] [FAB-5353]: Qualify sys. failure vs validation error Currently as stated in [FAB-5353], there is no clear separation during transaction validation during block commmit, between invalid transaction and some system failure which migh lead to inability to validate the transaction. For example db is down or file system is unavailable. This might lead to inconsistency of the state accross peers, therefore this commit takes care to distinguish between real case of invalid transaction versus system failure, later the error propagated down to the committer and forces peer to stop with panic, so admin will be able to take manual control and fix the problem therefore preventing peer state to diverge. Change-Id: I384e16d37e2f2b0fe144d504f566e0b744a5095c Signed-off-by: Artem Barger --- core/committer/txvalidator/validator.go | 78 ++++++++-- core/committer/txvalidator/validator_test.go | 146 +++++++++++++++++++ gossip/state/state.go | 4 +- 3 files changed, 214 insertions(+), 14 deletions(-) diff --git a/core/committer/txvalidator/validator.go b/core/committer/txvalidator/validator.go index fdbf1622e8f..52ff0a06d3c 100644 --- a/core/committer/txvalidator/validator.go +++ b/core/committer/txvalidator/validator.go @@ -86,6 +86,40 @@ type txValidator struct { vscc vsccValidator } +// VSCCInfoLookupFailureError error to indicate inability +// to obtain VSCC information from LCCC +type VSCCInfoLookupFailureError struct { + reason string +} + +// Error returns reasons which lead to the failure +func (e VSCCInfoLookupFailureError) Error() string { + return e.reason +} + +// VSCCEndorsementPolicyError error to mark transaction +// failed endrosement policy check +type VSCCEndorsementPolicyError struct { + reason string +} + +// Error returns reasons which lead to the failure +func (e VSCCEndorsementPolicyError) Error() string { + return e.reason +} + +// VSCCExecutionFailureError error to indicate +// failure during attempt of executing VSCC +// endorsement policy check +type VSCCExecutionFailureError struct { + reason string +} + +// Error returns reasons which lead to the failure +func (e VSCCExecutionFailureError) Error() string { + return e.reason +} + var logger *logging.Logger // package-level logger func init() { @@ -170,8 +204,15 @@ func (v *txValidator) Validate(block *common.Block) error { if err != nil { txID := txID logger.Errorf("VSCCValidateTx for transaction txId = %s returned error %s", txID, err) - txsfltr.SetFlag(tIdx, cde) - continue + switch err.(type) { + case *VSCCExecutionFailureError: + return err + case *VSCCInfoLookupFailureError: + return err + default: + txsfltr.SetFlag(tIdx, cde) + continue + } } invokeCC, upgradeCC, err := v.getTxCCInstance(payload) @@ -370,7 +411,8 @@ func (v *vsccValidatorImpl) GetInfoForValidate(txid, chID, ccID string) (*sysccp // obtain name of the VSCC and the policy from LSCC cd, err := v.getCDataForCC(ccID) if err != nil { - logger.Errorf("Unable to get chaincode data from ledger for txid %s, due to %s", txid, err) + msg := fmt.Sprintf("Unable to get chaincode data from ledger for txid %s, due to %s", txid, err) + logger.Errorf(msg) return nil, nil, nil, err } cc.ChaincodeName = cd.Name @@ -514,8 +556,12 @@ func (v *vsccValidatorImpl) VSCCValidateTx(payload *common.Payload, envBytes []b // do VSCC validation if err = v.VSCCValidateTxForCC(envBytes, chdr.TxId, chdr.ChannelId, vscc.ChaincodeName, vscc.ChaincodeVersion, policy); err != nil { - return fmt.Errorf("VSCCValidateTxForCC failed for cc %s, error %s", ccID, err), - peer.TxValidationCode_ENDORSEMENT_POLICY_FAILURE + switch err.(type) { + case VSCCEndorsementPolicyError: + return err, peer.TxValidationCode_ENDORSEMENT_POLICY_FAILURE + default: + return err, peer.TxValidationCode_INVALID_OTHER_REASON + } } } } else { @@ -541,8 +587,12 @@ func (v *vsccValidatorImpl) VSCCValidateTx(payload *common.Payload, envBytes []b // user creates a new system chaincode which is invokable from the outside // they have to modify VSCC to provide appropriate validation if err = v.VSCCValidateTxForCC(envBytes, chdr.TxId, vscc.ChainID, vscc.ChaincodeName, vscc.ChaincodeVersion, policy); err != nil { - return fmt.Errorf("VSCCValidateTxForCC failed for cc %s, error %s", ccID, err), - peer.TxValidationCode_ENDORSEMENT_POLICY_FAILURE + switch err.(type) { + case VSCCEndorsementPolicyError: + return err, peer.TxValidationCode_ENDORSEMENT_POLICY_FAILURE + default: + return err, peer.TxValidationCode_INVALID_OTHER_REASON + } } } @@ -552,8 +602,9 @@ func (v *vsccValidatorImpl) VSCCValidateTx(payload *common.Payload, envBytes []b func (v *vsccValidatorImpl) VSCCValidateTxForCC(envBytes []byte, txid, chid, vsccName, vsccVer string, policy []byte) error { ctxt, err := v.ccprovider.GetContext(v.support.Ledger()) if err != nil { - logger.Errorf("Cannot obtain context for txid=%s, err %s", txid, err) - return err + msg := fmt.Sprintf("Cannot obtain context for txid=%s, err %s", txid, err) + logger.Errorf(msg) + return &VSCCExecutionFailureError{msg} } defer v.ccprovider.ReleaseContext() @@ -571,12 +622,13 @@ func (v *vsccValidatorImpl) VSCCValidateTxForCC(envBytes []byte, txid, chid, vsc logger.Debug("Invoking VSCC txid", txid, "chaindID", chid) res, _, err := v.ccprovider.ExecuteChaincode(ctxt, cccid, args) if err != nil { - logger.Errorf("Invoke VSCC failed for transaction txid=%s, error %s", txid, err) - return err + msg := fmt.Sprintf("Invoke VSCC failed for transaction txid=%s, error %s", txid, err) + logger.Errorf(msg) + return &VSCCExecutionFailureError{msg} } if res.Status != shim.OK { logger.Errorf("VSCC check failed for transaction txid=%s, error %s", txid, res.Message) - return fmt.Errorf("%s", res.Message) + return &VSCCEndorsementPolicyError{fmt.Sprintf("%s", res.Message)} } return nil @@ -596,7 +648,7 @@ func (v *vsccValidatorImpl) getCDataForCC(ccid string) (*ccprovider.ChaincodeDat bytes, err := qe.GetState("lscc", ccid) if err != nil { - return nil, fmt.Errorf("Could not retrieve state for chaincode %s, error %s", ccid, err) + return nil, &VSCCInfoLookupFailureError{fmt.Sprintf("Could not retrieve state for chaincode %s, error %s", ccid, err)} } if bytes == nil { diff --git a/core/committer/txvalidator/validator_test.go b/core/committer/txvalidator/validator_test.go index b6ce7b78694..86ce49216a4 100644 --- a/core/committer/txvalidator/validator_test.go +++ b/core/committer/txvalidator/validator_test.go @@ -17,12 +17,14 @@ limitations under the License. package txvalidator import ( + "errors" "fmt" "os" "testing" "github.com/hyperledger/fabric/common/cauthdsl" ctxt "github.com/hyperledger/fabric/common/configtx/test" + ledger2 "github.com/hyperledger/fabric/common/ledger" "github.com/hyperledger/fabric/common/ledger/testutil" "github.com/hyperledger/fabric/common/mocks/scc" "github.com/hyperledger/fabric/common/util" @@ -41,6 +43,7 @@ import ( "github.com/hyperledger/fabric/protos/utils" "github.com/spf13/viper" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" ) func signedByAnyMember(ids []string) []byte { @@ -385,6 +388,149 @@ func TestInvokeNoBlock(t *testing.T) { assert.NoError(t, err) } +// mockLedger structure used to test ledger +// failure, therefore leveraging mocking +// library as need to simulate ledger which not +// able to get access to state db +type mockLedger struct { + mock.Mock +} + +// GetTransactionByID returns transaction by ud +func (m *mockLedger) GetTransactionByID(txID string) (*peer.ProcessedTransaction, error) { + args := m.Called(txID) + return args.Get(0).(*peer.ProcessedTransaction), args.Error(1) +} + +// GetBlockByHash returns block using its hash value +func (m *mockLedger) GetBlockByHash(blockHash []byte) (*common.Block, error) { + args := m.Called(blockHash) + return args.Get(0).(*common.Block), nil +} + +// GetBlockByTxID given transaction id return block transaction was committed with +func (m *mockLedger) GetBlockByTxID(txID string) (*common.Block, error) { + args := m.Called(txID) + return args.Get(0).(*common.Block), nil +} + +// GetTxValidationCodeByTxID returns validation code of give tx +func (m *mockLedger) GetTxValidationCodeByTxID(txID string) (peer.TxValidationCode, error) { + args := m.Called(txID) + return args.Get(0).(peer.TxValidationCode), nil +} + +// NewTxSimulator creates new transaction simulator +func (m *mockLedger) NewTxSimulator() (ledger.TxSimulator, error) { + args := m.Called() + return args.Get(0).(ledger.TxSimulator), nil +} + +// NewQueryExecutor creates query executor +func (m *mockLedger) NewQueryExecutor() (ledger.QueryExecutor, error) { + args := m.Called() + return args.Get(0).(ledger.QueryExecutor), nil +} + +// NewHistoryQueryExecutor history query executor +func (m *mockLedger) NewHistoryQueryExecutor() (ledger.HistoryQueryExecutor, error) { + args := m.Called() + return args.Get(0).(ledger.HistoryQueryExecutor), nil +} + +// Prune prune using policy +func (m *mockLedger) Prune(policy ledger2.PrunePolicy) error { + return nil +} + +func (m *mockLedger) GetBlockchainInfo() (*common.BlockchainInfo, error) { + args := m.Called() + return args.Get(0).(*common.BlockchainInfo), nil +} + +func (m *mockLedger) GetBlockByNumber(blockNumber uint64) (*common.Block, error) { + args := m.Called(blockNumber) + return args.Get(0).(*common.Block), nil +} + +func (m *mockLedger) GetBlocksIterator(startBlockNumber uint64) (ledger2.ResultsIterator, error) { + args := m.Called(startBlockNumber) + return args.Get(0).(ledger2.ResultsIterator), nil +} + +func (m *mockLedger) Close() { + +} + +func (m *mockLedger) Commit(block *common.Block) error { + return nil +} + +// mockQueryExecutor mock of the query executor, +// needed to simulate inability to access state db, e.g. +// the case where due to db failure it's not possible to +// query for state, for example if we would like to query +// the lccc for VSCC info and db is not avaible we expect +// to stop validating block and fail commit procedure with +// an error. +type mockQueryExecutor struct { + mock.Mock +} + +func (exec *mockQueryExecutor) GetState(namespace string, key string) ([]byte, error) { + args := exec.Called(namespace, key) + return args.Get(0).([]byte), args.Error(1) +} + +func (exec *mockQueryExecutor) GetStateMultipleKeys(namespace string, keys []string) ([][]byte, error) { + args := exec.Called(namespace, keys) + return args.Get(0).([][]byte), args.Error(1) +} + +func (exec *mockQueryExecutor) GetStateRangeScanIterator(namespace string, startKey string, endKey string) (ledger2.ResultsIterator, error) { + args := exec.Called(namespace, startKey, endKey) + return args.Get(0).(ledger2.ResultsIterator), args.Error(1) +} + +func (exec *mockQueryExecutor) ExecuteQuery(namespace, query string) (ledger2.ResultsIterator, error) { + args := exec.Called(namespace) + return args.Get(0).(ledger2.ResultsIterator), args.Error(1) +} + +func (exec *mockQueryExecutor) Done() { +} + +// TestLedgerIsNoAvailable simulates and provides a test for following scenario, +// which is based on FAB-535. Test checks the validation path which expects that +// DB won't available while trying to lookup for VSCC from LCCC and therefore +// transaction validation will have to fail. In such case the outcome should be +// the error return from validate block method and proccessing of transactions +// has to stop. There is suppose to be clear indication of the failure with error +// returned from the function call. +func TestLedgerIsNoAvailable(t *testing.T) { + theLedger := new(mockLedger) + validator := NewTxValidator(&mockSupport{l: theLedger}) + + ccID := "mycc" + tx := getEnv(ccID, createRWset(t, ccID), t) + + theLedger.On("GetTransactionByID", mock.Anything).Return(&peer.ProcessedTransaction{}, errors.New("Cannot find the transaction")) + + queryExecutor := new(mockQueryExecutor) + queryExecutor.On("GetState", mock.Anything, mock.Anything).Return([]byte{}, errors.New("Unable to connect to DB")) + theLedger.On("NewQueryExecutor", mock.Anything).Return(queryExecutor, nil) + + b := &common.Block{Data: &common.BlockData{Data: [][]byte{utils.MarshalOrPanic(tx)}}} + + err := validator.Validate(b) + + assertion := assert.New(t) + // We suppose to get the error which indicates we cannot commit the block + assertion.Error(err) + // The error exptected to be of type VSCCInfoLookupFailureError + assertion.NotNil(err.(*VSCCInfoLookupFailureError)) +} + var signer msp.SigningIdentity var signerSerialized []byte diff --git a/gossip/state/state.go b/gossip/state/state.go index 84cc7174284..7141f197536 100644 --- a/gossip/state/state.go +++ b/gossip/state/state.go @@ -428,7 +428,9 @@ func (s *GossipStateProviderImpl) deliverPayloads() { continue } logger.Debug("New block with claimed sequence number ", payload.SeqNum, " transactions num ", len(rawBlock.Data.Data)) - s.commitBlock(rawBlock) + if err := s.commitBlock(rawBlock); err != nil { + logger.Panicf("Cannot commit block to the ledger due to %s", err) + } } case <-s.stopCh: s.stopCh <- struct{}{}