Maintain lexographic sort order for composite keys

https://jira.hyperledger.org/browse/FAB-2008

This CR
 - Changes the way composite key and prefix scan range are encoded
   so as to maintain the lexographical sort order of the results
   of a range query
- Cleans up escaping used in couchdb for start/end Key for range scan
- Adds couchdb test for a prefix scan

Change-Id: Icbc95ddf3916b5286a22eb83c173efd8dc68e75d
Signed-off-by: manish <manish.sethi@gmail.com>

core/chaincode/shim/chaincode.go

@@ -19,15 +19,13 @@ limitations under the License.
 package shim
 
 import (
-	"bytes"
 	"errors"
 	"flag"
 	"fmt"
 	"io"
 	"os"
-	"regexp"
-	"strconv"
 	"strings"
+	"unicode/utf8"
 
 	"github.com/golang/protobuf/proto"
 	"github.com/golang/protobuf/ptypes/timestamp"
@@ -43,6 +41,11 @@ import (
 // Logger for the shim package.
 var chaincodeLogger = logging.MustGetLogger("shim")
 
+const (
+	minUnicodeRuneValue = 0            //U+0000
+	maxUnicodeRuneValue = utf8.MaxRune //U+10FFFF - maximum (and unallocated) code point
+)
+
 // ChaincodeStub is an object passed to chaincode for shim side handling of
 // APIs.
 type ChaincodeStub struct {
@@ -359,35 +362,52 @@ func (stub *ChaincodeStub) GetQueryResult(query string) (StateQueryIteratorInter
 
 //CreateCompositeKey combines the given attributes to form a composite key.
 func (stub *ChaincodeStub) CreateCompositeKey(objectType string, attributes []string) (string, error) {
-	return createCompositeKey(stub, objectType, attributes)
+	return createCompositeKey(objectType, attributes)
+}
+
+//SplitCompositeKey splits the key into attributes on which the composite key was formed.
+func (stub *ChaincodeStub) SplitCompositeKey(compositeKey string) (string, []string, error) {
+	return splitCompositeKey(compositeKey)
 }
 
-func createCompositeKey(stub ChaincodeStubInterface, objectType string, attributes []string) (string, error) {
-	var compositeKey bytes.Buffer
-	replacer := strings.NewReplacer("\x1E", "\x1E\x1E", "\x1F", "\x1E\x1F")
-	compositeKey.WriteString(replacer.Replace(objectType))
-	for _, attribute := range attributes {
-		compositeKey.WriteString("\x1F" + strconv.Itoa(len(attribute)) + "\x1F")
-		compositeKey.WriteString(replacer.Replace(attribute))
+func createCompositeKey(objectType string, attributes []string) (string, error) {
+	if err := validateCompositeKeyAttribute(objectType); err != nil {
+		return "", err
+	}
+	ck := objectType + string(minUnicodeRuneValue)
+	for _, att := range attributes {
+		if err := validateCompositeKeyAttribute(att); err != nil {
+			return "", err
+		}
+		ck += att + string(minUnicodeRuneValue)
 	}
-	return compositeKey.String(), nil
+	return ck, nil
 }
 
-//SplitCompositeKey splits the key into attributes on which the composite key was formed.
-func (stub *ChaincodeStub) SplitCompositeKey(compositeKey string) (string, []string, error) {
-	return splitCompositeKey(stub, compositeKey)
+func splitCompositeKey(compositeKey string) (string, []string, error) {
+	componentIndex := 0
+	components := []string{}
+	for i := 0; i < len(compositeKey); i++ {
+		if compositeKey[i] == minUnicodeRuneValue {
+			components = append(components, compositeKey[componentIndex:i])
+			componentIndex = i + 1
+		}
+	}
+	return components[0], components[1:], nil
 }
 
-func splitCompositeKey(stub ChaincodeStubInterface, compositeKey string) (string, []string, error) {
-	re := regexp.MustCompile("\x1F[0-9]+\x1F")
-	splittedKey := re.Split(compositeKey, -1)
-	attributes := make([]string, 0)
-	replacer := strings.NewReplacer("\x1E\x1F", "\x1F", "\x1E\x1E", "\x1E")
-	objectType := replacer.Replace(splittedKey[0])
-	for _, attr := range splittedKey[1:] {
-		attributes = append(attributes, replacer.Replace(attr))
+func validateCompositeKeyAttribute(str string) error {
+	for index, runeValue := range str {
+		if !utf8.ValidRune(runeValue) {
+			return fmt.Errorf("Not a valid utf8 string. Contains rune [%d] starting at byte position [%d]",
+				runeValue, index)
+		}
+		if runeValue == minUnicodeRuneValue || runeValue == maxUnicodeRuneValue {
+			return fmt.Errorf(`Input contain unicode %#U starting at position [%d]. %#U and %#U are not allowed in the input attribute of a composite key`,
+				runeValue, index, minUnicodeRuneValue, maxUnicodeRuneValue)
+		}
 	}
-	return objectType, attributes, nil
+	return nil
 }
 
 //PartialCompositeKeyQuery function can be invoked by a chaincode to query the
@@ -402,7 +422,7 @@ func (stub *ChaincodeStub) PartialCompositeKeyQuery(objectType string, attribute
 
 func partialCompositeKeyQuery(stub ChaincodeStubInterface, objectType string, attributes []string) (StateQueryIteratorInterface, error) {
 	partialCompositeKey, _ := stub.CreateCompositeKey(objectType, attributes)
-	keysIter, err := stub.RangeQueryState(partialCompositeKey, partialCompositeKey+"\xFF")
+	keysIter, err := stub.RangeQueryState(partialCompositeKey, partialCompositeKey+string(maxUnicodeRuneValue))
 	if err != nil {
 		return nil, fmt.Errorf("Error fetching rows: %s", err)
 	}

core/chaincode/shim/interfaces.go

@@ -77,24 +77,26 @@ type ChaincodeStubInterface interface {
 	// iterator which can be used to iterate over all composite keys whose prefix
 	// matches the given partial composite key. This function should be used only for
 	// a partial composite key. For a full composite key, an iter with empty response
-	// would be returned.
+	// would be returned. The objectType and attributes are expected to have only
+	// valid utf8 strings and should not contain U+0000 (nil byte) and U+10FFFF (biggest and unallocated code point)
 	PartialCompositeKeyQuery(objectType string, keys []string) (StateQueryIteratorInterface, error)
 
 	// Given a list of attributes, CreateCompositeKey function combines these attributes
-	// to form a composite key.
+	// to form a composite key. The objectType and attributes are expected to have only
+	// valid utf8 strings and should not contain U+0000 (nil byte) and U+10FFFF (biggest and unallocated code point)
 	CreateCompositeKey(objectType string, attributes []string) (string, error)
 
+	// Given a composite key, SplitCompositeKey function splits the key into attributes
+	// on which the composite key was formed.
+	SplitCompositeKey(compositeKey string) (string, []string, error)
+
 	// GetQueryResult function can be invoked by a chaincode to perform a
 	// rich query against state database.  Only supported by state database implementations
 	// that support rich query.  The query string is in the syntax of the underlying
 	// state database. An iterator is returned which can be used to iterate (next) over
 	// the query result set
 	GetQueryResult(query string) (StateQueryIteratorInterface, error)
 
-	// Given a composite key, SplitCompositeKey function splits the key into attributes
-	// on which the composite key was formed.
-	SplitCompositeKey(compositeKey string) (string, []string, error)
-
 	// GetCallerCertificate returns caller certificate
 	GetCallerCertificate() ([]byte, error)
 

core/chaincode/shim/mockstub.go

@@ -217,13 +217,13 @@ func (stub *MockStub) PartialCompositeKeyQuery(objectType string, attributes []s
 // CreateCompositeKey combines the list of attributes
 //to form a composite key.
 func (stub *MockStub) CreateCompositeKey(objectType string, attributes []string) (string, error) {
-	return createCompositeKey(stub, objectType, attributes)
+	return createCompositeKey(objectType, attributes)
 }
 
 // SplitCompositeKey splits the composite key into attributes
 // on which the composite key was formed.
 func (stub *MockStub) SplitCompositeKey(compositeKey string) (string, []string, error) {
-	return splitCompositeKey(stub, compositeKey)
+	return splitCompositeKey(compositeKey)
 }
 
 // InvokeChaincode calls a peered chaincode.

core/chaincode/shim/mockstub_test.go

@@ -111,9 +111,10 @@ func TestPartialCompositeKeyQuery(t *testing.T) {
 	stub.PutState(compositeKey3, marbleJSONBytes3)
 
 	stub.MockTransactionEnd("init")
-	expectKeys := []string{compositeKey1, compositeKey2}
-	expectKeysAttributes := [][]string{{"set-1", "red"}, {"set-1", "blue"}}
-	expectValues := [][]byte{marbleJSONBytes1, marbleJSONBytes2}
+	// should return in sorted order of attributes
+	expectKeys := []string{compositeKey2, compositeKey1}
+	expectKeysAttributes := [][]string{{"set-1", "blue"}, {"set-1", "red"}}
+	expectValues := [][]byte{marbleJSONBytes2, marbleJSONBytes1}
 
 	rqi, _ := stub.PartialCompositeKeyQuery("marble", []string{"set-1"})
 	fmt.Println("Running loop")

core/ledger/util/couchdb/couchdb.go

@@ -33,6 +33,7 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
+	"unicode/utf8"
 
 	"github.com/hyperledger/fabric/core/ledger/kvledger/txmgmt/version"
 	logging "github.com/op/go-logging"
@@ -335,16 +336,18 @@ func (dbclient *CouchDatabase) EnsureFullCommit() (*DBOperationResponse, error)
 
 //SaveDoc method provides a function to save a document, id and byte array
 func (dbclient *CouchDatabase) SaveDoc(id string, rev string, bytesDoc []byte, attachments []Attachment) (string, error) {
-
 	logger.Debugf("Entering SaveDoc()")
-
+	if !utf8.ValidString(id) {
+		return "", fmt.Errorf("doc id [%x] not a valid utf8 string", id)
+	}
 	saveURL, err := url.Parse(dbclient.couchInstance.conf.URL)
 	if err != nil {
 		logger.Errorf("URL parse error: %s", err.Error())
 		return "", err
 	}
-	saveURL.Path = dbclient.dbName + "/" + id
-
+	saveURL.Path = dbclient.dbName
+	// id can contain a '/', so encode separately
+	saveURL = &url.URL{Opaque: saveURL.String() + "/" + encodePathElement(id)}
 	logger.Debugf("  id=%s,  value=%s", id, string(bytesDoc))
 
 	if rev == "" {
@@ -501,14 +504,17 @@ func getRevisionHeader(resp *http.Response) (string, error) {
 func (dbclient *CouchDatabase) ReadDoc(id string) ([]byte, string, error) {
 
 	logger.Debugf("Entering ReadDoc()  id=%s", id)
-
+	if !utf8.ValidString(id) {
+		return nil, "", fmt.Errorf("doc id [%x] not a valid utf8 string", id)
+	}
 	readURL, err := url.Parse(dbclient.couchInstance.conf.URL)
 	if err != nil {
 		logger.Errorf("URL parse error: %s", err.Error())
 		return nil, "", err
 	}
-	readURL.Path = dbclient.dbName + "/" + id
-
+	readURL.Path = dbclient.dbName
+	// id can contain a '/', so encode separately
+	readURL = &url.URL{Opaque: readURL.String() + "/" + encodePathElement(id)}
 	query := readURL.Query()
 	query.Add("attachments", "true")
 
@@ -644,26 +650,19 @@ func (dbclient *CouchDatabase) ReadDocRange(startKey, endKey string, limit, skip
 
 	//Append the startKey if provided
 	if startKey != "" {
-		startKey = strconv.QuoteToGraphic(startKey)
-		startKey = strings.Replace(startKey, "\\x00", "\\u0000", -1)
-		startKey = strings.Replace(startKey, "\\x1e", "\\u001e", -1)
-		startKey = strings.Replace(startKey, "\\x1f", "\\u001f", -1)
-		startKey = strings.Replace(startKey, "\\xff", "\\u00ff", -1)
-		//TODO add general unicode support instead of special cases
-
+		var err error
+		if startKey, err = encodeForJSON(startKey); err != nil {
+			return nil, err
+		}
 		queryParms.Add("startkey", startKey)
 	}
 
 	//Append the endKey if provided
 	if endKey != "" {
-		endKey = strconv.QuoteToGraphic(endKey)
-		endKey = strings.Replace(endKey, "\\x00", "\\u0000", -1)
-		endKey = strings.Replace(endKey, "\\x01", "\\u0001", -1)
-		endKey = strings.Replace(endKey, "\\x1e", "\\u001e", -1)
-		endKey = strings.Replace(endKey, "\\x1f", "\\u001f", -1)
-		endKey = strings.Replace(endKey, "\\xff", "\\u00ff", -1)
-		//TODO add general unicode support instead of special cases
-
+		var err error
+		if endKey, err = encodeForJSON(endKey); err != nil {
+			return nil, err
+		}
 		queryParms.Add("endkey", endKey)
 	}
 
@@ -919,3 +918,22 @@ func IsJSON(s string) bool {
 	var js map[string]interface{}
 	return json.Unmarshal([]byte(s), &js) == nil
 }
+
+// encodePathElement uses Golang for encoding and in addition, replaces a '/' by %2F.
+// Otherwise, in the regular encoding, a '/' is treated as a path separator in the url
+func encodePathElement(str string) string {
+	u := &url.URL{}
+	u.Path = str
+	encodedStr := u.String()
+	encodedStr = strings.Replace(encodedStr, "/", "%2F", -1)
+	return encodedStr
+}
+
+func encodeForJSON(str string) (string, error) {
+	buf := &bytes.Buffer{}
+	encoder := json.NewEncoder(buf)
+	if err := encoder.Encode(str); err != nil {
+		return "", err
+	}
+	return buf.String(), nil
+}