Skip to content

Commit

Permalink
[FAB-6175] Add index management to couchdb layer
Browse files Browse the repository at this point in the history 
Add the ability to create/update, delete and list indexes
to the couchdb layer.

Change-Id: Ia79243a6aff4665cca0f3d144ea24cdd8f312fc4
Signed-off-by: Chris Elder <chris.elder@us.ibm.com>
  • Loading branch information
Chris Elder committed Dec 14, 2017
1 parent 3e9b686 commit 1dc96ee
Show file tree
Hide file tree
Showing 2 changed files with 324 additions and 0 deletions.
137 changes: 137 additions & 0 deletions core/ledger/util/couchdb/couchdb.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,27 @@ type Base64Attachment struct {
AttachmentData string `json:"data"`
}

//ListIndexResponse contains the definition for listing couchdb indexes
type ListIndexResponse struct {
TotalRows int `json:"total_rows"`
Indexes []IndexDefinition `json:"indexes"`
}

//IndexDefinition contains the definition for a couchdb index
type IndexDefinition struct {
DesignDocument string `json:"ddoc"`
Name string `json:"name"`
Type string `json:"type"`
Definition json.RawMessage `json:"def"`
}

//IndexResult contains the definition for a couchdb index
type IndexResult struct {
DesignDocument string `json:"designdoc"`
Name string `json:"name"`
Definition string `json:"definition"`
}

// closeResponseBody discards the body and then closes it to enable returning it to
// connection pool
func closeResponseBody(resp *http.Response) {
Expand Down Expand Up @@ -1006,6 +1027,122 @@ func (dbclient *CouchDatabase) QueryDocuments(query string) (*[]QueryResult, err

}

// ListIndex method lists the defined indexes for a database
func (dbclient *CouchDatabase) ListIndex() (*[]IndexResult, error) {

logger.Debugf("Entering ListIndex()")

indexURL, err := url.Parse(dbclient.CouchInstance.conf.URL)
if err != nil {
logger.Errorf("URL parse error: %s", err.Error())
return nil, err
}

indexURL.Path = dbclient.DBName + "/_index/"

//get the number of retries
maxRetries := dbclient.CouchInstance.conf.MaxRetries

resp, _, err := dbclient.CouchInstance.handleRequest(http.MethodGet, indexURL.String(), nil, "", "", maxRetries, true)
if err != nil {
return nil, err
}
defer closeResponseBody(resp)

//handle as JSON document
jsonResponseRaw, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}

var jsonResponse = &ListIndexResponse{}

err2 := json.Unmarshal(jsonResponseRaw, &jsonResponse)
if err2 != nil {
return nil, err2
}

var results []IndexResult

for _, row := range jsonResponse.Indexes {

//if the DesignDocument does not begin with "_design/", then this is a system
//level index and is not meaningful and cannot be edited or deleted
designDoc := row.DesignDocument
s := strings.SplitAfterN(designDoc, "_design/", 2)
if len(s) > 1 {
designDoc = s[1]

//Add the index definition to the results
var addIndexResult = &IndexResult{DesignDocument: designDoc, Name: row.Name, Definition: fmt.Sprintf("%s", row.Definition)}
results = append(results, *addIndexResult)
}

}

logger.Debugf("Exiting ListIndex()")

return &results, nil

}

// CreateIndex method provides a function creating an index
func (dbclient *CouchDatabase) CreateIndex(indexdefinition string) error {

logger.Debugf("Entering CreateIndex() indexdefinition=%s", indexdefinition)

//Test to see if this is a valid JSON
if IsJSON(indexdefinition) != true {
return fmt.Errorf("JSON format is not valid")
}

indexURL, err := url.Parse(dbclient.CouchInstance.conf.URL)
if err != nil {
logger.Errorf("URL parse error: %s", err.Error())
return err
}

indexURL.Path = dbclient.DBName + "/_index"

//get the number of retries
maxRetries := dbclient.CouchInstance.conf.MaxRetries

resp, _, err := dbclient.CouchInstance.handleRequest(http.MethodPost, indexURL.String(), []byte(indexdefinition), "", "", maxRetries, true)
if err != nil {
return err
}
defer closeResponseBody(resp)

return nil

}

// DeleteIndex method provides a function deleting an index
func (dbclient *CouchDatabase) DeleteIndex(designdoc, indexname string) error {

logger.Debugf("Entering DeleteIndex() designdoc=%s indexname=%s", designdoc, indexname)

indexURL, err := url.Parse(dbclient.CouchInstance.conf.URL)
if err != nil {
logger.Errorf("URL parse error: %s", err.Error())
return err
}

indexURL.Path = dbclient.DBName + "/_index/" + designdoc + "/json/" + indexname

//get the number of retries
maxRetries := dbclient.CouchInstance.conf.MaxRetries

resp, _, err := dbclient.CouchInstance.handleRequest(http.MethodDelete, indexURL.String(), nil, "", "", maxRetries, true)
if err != nil {
return err
}
defer closeResponseBody(resp)

return nil

}

//BatchRetrieveDocumentMetadata - batch method to retrieve document metadata for a set of keys,
// including ID, couchdb revision number, and ledger version
func (dbclient *CouchDatabase) BatchRetrieveDocumentMetadata(keys []string) ([]*DocMetadata, error) {
Expand Down
187 changes: 187 additions & 0 deletions core/ledger/util/couchdb/couchdb_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import (
"github.com/hyperledger/fabric/common/ledger/testutil"
"github.com/hyperledger/fabric/core/ledger/ledgerconfig"
ledgertestutil "github.com/hyperledger/fabric/core/ledger/testutil"
logging "github.com/op/go-logging"
"github.com/spf13/viper"
)

Expand Down Expand Up @@ -82,6 +83,11 @@ func TestMain(m *testing.M) {
viper.Set("ledger.state.couchDBConfig.maxRetriesOnStartup", 10)
viper.Set("ledger.state.couchDBConfig.requestTimeout", time.Second*35)

//set the logging level to DEBUG to test debug only code
logging.SetLevel(logging.DEBUG, "couchdb")

viper.Set("logging.peer", "debug")

// Create CouchDB definition from config parameters
couchDBDef = GetCouchDBDefinition()

Expand Down Expand Up @@ -205,6 +211,18 @@ func TestBadCouchDBInstance(t *testing.T) {
_, err = badDB.BatchUpdateDocuments(nil)
testutil.AssertError(t, err, "Error should have been thrown with BatchUpdateDocuments and invalid connection")

//Test ListIndex with bad connection
_, err = badDB.ListIndex()
testutil.AssertError(t, err, "Error should have been thrown with ListIndex and invalid connection")

//Test CreateIndex with bad connection
err = badDB.CreateIndex("")
testutil.AssertError(t, err, "Error should have been thrown with CreateIndex and invalid connection")

//Test DeleteIndex with bad connection
err = badDB.DeleteIndex("", "")
testutil.AssertError(t, err, "Error should have been thrown with DeleteIndex and invalid connection")

}

func TestDBCreateSaveWithoutRevision(t *testing.T) {
Expand Down Expand Up @@ -316,6 +334,27 @@ func TestDBBadConnection(t *testing.T) {
}
}

func TestBadDBCredentials(t *testing.T) {

if ledgerconfig.IsCouchDBEnabled() {

database := "testdbbadcredentials"
err := cleanup(database)
testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to cleanup Error: %s", err))
defer cleanup(database)

if err == nil {
//create a new instance and database object
_, err := CreateCouchInstance(couchDBDef.URL, "fred", "fred",
couchDBDef.MaxRetries, couchDBDef.MaxRetriesOnStartup, couchDBDef.RequestTimeout)
testutil.AssertError(t, err, fmt.Sprintf("Error should have been thrown for bad credentials"))

}

}

}

func TestDBCreateDatabaseAndPersist(t *testing.T) {

if ledgerconfig.IsCouchDBEnabled() {
Expand Down Expand Up @@ -862,6 +901,154 @@ func TestCouchDBVersion(t *testing.T) {

}

func TestIndexOperations(t *testing.T) {

if ledgerconfig.IsCouchDBEnabled() {

database := "testindexoperations"
err := cleanup(database)
testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to cleanup Error: %s", err))
defer cleanup(database)

byteJSON1 := []byte(`{"_id":"1", "asset_name":"marble1","color":"blue","size":1,"owner":"jerry"}`)
byteJSON2 := []byte(`{"_id":"2", "asset_name":"marble2","color":"red","size":2,"owner":"tom"}`)
byteJSON3 := []byte(`{"_id":"3", "asset_name":"marble3","color":"green","size":3,"owner":"jerry"}`)
byteJSON4 := []byte(`{"_id":"4", "asset_name":"marble4","color":"purple","size":4,"owner":"tom"}`)
byteJSON5 := []byte(`{"_id":"5", "asset_name":"marble5","color":"blue","size":5,"owner":"jerry"}`)
byteJSON6 := []byte(`{"_id":"6", "asset_name":"marble6","color":"white","size":6,"owner":"tom"}`)
byteJSON7 := []byte(`{"_id":"7", "asset_name":"marble7","color":"white","size":7,"owner":"tom"}`)
byteJSON8 := []byte(`{"_id":"8", "asset_name":"marble8","color":"white","size":8,"owner":"tom"}`)
byteJSON9 := []byte(`{"_id":"9", "asset_name":"marble9","color":"white","size":9,"owner":"tom"}`)
byteJSON10 := []byte(`{"_id":"10", "asset_name":"marble10","color":"white","size":10,"owner":"tom"}`)

//create a new instance and database object --------------------------------------------------------
couchInstance, err := CreateCouchInstance(couchDBDef.URL, couchDBDef.Username, couchDBDef.Password,
couchDBDef.MaxRetries, couchDBDef.MaxRetriesOnStartup, couchDBDef.RequestTimeout)
testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to create couch instance"))
db := CouchDatabase{CouchInstance: *couchInstance, DBName: database}

//create a new database
_, errdb := db.CreateDatabaseIfNotExist()
testutil.AssertNoError(t, errdb, fmt.Sprintf("Error when trying to create database"))

batchUpdateDocs := []*CouchDoc{}

batchUpdateDocs = append(batchUpdateDocs, &CouchDoc{JSONValue: byteJSON1, Attachments: nil})
batchUpdateDocs = append(batchUpdateDocs, &CouchDoc{JSONValue: byteJSON2, Attachments: nil})
batchUpdateDocs = append(batchUpdateDocs, &CouchDoc{JSONValue: byteJSON3, Attachments: nil})
batchUpdateDocs = append(batchUpdateDocs, &CouchDoc{JSONValue: byteJSON4, Attachments: nil})
batchUpdateDocs = append(batchUpdateDocs, &CouchDoc{JSONValue: byteJSON5, Attachments: nil})
batchUpdateDocs = append(batchUpdateDocs, &CouchDoc{JSONValue: byteJSON6, Attachments: nil})
batchUpdateDocs = append(batchUpdateDocs, &CouchDoc{JSONValue: byteJSON7, Attachments: nil})
batchUpdateDocs = append(batchUpdateDocs, &CouchDoc{JSONValue: byteJSON8, Attachments: nil})
batchUpdateDocs = append(batchUpdateDocs, &CouchDoc{JSONValue: byteJSON9, Attachments: nil})
batchUpdateDocs = append(batchUpdateDocs, &CouchDoc{JSONValue: byteJSON10, Attachments: nil})

_, err = db.BatchUpdateDocuments(batchUpdateDocs)
testutil.AssertNoError(t, err, fmt.Sprintf("Error adding batch of documents"))

//Create an index definition
indexDefSize := "{\"index\":{\"fields\":[{\"size\":\"desc\"}]},\"ddoc\":\"indexSizeSortDoc\", \"name\":\"indexSizeSortName\",\"type\":\"json\"}"

//Create the index
err = db.CreateIndex(indexDefSize)
testutil.AssertNoError(t, err, fmt.Sprintf("Error thrown while creating an index"))

//Retrieve the index
listResult, err := db.ListIndex()
testutil.AssertNoError(t, err, fmt.Sprintf("Error thrown while retrieving indexes"))

//There should only be one item returned
testutil.AssertEquals(t, len(*listResult), 1)
for _, elem := range *listResult {
testutil.AssertEquals(t, elem.DesignDocument, "indexSizeSortDoc")
testutil.AssertEquals(t, elem.Name, "indexSizeSortName")
testutil.AssertEquals(t, elem.Definition, "{\"fields\":[{\"size\":\"desc\"}]}")
}

//Create an index definition with no DesignDocument or name
indexDefColor := "{\"index\":{\"fields\":[{\"color\":\"desc\"}]}}"

//Create the index
err = db.CreateIndex(indexDefColor)
testutil.AssertNoError(t, err, fmt.Sprintf("Error thrown while creating an index"))

//Retrieve the list of indexes
listResult, err = db.ListIndex()
testutil.AssertNoError(t, err, fmt.Sprintf("Error thrown while retrieving indexes"))

//There should be two indexes returned
testutil.AssertEquals(t, len(*listResult), 2)

//Delete the named index
err = db.DeleteIndex("indexSizeSortDoc", "indexSizeSortName")
testutil.AssertNoError(t, err, fmt.Sprintf("Error thrown while deleting an index"))

//Retrieve the list of indexes
listResult, err = db.ListIndex()
testutil.AssertNoError(t, err, fmt.Sprintf("Error thrown while retrieving indexes"))

//There should be one index returned
testutil.AssertEquals(t, len(*listResult), 1)

//Delete the unnamed index
for _, elem := range *listResult {
err = db.DeleteIndex(elem.DesignDocument, string(elem.Name))
testutil.AssertNoError(t, err, fmt.Sprintf("Error thrown while deleting an index"))
}

//Retrieve the list of indexes, should be zero
listResult, err = db.ListIndex()
testutil.AssertNoError(t, err, fmt.Sprintf("Error thrown while retrieving indexes"))
testutil.AssertEquals(t, len(*listResult), 0)

//Create a query string with a descending sort, this will require an index
queryString := "{\"selector\":{\"size\": {\"$gt\": 0}},\"fields\": [\"_id\", \"_rev\", \"owner\", \"asset_name\", \"color\", \"size\"], \"sort\":[{\"size\":\"desc\"}], \"limit\": 10,\"skip\": 0}"

//Execute a query with a sort, this should throw the exception
_, err = db.QueryDocuments(queryString)
testutil.AssertError(t, err, fmt.Sprintf("Error thrown while querying without a valid index"))

//Create the index
err = db.CreateIndex(indexDefSize)
testutil.AssertNoError(t, err, fmt.Sprintf("Error thrown while creating an index"))

//Execute a query with an index, this should succeed
_, err = db.QueryDocuments(queryString)
testutil.AssertNoError(t, err, fmt.Sprintf("Error thrown while querying with an index"))

//Create another index definition
indexDefSize = "{\"index\":{\"fields\":[{\"data.size\":\"desc\"},{\"data.owner\":\"desc\"}]},\"ddoc\":\"indexSizeOwnerSortDoc\", \"name\":\"indexSizeOwnerSortName\",\"type\":\"json\"}"

//Create the index
err = db.CreateIndex(indexDefSize)
testutil.AssertNoError(t, err, fmt.Sprintf("Error thrown while creating an index"))

//Retrieve the indexes
listResult, err = db.ListIndex()
testutil.AssertNoError(t, err, fmt.Sprintf("Error thrown while retrieving indexes"))

//There should only be two definitions
testutil.AssertEquals(t, len(*listResult), 2)

//Create an invalid index definition with an invalid JSON
indexDefSize = "{\"index\"{\"fields\":[{\"data.size\":\"desc\"},{\"data.owner\":\"desc\"}]},\"ddoc\":\"indexSizeOwnerSortDoc\", \"name\":\"indexSizeOwnerSortName\",\"type\":\"json\"}"

//Create the index
err = db.CreateIndex(indexDefSize)
testutil.AssertError(t, err, fmt.Sprintf("Error should have been thrown for an invalid index JSON"))

//Create an invalid index definition with a valid JSON and an invalid index definition
indexDefSize = "{\"index\":{\"fields2\":[{\"data.size\":\"desc\"},{\"data.owner\":\"desc\"}]},\"ddoc\":\"indexSizeOwnerSortDoc\", \"name\":\"indexSizeOwnerSortName\",\"type\":\"json\"}"

//Create the index
err = db.CreateIndex(indexDefSize)
testutil.AssertError(t, err, fmt.Sprintf("Error should have been thrown for an invalid index definition"))

}

}

func TestRichQuery(t *testing.T) {

if ledgerconfig.IsCouchDBEnabled() {
Expand Down

0 comments on commit 1dc96ee

Please sign in to comment.