[FAB-8446] Add couchdb index validation to LSCC

The existing validation for packaged couchdb indexes in chaincode
was only getting called if peer CLI packaged the index during
chaincode install (client side).
Need the same validation logic called on server side of chaincocde
install, so that the same validation logic will be in effect
regardless of whether a peer CLI client or an SDK client performed
the chaincode install.  This implies making the same validation
logic available in the LSCC chaincode install function. The benefit
is that each client does not need to perform their own validation,
they can rely on the server side LSCC validation, and any validation
errors will get returned to them on the chaincode install failure
response.

This change ensures that the same validation is called during peer
CLI packaging and during LSCC install chaincode.

Change-Id: I44692141f6efe430fd5e298c8df7d59e519ce028
Signed-off-by: David Enyeart <enyeart@us.ibm.com>

core/chaincode/platforms/golang/platform.go

@@ -22,6 +22,7 @@ import (
 	"compress/gzip"
 	"errors"
 	"fmt"
+	"io/ioutil"
 	"net/url"
 	"os"
 	"path/filepath"
@@ -428,22 +429,21 @@ func (goPlatform *Platform) GetDeploymentPayload(spec *pb.ChaincodeSpec) ([]byte
 
 	for _, file := range files {
 
+		// file.Path represents os localpath
+		// file.Name represents tar packagepath
+
 		// If the file is metadata rather than golang code, remove the leading go code path, for example:
 		// original file.Name:  src/github.com/hyperledger/fabric/examples/chaincode/go/marbles02/META-INF/statedb/couchdb/indexes/indexOwner.json
 		// updated file.Name:   META-INF/statedb/couchdb/indexes/indexOwner.json
 		if file.IsMetadata {
 
-			// Ensure META-INF directory can be found, then grab the META-INF relative path to use for packaging
-			if !strings.HasPrefix(file.Name, filepath.Join("src", code.Pkg, "META-INF")) {
-				return nil, fmt.Errorf("Could not find META-INF directory in metadata file %s.", file.Name)
-			}
 			file.Name, err = filepath.Rel(filepath.Join("src", code.Pkg), file.Name)
 			if err != nil {
-				return nil, fmt.Errorf("Could not get relative path for META-INF directory %s. Error:%s", file.Name, err)
+				return nil, fmt.Errorf("This error was caused by bad packaging of the metadata.  The file [%s] is marked as MetaFile, however not located under META-INF   Error:[%s]", file.Name, err)
 			}
 
-			// Split the filename itself from its path
-			_, filename := filepath.Split(file.Name)
+			// Split the tar location (file.Name) into a tar package directory and filename
+			packageDir, filename := filepath.Split(file.Name)
 
 			// Hidden files are not supported as metadata, therefore ignore them.
 			// User often doesn't know that hidden files are there, and may not be able to delete them, therefore warn user rather than error out.
@@ -452,9 +452,15 @@ func (goPlatform *Platform) GetDeploymentPayload(spec *pb.ChaincodeSpec) ([]byte
 				continue
 			}
 
+			fileBytes, err := ioutil.ReadFile(file.Path)
+			if err != nil {
+				return nil, err
+			}
+
 			// Validate metadata file for inclusion in tar
 			// Validation is based on the passed metadata directory, e.g. META-INF/statedb/couchdb/indexes
-			err = ccmetadata.ValidateMetadataFile(file.Path, filepath.Dir(file.Name))
+			// Clean metadata directory to remove trailing slash
+			err = ccmetadata.ValidateMetadataFile(filename, fileBytes, filepath.Clean(packageDir))
 			if err != nil {
 				return nil, err
 			}

core/common/ccprovider/cc_statedb_artifacts_provider.go

@@ -11,13 +11,21 @@ import (
 	"bytes"
 	"compress/gzip"
 	"io"
+	"io/ioutil"
+	"path/filepath"
 	"strings"
 )
 
 const (
 	ccPackageStatedbDir = "META-INF/statedb/"
 )
 
+// tarFileEntry encapsulates a file entry and it's contents inside a tar
+type TarFileEntry struct {
+	FileHeader  *tar.Header
+	FileContent []byte
+}
+
 // ExtractStatedbArtifactsAsTarbytes extracts the statedb artifacts from the code package tar and create a statedb artifact tar.
 // The state db artifacts are expected to contain state db specific artifacts such as index specification in the case of couchdb.
 // This function is intented to be used during chaincode instantiate/upgrade so that statedb artifacts can be created.
@@ -83,3 +91,35 @@ func ExtractStatedbArtifactsFromCCPackage(ccpackage CCPackage) (statedbArtifacts
 	ccproviderLogger.Debug("Created statedb artifact tar")
 	return statedbTarBuffer.Bytes(), nil
 }
+
+// ExtractFileEntries extract file entries from the given `tarBytes`. A file entry is included in the
+// returned results only if it is located in the dir specified in the `filterDirs` parameter
+func ExtractFileEntries(tarBytes []byte, filterDirs map[string]bool) ([]*TarFileEntry, error) {
+	var fileEntries []*TarFileEntry
+	//initialize a tar reader
+	tarReader := tar.NewReader(bytes.NewReader(tarBytes))
+	for {
+		//read the next header from the tar
+		tarHeader, err := tarReader.Next()
+		//if the EOF is detected, then exit
+		if err == io.EOF {
+			// end of tar archive
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+		ccproviderLogger.Debugf("Processing entry from tar: %s", tarHeader.Name)
+		//Ensure that this is a file located in the dir present in the 'filterDirs'
+		if !tarHeader.FileInfo().IsDir() && filterDirs[filepath.Dir(tarHeader.Name)] {
+			ccproviderLogger.Debugf("Selecting file entry from tar: %s", tarHeader.Name)
+			//read the tar entry into a byte array
+			fileContent, err := ioutil.ReadAll(tarReader)
+			if err != nil {
+				return nil, err
+			}
+			fileEntries = append(fileEntries, &TarFileEntry{tarHeader, fileContent})
+		}
+	}
+	return fileEntries, nil
+}

core/common/ccprovider/metadata/validators.go

@@ -9,7 +9,6 @@ package metadata
 import (
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"path/filepath"
 	"reflect"
 	"strings"
@@ -20,7 +19,7 @@ import (
 var logger = flogging.MustGetLogger("metadata")
 
 // fileValidators are used as handlers to validate specific metadata directories
-type fileValidator func(srcPath string) error
+type fileValidator func(fileName string, fileBytes []byte) error
 
 // Currently, the only metadata expected and allowed is for META-INF/statedb/couchdb/indexes.
 var fileValidators = map[string]fileValidator{
@@ -56,8 +55,7 @@ func (e *InvalidIndexContentError) Error() string {
 
 // ValidateMetadataFile checks that metadata files are valid
 // according to the validation rules of the metadata directory (metadataType)
-func ValidateMetadataFile(srcPath, metadataType string) error {
-
+func ValidateMetadataFile(fileName string, fileBytes []byte, metadataType string) error {
 	// Get the validator handler for the metadata directory
 	fileValidator, ok := fileValidators[metadataType]
 
@@ -66,8 +64,8 @@ func ValidateMetadataFile(srcPath, metadataType string) error {
 		return &UnhandledDirectoryError{fmt.Sprintf("Metadata not supported in directory: %s", metadataType)}
 	}
 
-	// If the file is not valid for the given metadata directory, return an error
-	err := fileValidator(srcPath)
+	// If the file is not valid for the given metadata directory, return the corresponding error
+	err := fileValidator(fileName, fileBytes)
 	if err != nil {
 		return err
 	}
@@ -77,30 +75,25 @@ func ValidateMetadataFile(srcPath, metadataType string) error {
 }
 
 // couchdbIndexFileValidator implements fileValidator
-func couchdbIndexFileValidator(srcPath string) error {
+func couchdbIndexFileValidator(fileName string, fileBytes []byte) error {
 
-	ext := filepath.Ext(srcPath)
+	ext := filepath.Ext(fileName)
 
 	// if the file does not have a .json extension, then return as error
 	if ext != ".json" {
-		return &BadExtensionError{fmt.Sprintf("Index metadata file [%s] does not have a .json extension", srcPath)}
-	}
-
-	fileBytes, err := ioutil.ReadFile(srcPath)
-	if err != nil {
-		return err
+		return &BadExtensionError{fmt.Sprintf("Index metadata file [%s] does not have a .json extension", fileName)}
 	}
 
 	// if the content does not validate as JSON, return err to invalidate the file
 	boolIsJSON, indexDefinition := isJSON(fileBytes)
 	if !boolIsJSON {
-		return &InvalidIndexContentError{fmt.Sprintf("Index metadata file [%s] is not a valid JSON", srcPath)}
+		return &InvalidIndexContentError{fmt.Sprintf("Index metadata file [%s] is not a valid JSON", fileName)}
 	}
 
 	// validate the index definition
-	err = validateIndexJSON(indexDefinition)
+	err := validateIndexJSON(indexDefinition)
 	if err != nil {
-		return &InvalidIndexContentError{fmt.Sprintf("Index metadata file [%s] is not a valid index definition: %s", srcPath, err)}
+		return &InvalidIndexContentError{fmt.Sprintf("Index metadata file [%s] is not a valid index definition: %s", fileName, err)}
 	}
 
 	return nil

core/common/ccprovider/metadata/validators_test.go

@@ -22,13 +22,11 @@ func TestGoodIndexJSON(t *testing.T) {
 	cleanupDir(testDir)
 	defer cleanupDir(testDir)
 
-	filename := filepath.Join(testDir, "META-INF/statedb/couchdb/indexes", "myIndex.json")
-	filebytes := []byte(`{"index":{"fields":["data.docType","data.owner"]},"name":"indexOwner","type":"json"}`)
+	fileName := "myIndex.json"
+	fileBytes := []byte(`{"index":{"fields":["data.docType","data.owner"]},"name":"indexOwner","type":"json"}`)
+	metadataType := "META-INF/statedb/couchdb/indexes"
 
-	err := writeToFile(filename, filebytes)
-	assert.NoError(t, err, "Error writing to file")
-
-	err = ValidateMetadataFile(filename, "META-INF/statedb/couchdb/indexes")
+	err := ValidateMetadataFile(fileName, fileBytes, metadataType)
 	assert.NoError(t, err, "Error validating a good index")
 }
 
@@ -37,13 +35,11 @@ func TestBadIndexJSON(t *testing.T) {
 	cleanupDir(testDir)
 	defer cleanupDir(testDir)
 
-	filename := filepath.Join(testDir, "META-INF/statedb/couchdb/indexes", "myIndex.json")
-	filebytes := []byte("invalid json")
-
-	err := writeToFile(filename, filebytes)
-	assert.NoError(t, err, "Error writing to file")
+	fileName := "myIndex.json"
+	fileBytes := []byte("invalid json")
+	metadataType := "META-INF/statedb/couchdb/indexes"
 
-	err = ValidateMetadataFile(filename, "META-INF/statedb/couchdb/indexes")
+	err := ValidateMetadataFile(fileName, fileBytes, metadataType)
 
 	assert.Error(t, err, "Should have received an InvalidIndexContentError")
 
@@ -59,14 +55,12 @@ func TestIndexWrongLocation(t *testing.T) {
 	cleanupDir(testDir)
 	defer cleanupDir(testDir)
 
+	fileName := "myIndex.json"
+	fileBytes := []byte(`{"index":{"fields":["data.docType","data.owner"]},"name":"indexOwner","type":"json"}`)
 	// place the index one directory too high
-	filename := filepath.Join(testDir, "META-INF/statedb/couchdb", "myIndex.json")
-	filebytes := []byte("invalid json")
+	metadataType := "META-INF/statedb/couchdb"
 
-	err := writeToFile(filename, filebytes)
-	assert.NoError(t, err, "Error writing to file")
-
-	err = ValidateMetadataFile(filename, "META-INF/statedb/couchdb")
+	err := ValidateMetadataFile(fileName, fileBytes, metadataType)
 	assert.Error(t, err, "Should have received an UnhandledDirectoryError")
 
 	// Type assertion on UnhandledDirectoryError
@@ -81,48 +75,28 @@ func TestInvalidMetadataType(t *testing.T) {
 	cleanupDir(testDir)
 	defer cleanupDir(testDir)
 
-	filename := filepath.Join(testDir, "META-INF/statedb/couchdb/indexes", "myIndex.json")
-	filebytes := []byte("invalid json")
-
-	err := writeToFile(filename, filebytes)
-	assert.NoError(t, err, "Error writing to file")
+	fileName := "myIndex.json"
+	fileBytes := []byte(`{"index":{"fields":["data.docType","data.owner"]},"name":"indexOwner","type":"json"}`)
+	metadataType := "Invalid metadata type"
 
-	err = ValidateMetadataFile(filename, "Invalid metadata type")
+	err := ValidateMetadataFile(fileName, fileBytes, metadataType)
 	assert.Error(t, err, "Should have received an UnhandledDirectoryError")
 
 	// Type assertion on UnhandledDirectoryError
 	_, ok := err.(*UnhandledDirectoryError)
 	assert.True(t, ok, "Should have received an UnhandledDirectoryError")
 }
 
-func TestCantReadFile(t *testing.T) {
-	testDir := filepath.Join(packageTestDir, "CantReadFile")
-	cleanupDir(testDir)
-	defer cleanupDir(testDir)
-
-	filename := filepath.Join(testDir, "META-INF/statedb/couchdb/indexes", "myIndex.json")
-
-	// Don't write the file - test for can't read file
-	// err := writeToFile(filename, filebytes)
-	// assert.NoError(t, err, "Error writing to file")
-
-	err := ValidateMetadataFile(filename, "META-INF/statedb/couchdb/indexes")
-	assert.Error(t, err, "Should have received error reading file")
-
-}
-
 func TestBadMetadataExtension(t *testing.T) {
 	testDir := filepath.Join(packageTestDir, "BadMetadataExtension")
 	cleanupDir(testDir)
 	defer cleanupDir(testDir)
 
-	filename := filepath.Join(testDir, "META-INF/statedb/couchdb/indexes", "myIndex.go")
-	filebytes := []byte(`{"index":{"fields":["data.docType","data.owner"]},"name":"indexOwner","type":"json"}`)
-
-	err := writeToFile(filename, filebytes)
-	assert.NoError(t, err, "Error writing to file")
+	fileName := "myIndex.go"
+	fileBytes := []byte(`{"index":{"fields":["data.docType","data.owner"]},"name":"indexOwner","type":"json"}`)
+	metadataType := "META-INF/statedb/couchdb/indexes"
 
-	err = ValidateMetadataFile(filename, "META-INF/statedb/couchdb/indexes")
+	err := ValidateMetadataFile(fileName, fileBytes, metadataType)
 	assert.Error(t, err, "Should have received an BadExtensionError")
 
 	// Type assertion on BadExtensionError

core/container/util/writer.go

@@ -11,13 +11,14 @@ import (
 	"bufio"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
 	"time"
 
 	"github.com/hyperledger/fabric/common/flogging"
-	ccmetadata "github.com/hyperledger/fabric/core/common/ccprovider/metadata"
+	"github.com/hyperledger/fabric/core/common/ccprovider/metadata"
 	"github.com/pkg/errors"
 )
 
@@ -49,10 +50,10 @@ func WriteFolderToTarPackage(tw *tar.Writer, srcPath string, excludeDir string,
 	}
 
 	rootDirLen := len(rootDirectory)
-	walkFn := func(path string, info os.FileInfo, err error) error {
+	walkFn := func(localpath string, info os.FileInfo, err error) error {
 
-		// If path includes .git, ignore
-		if strings.Contains(path, ".git") {
+		// If localpath includes .git, ignore
+		if strings.Contains(localpath, ".git") {
 			return nil
 		}
 
@@ -61,15 +62,15 @@ func WriteFolderToTarPackage(tw *tar.Writer, srcPath string, excludeDir string,
 		}
 
 		//exclude any files with excludeDir prefix. They should already be in the tar
-		if excludeDir != "" && strings.Index(path, excludeDir) == rootDirLen+1 {
+		if excludeDir != "" && strings.Index(localpath, excludeDir) == rootDirLen+1 {
 			//1 for "/"
 			return nil
 		}
 		// Because of scoping we can reference the external rootDirectory variable
-		if len(path[rootDirLen:]) == 0 {
+		if len(localpath[rootDirLen:]) == 0 {
 			return nil
 		}
-		ext := filepath.Ext(path)
+		ext := filepath.Ext(localpath)
 
 		if includeFileTypeMap != nil {
 			// we only want 'fileTypes' source files at this point
@@ -85,35 +86,41 @@ func WriteFolderToTarPackage(tw *tar.Writer, srcPath string, excludeDir string,
 			}
 		}
 
-		var newPath string
+		var packagepath string
 
 		// if file is metadata, keep the /META-INF directory, e.g: META-INF/statedb/couchdb/indexes/indexOwner.json
 		// otherwise file is source code, put it in /src dir, e.g: src/marbles_chaincode.js
-		if strings.HasPrefix(path, filepath.Join(rootDirectory, "META-INF")) {
-			newPath = path[rootDirLen+1:]
+		if strings.HasPrefix(localpath, filepath.Join(rootDirectory, "META-INF")) {
+			packagepath = localpath[rootDirLen+1:]
 
-			// Split the filename itself from its path
-			_, filename := filepath.Split(newPath)
+			// Split the tar packagepath into a tar package directory and filename
+			packageDir, filename := filepath.Split(packagepath)
 
 			// Hidden files are not supported as metadata, therefore ignore them.
 			// User often doesn't know that hidden files are there, and may not be able to delete them, therefore warn user rather than error out.
 			if strings.HasPrefix(filename, ".") {
-				vmLogger.Warningf("Ignoring hidden file in metadata directory: %s", newPath)
+				vmLogger.Warningf("Ignoring hidden file in metadata directory: %s", packagepath)
 				return nil
 			}
 
+			fileBytes, err := ioutil.ReadFile(localpath)
+			if err != nil {
+				return err
+			}
+
 			// Validate metadata file for inclusion in tar
 			// Validation is based on the passed metadata directory, e.g. META-INF/statedb/couchdb/indexes
-			err = ccmetadata.ValidateMetadataFile(path, filepath.Dir(newPath))
+			// Clean metadata directory to remove trailing slash
+			err = metadata.ValidateMetadataFile(filename, fileBytes, filepath.Clean(packageDir))
 			if err != nil {
 				return err
 			}
 
 		} else { // file is not metadata, include in src
-			newPath = fmt.Sprintf("src%s", path[rootDirLen:])
+			packagepath = fmt.Sprintf("src%s", localpath[rootDirLen:])
 		}
 
-		err = WriteFileToPackage(path, newPath, tw)
+		err = WriteFileToPackage(localpath, packagepath, tw)
 		if err != nil {
 			return fmt.Errorf("Error writing file to package: %s", err)
 		}