Merge "[FAB-8446] Add couchdb index validation to LSCC"

core/chaincode/platforms/golang/platform.go

@@ -22,6 +22,7 @@ import (
 	"compress/gzip"
 	"errors"
 	"fmt"
+	"io/ioutil"
 	"net/url"
 	"os"
 	"path/filepath"
@@ -428,22 +429,21 @@ func (goPlatform *Platform) GetDeploymentPayload(spec *pb.ChaincodeSpec) ([]byte
 
 	for _, file := range files {
 
+		// file.Path represents os localpath
+		// file.Name represents tar packagepath
+
 		// If the file is metadata rather than golang code, remove the leading go code path, for example:
 		// original file.Name:  src/github.com/hyperledger/fabric/examples/chaincode/go/marbles02/META-INF/statedb/couchdb/indexes/indexOwner.json
 		// updated file.Name:   META-INF/statedb/couchdb/indexes/indexOwner.json
 		if file.IsMetadata {
 
-			// Ensure META-INF directory can be found, then grab the META-INF relative path to use for packaging
-			if !strings.HasPrefix(file.Name, filepath.Join("src", code.Pkg, "META-INF")) {
-				return nil, fmt.Errorf("Could not find META-INF directory in metadata file %s.", file.Name)
-			}
 			file.Name, err = filepath.Rel(filepath.Join("src", code.Pkg), file.Name)
 			if err != nil {
-				return nil, fmt.Errorf("Could not get relative path for META-INF directory %s. Error:%s", file.Name, err)
+				return nil, fmt.Errorf("This error was caused by bad packaging of the metadata.  The file [%s] is marked as MetaFile, however not located under META-INF   Error:[%s]", file.Name, err)
 			}
 
-			// Split the filename itself from its path
-			_, filename := filepath.Split(file.Name)
+			// Split the tar location (file.Name) into a tar package directory and filename
+			packageDir, filename := filepath.Split(file.Name)
 
 			// Hidden files are not supported as metadata, therefore ignore them.
 			// User often doesn't know that hidden files are there, and may not be able to delete them, therefore warn user rather than error out.
@@ -452,9 +452,15 @@ func (goPlatform *Platform) GetDeploymentPayload(spec *pb.ChaincodeSpec) ([]byte
 				continue
 			}
 
+			fileBytes, err := ioutil.ReadFile(file.Path)
+			if err != nil {
+				return nil, err
+			}
+
 			// Validate metadata file for inclusion in tar
 			// Validation is based on the passed metadata directory, e.g. META-INF/statedb/couchdb/indexes
-			err = ccmetadata.ValidateMetadataFile(file.Path, filepath.Dir(file.Name))
+			// Clean metadata directory to remove trailing slash
+			err = ccmetadata.ValidateMetadataFile(filename, fileBytes, filepath.Clean(packageDir))
 			if err != nil {
 				return nil, err
 			}

core/common/ccprovider/cc_statedb_artifacts_provider.go

@@ -11,13 +11,21 @@ import (
 	"bytes"
 	"compress/gzip"
 	"io"
+	"io/ioutil"
+	"path/filepath"
 	"strings"
 )
 
 const (
 	ccPackageStatedbDir = "META-INF/statedb/"
 )
 
+// tarFileEntry encapsulates a file entry and it's contents inside a tar
+type TarFileEntry struct {
+	FileHeader  *tar.Header
+	FileContent []byte
+}
+
 // ExtractStatedbArtifactsAsTarbytes extracts the statedb artifacts from the code package tar and create a statedb artifact tar.
 // The state db artifacts are expected to contain state db specific artifacts such as index specification in the case of couchdb.
 // This function is intented to be used during chaincode instantiate/upgrade so that statedb artifacts can be created.
@@ -83,3 +91,35 @@ func ExtractStatedbArtifactsFromCCPackage(ccpackage CCPackage) (statedbArtifacts
 	ccproviderLogger.Debug("Created statedb artifact tar")
 	return statedbTarBuffer.Bytes(), nil
 }
+
+// ExtractFileEntries extract file entries from the given `tarBytes`. A file entry is included in the
+// returned results only if it is located in the dir specified in the `filterDirs` parameter
+func ExtractFileEntries(tarBytes []byte, filterDirs map[string]bool) ([]*TarFileEntry, error) {
+	var fileEntries []*TarFileEntry
+	//initialize a tar reader
+	tarReader := tar.NewReader(bytes.NewReader(tarBytes))
+	for {
+		//read the next header from the tar
+		tarHeader, err := tarReader.Next()
+		//if the EOF is detected, then exit
+		if err == io.EOF {
+			// end of tar archive
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+		ccproviderLogger.Debugf("Processing entry from tar: %s", tarHeader.Name)
+		//Ensure that this is a file located in the dir present in the 'filterDirs'
+		if !tarHeader.FileInfo().IsDir() && filterDirs[filepath.Dir(tarHeader.Name)] {
+			ccproviderLogger.Debugf("Selecting file entry from tar: %s", tarHeader.Name)
+			//read the tar entry into a byte array
+			fileContent, err := ioutil.ReadAll(tarReader)
+			if err != nil {
+				return nil, err
+			}
+			fileEntries = append(fileEntries, &TarFileEntry{tarHeader, fileContent})
+		}
+	}
+	return fileEntries, nil
+}

core/common/ccprovider/metadata/validators.go

@@ -9,7 +9,6 @@ package metadata
 import (
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"path/filepath"
 	"reflect"
 	"strings"
@@ -20,7 +19,7 @@ import (
 var logger = flogging.MustGetLogger("metadata")
 
 // fileValidators are used as handlers to validate specific metadata directories
-type fileValidator func(srcPath string) error
+type fileValidator func(fileName string, fileBytes []byte) error
 
 // Currently, the only metadata expected and allowed is for META-INF/statedb/couchdb/indexes.
 var fileValidators = map[string]fileValidator{
@@ -56,8 +55,7 @@ func (e *InvalidIndexContentError) Error() string {
 
 // ValidateMetadataFile checks that metadata files are valid
 // according to the validation rules of the metadata directory (metadataType)
-func ValidateMetadataFile(srcPath, metadataType string) error {
-
+func ValidateMetadataFile(fileName string, fileBytes []byte, metadataType string) error {
 	// Get the validator handler for the metadata directory
 	fileValidator, ok := fileValidators[metadataType]
 
@@ -66,8 +64,8 @@ func ValidateMetadataFile(srcPath, metadataType string) error {
 		return &UnhandledDirectoryError{fmt.Sprintf("Metadata not supported in directory: %s", metadataType)}
 	}
 
-	// If the file is not valid for the given metadata directory, return an error
-	err := fileValidator(srcPath)
+	// If the file is not valid for the given metadata directory, return the corresponding error
+	err := fileValidator(fileName, fileBytes)
 	if err != nil {
 		return err
 	}
@@ -77,30 +75,25 @@ func ValidateMetadataFile(srcPath, metadataType string) error {
 }
 
 // couchdbIndexFileValidator implements fileValidator
-func couchdbIndexFileValidator(srcPath string) error {
+func couchdbIndexFileValidator(fileName string, fileBytes []byte) error {
 
-	ext := filepath.Ext(srcPath)
+	ext := filepath.Ext(fileName)
 
 	// if the file does not have a .json extension, then return as error
 	if ext != ".json" {
-		return &BadExtensionError{fmt.Sprintf("Index metadata file [%s] does not have a .json extension", srcPath)}
-	}
-
-	fileBytes, err := ioutil.ReadFile(srcPath)
-	if err != nil {
-		return err
+		return &BadExtensionError{fmt.Sprintf("Index metadata file [%s] does not have a .json extension", fileName)}
 	}
 
 	// if the content does not validate as JSON, return err to invalidate the file
 	boolIsJSON, indexDefinition := isJSON(fileBytes)
 	if !boolIsJSON {
-		return &InvalidIndexContentError{fmt.Sprintf("Index metadata file [%s] is not a valid JSON", srcPath)}
+		return &InvalidIndexContentError{fmt.Sprintf("Index metadata file [%s] is not a valid JSON", fileName)}
 	}
 
 	// validate the index definition
-	err = validateIndexJSON(indexDefinition)
+	err := validateIndexJSON(indexDefinition)
 	if err != nil {
-		return &InvalidIndexContentError{fmt.Sprintf("Index metadata file [%s] is not a valid index definition: %s", srcPath, err)}
+		return &InvalidIndexContentError{fmt.Sprintf("Index metadata file [%s] is not a valid index definition: %s", fileName, err)}
 	}
 
 	return nil

core/common/ccprovider/metadata/validators_test.go

@@ -22,13 +22,11 @@ func TestGoodIndexJSON(t *testing.T) {
 	cleanupDir(testDir)
 	defer cleanupDir(testDir)
 
-	filename := filepath.Join(testDir, "META-INF/statedb/couchdb/indexes", "myIndex.json")
-	filebytes := []byte(`{"index":{"fields":["data.docType","data.owner"]},"name":"indexOwner","type":"json"}`)
+	fileName := "myIndex.json"
+	fileBytes := []byte(`{"index":{"fields":["data.docType","data.owner"]},"name":"indexOwner","type":"json"}`)
+	metadataType := "META-INF/statedb/couchdb/indexes"
 
-	err := writeToFile(filename, filebytes)
-	assert.NoError(t, err, "Error writing to file")
-
-	err = ValidateMetadataFile(filename, "META-INF/statedb/couchdb/indexes")
+	err := ValidateMetadataFile(fileName, fileBytes, metadataType)
 	assert.NoError(t, err, "Error validating a good index")
 }
 
@@ -37,13 +35,11 @@ func TestBadIndexJSON(t *testing.T) {
 	cleanupDir(testDir)
 	defer cleanupDir(testDir)
 
-	filename := filepath.Join(testDir, "META-INF/statedb/couchdb/indexes", "myIndex.json")
-	filebytes := []byte("invalid json")
-
-	err := writeToFile(filename, filebytes)
-	assert.NoError(t, err, "Error writing to file")
+	fileName := "myIndex.json"
+	fileBytes := []byte("invalid json")
+	metadataType := "META-INF/statedb/couchdb/indexes"
 
-	err = ValidateMetadataFile(filename, "META-INF/statedb/couchdb/indexes")
+	err := ValidateMetadataFile(fileName, fileBytes, metadataType)
 
 	assert.Error(t, err, "Should have received an InvalidIndexContentError")
 
@@ -59,14 +55,12 @@ func TestIndexWrongLocation(t *testing.T) {
 	cleanupDir(testDir)
 	defer cleanupDir(testDir)
 
+	fileName := "myIndex.json"
+	fileBytes := []byte(`{"index":{"fields":["data.docType","data.owner"]},"name":"indexOwner","type":"json"}`)
 	// place the index one directory too high
-	filename := filepath.Join(testDir, "META-INF/statedb/couchdb", "myIndex.json")
-	filebytes := []byte("invalid json")
+	metadataType := "META-INF/statedb/couchdb"
 
-	err := writeToFile(filename, filebytes)
-	assert.NoError(t, err, "Error writing to file")
-
-	err = ValidateMetadataFile(filename, "META-INF/statedb/couchdb")
+	err := ValidateMetadataFile(fileName, fileBytes, metadataType)
 	assert.Error(t, err, "Should have received an UnhandledDirectoryError")
 
 	// Type assertion on UnhandledDirectoryError
@@ -81,48 +75,28 @@ func TestInvalidMetadataType(t *testing.T) {
 	cleanupDir(testDir)
 	defer cleanupDir(testDir)
 
-	filename := filepath.Join(testDir, "META-INF/statedb/couchdb/indexes", "myIndex.json")
-	filebytes := []byte("invalid json")
-
-	err := writeToFile(filename, filebytes)
-	assert.NoError(t, err, "Error writing to file")
+	fileName := "myIndex.json"
+	fileBytes := []byte(`{"index":{"fields":["data.docType","data.owner"]},"name":"indexOwner","type":"json"}`)
+	metadataType := "Invalid metadata type"
 
-	err = ValidateMetadataFile(filename, "Invalid metadata type")
+	err := ValidateMetadataFile(fileName, fileBytes, metadataType)
 	assert.Error(t, err, "Should have received an UnhandledDirectoryError")
 
 	// Type assertion on UnhandledDirectoryError
 	_, ok := err.(*UnhandledDirectoryError)
 	assert.True(t, ok, "Should have received an UnhandledDirectoryError")
 }
 
-func TestCantReadFile(t *testing.T) {
-	testDir := filepath.Join(packageTestDir, "CantReadFile")
-	cleanupDir(testDir)
-	defer cleanupDir(testDir)
-
-	filename := filepath.Join(testDir, "META-INF/statedb/couchdb/indexes", "myIndex.json")
-
-	// Don't write the file - test for can't read file
-	// err := writeToFile(filename, filebytes)
-	// assert.NoError(t, err, "Error writing to file")
-
-	err := ValidateMetadataFile(filename, "META-INF/statedb/couchdb/indexes")
-	assert.Error(t, err, "Should have received error reading file")
-
-}
-
 func TestBadMetadataExtension(t *testing.T) {
 	testDir := filepath.Join(packageTestDir, "BadMetadataExtension")
 	cleanupDir(testDir)
 	defer cleanupDir(testDir)
 
-	filename := filepath.Join(testDir, "META-INF/statedb/couchdb/indexes", "myIndex.go")
-	filebytes := []byte(`{"index":{"fields":["data.docType","data.owner"]},"name":"indexOwner","type":"json"}`)
-
-	err := writeToFile(filename, filebytes)
-	assert.NoError(t, err, "Error writing to file")
+	fileName := "myIndex.go"
+	fileBytes := []byte(`{"index":{"fields":["data.docType","data.owner"]},"name":"indexOwner","type":"json"}`)
+	metadataType := "META-INF/statedb/couchdb/indexes"
 
-	err = ValidateMetadataFile(filename, "META-INF/statedb/couchdb/indexes")
+	err := ValidateMetadataFile(fileName, fileBytes, metadataType)
 	assert.Error(t, err, "Should have received an BadExtensionError")
 
 	// Type assertion on BadExtensionError

core/container/util/writer.go

@@ -11,13 +11,14 @@ import (
 	"bufio"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
 	"time"
 
 	"github.com/hyperledger/fabric/common/flogging"
-	ccmetadata "github.com/hyperledger/fabric/core/common/ccprovider/metadata"
+	"github.com/hyperledger/fabric/core/common/ccprovider/metadata"
 	"github.com/pkg/errors"
 )
 
@@ -49,10 +50,10 @@ func WriteFolderToTarPackage(tw *tar.Writer, srcPath string, excludeDir string,
 	}
 
 	rootDirLen := len(rootDirectory)
-	walkFn := func(path string, info os.FileInfo, err error) error {
+	walkFn := func(localpath string, info os.FileInfo, err error) error {
 
-		// If path includes .git, ignore
-		if strings.Contains(path, ".git") {
+		// If localpath includes .git, ignore
+		if strings.Contains(localpath, ".git") {
 			return nil
 		}
 
@@ -61,15 +62,15 @@ func WriteFolderToTarPackage(tw *tar.Writer, srcPath string, excludeDir string,
 		}
 
 		//exclude any files with excludeDir prefix. They should already be in the tar
-		if excludeDir != "" && strings.Index(path, excludeDir) == rootDirLen+1 {
+		if excludeDir != "" && strings.Index(localpath, excludeDir) == rootDirLen+1 {
 			//1 for "/"
 			return nil
 		}
 		// Because of scoping we can reference the external rootDirectory variable
-		if len(path[rootDirLen:]) == 0 {
+		if len(localpath[rootDirLen:]) == 0 {
 			return nil
 		}
-		ext := filepath.Ext(path)
+		ext := filepath.Ext(localpath)
 
 		if includeFileTypeMap != nil {
 			// we only want 'fileTypes' source files at this point
@@ -85,35 +86,41 @@ func WriteFolderToTarPackage(tw *tar.Writer, srcPath string, excludeDir string,
 			}
 		}
 
-		var newPath string
+		var packagepath string
 
 		// if file is metadata, keep the /META-INF directory, e.g: META-INF/statedb/couchdb/indexes/indexOwner.json
 		// otherwise file is source code, put it in /src dir, e.g: src/marbles_chaincode.js
-		if strings.HasPrefix(path, filepath.Join(rootDirectory, "META-INF")) {
-			newPath = path[rootDirLen+1:]
+		if strings.HasPrefix(localpath, filepath.Join(rootDirectory, "META-INF")) {
+			packagepath = localpath[rootDirLen+1:]
 
-			// Split the filename itself from its path
-			_, filename := filepath.Split(newPath)
+			// Split the tar packagepath into a tar package directory and filename
+			packageDir, filename := filepath.Split(packagepath)
 
 			// Hidden files are not supported as metadata, therefore ignore them.
 			// User often doesn't know that hidden files are there, and may not be able to delete them, therefore warn user rather than error out.
 			if strings.HasPrefix(filename, ".") {
-				vmLogger.Warningf("Ignoring hidden file in metadata directory: %s", newPath)
+				vmLogger.Warningf("Ignoring hidden file in metadata directory: %s", packagepath)
 				return nil
 			}
 
+			fileBytes, err := ioutil.ReadFile(localpath)
+			if err != nil {
+				return err
+			}
+
 			// Validate metadata file for inclusion in tar
 			// Validation is based on the passed metadata directory, e.g. META-INF/statedb/couchdb/indexes
-			err = ccmetadata.ValidateMetadataFile(path, filepath.Dir(newPath))
+			// Clean metadata directory to remove trailing slash
+			err = metadata.ValidateMetadataFile(filename, fileBytes, filepath.Clean(packageDir))
 			if err != nil {
 				return err
 			}
 
 		} else { // file is not metadata, include in src
-			newPath = fmt.Sprintf("src%s", path[rootDirLen:])
+			packagepath = fmt.Sprintf("src%s", localpath[rootDirLen:])
 		}
 
-		err = WriteFileToPackage(path, newPath, tw)
+		err = WriteFileToPackage(localpath, packagepath, tw)
 		if err != nil {
 			return fmt.Errorf("Error writing file to package: %s", err)
 		}