Skip to content

Commit

Permalink
Merge pull request #2 from ribasushi/feat/implement_split_codec_mh
Browse files Browse the repository at this point in the history 
Implement final FilMultihash/FilCodec
  • Loading branch information
@whyrusleeping
whyrusleeping authored Jul 10, 2020
2 parents 05624bd + 226ec33 commit 96d863c
Show file tree
Hide file tree
Showing 4 changed files with 102 additions and 144 deletions.
188 changes: 71 additions & 117 deletions commcid.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,180 +10,134 @@ import (
"golang.org/x/xerrors"
)

// FilecoinMultihashCode is a multicodec index that identifiesh a multihash
// type for Filecoin
type FilecoinMultihashCode uint64
type FilMultiCodec uint64
type FilMultiHash uint64

const (
// FC_UNSEALED_V1 is the v1 hashing algorithm used in
// constructing merkleproofs of unsealed data
FC_UNSEALED_V1 FilecoinMultihashCode = 0xfc1 + iota

// FC_SEALED_V1 is the v1 hashing algorithm used in
// constructing merkleproofs of sealed replicated data
FC_SEALED_V1

// FC_RESERVED3 is reserved for future use
FC_RESERVED3

// FC_RESERVED4 is reserved for future use
FC_RESERVED4

// FC_RESERVED5 is reserved for future use
FC_RESERVED5

// FC_RESERVED6 is reserved for future use
FC_RESERVED6

// FC_RESERVED7 is reserved for future use
FC_RESERVED7

// FC_RESERVED8 is reserved for future use
FC_RESERVED8

// FC_RESERVED9 is reserved for future use
FC_RESERVED9

// FC_RESERVED10 is reserved for future use
FC_RESERVED10
)

// FilecoinMultihashNames maps filecoin multihash codes to a text descriptions
var FilecoinMultihashNames = map[FilecoinMultihashCode]string{
FC_UNSEALED_V1: "Filecoin Merkleproof Of Unsealed Data, V1",
FC_SEALED_V1: "Filecoin Merkleproof Of Sealed Data, V1",
FC_RESERVED3: "Reserved",
FC_RESERVED4: "Reserved",
FC_RESERVED5: "Reserved",
FC_RESERVED6: "Reserved",
FC_RESERVED7: "Reserved",
FC_RESERVED8: "Reserved",
FC_RESERVED9: "Reserved",
FC_RESERVED10: "Reserved",
}

// FC_UNDEFINED is just a signifier for no hash type determined
const FC_UNDEFINED = FilecoinMultihashCode(0)

// FilecoinCodecType is the serialization type for a Commitment CID
// = always just raw for now
const FilecoinCodecType = cid.Raw
// FC_UNDEFINED is just a signifier for no codec determined
const FC_UNDEFINED = FilMultiCodec(0)

var (
// ErrIncorrectCodec means the codec for a CID is a block format that does not match
// a commitment hash
ErrIncorrectCodec = errors.New("codec for all commitments is raw")
ErrIncorrectCodec = errors.New("unexpected commitment codec")
// ErrIncorrectHash means the hash function for this CID does not match the expected
// hash for this type of commitment
ErrIncorrectHash = errors.New("incorrect hashing function for data commitment")
)

// CommitmentToCID converts a raw commitment hash to a CID
// by adding:
// - serialization type of raw
// - the given filecoin codec type
// - the given filecoin hash type
func CommitmentToCID(commitment []byte, code FilecoinMultihashCode) (cid.Cid, error) {
if len(commitment) != 32 {
return cid.Undef, fmt.Errorf("commitments must be 32 bytes long")
func CommitmentToCID(commitment []byte, mc FilMultiCodec, mh FilMultiHash) (cid.Cid, error) {
if err := ValidateFilecoinCidSegments(mc, mh, commitment); err != nil {
return cid.Undef, err
}

if !ValidFilecoinMultihash(code) {
return cid.Undef, ErrIncorrectHash
}
mh := rawMultiHash(uint64(code), commitment)
return cid.NewCidV1(FilecoinCodecType, mh), nil
mhBuf := make(
[]byte,
(varint.UvarintSize(uint64(mh)) + varint.UvarintSize(uint64(len(commitment))) + len(commitment)),
)

pos := varint.PutUvarint(mhBuf, uint64(mh))
pos += varint.PutUvarint(mhBuf[pos:], uint64(len(commitment)))
copy(mhBuf[pos:], commitment)

return cid.NewCidV1(uint64(mc), multihash.Multihash(mhBuf)), nil
}

// CIDToCommitment extracts the raw data commitment from a CID
// assuming that it has the correct hashing function and
// serialization types
func CIDToCommitment(c cid.Cid) ([]byte, FilecoinMultihashCode, error) {
if c.Type() != FilecoinCodecType {
return nil, FC_UNDEFINED, ErrIncorrectCodec
}
mh := c.Hash()
decoded, err := multihash.Decode([]byte(mh))
// after validating that the codec and hash type are consistent
func CIDToCommitment(c cid.Cid) ([]byte, FilMultiCodec, error) {
decoded, err := multihash.Decode([]byte(c.Hash()))
if err != nil {
return nil, FC_UNDEFINED, xerrors.Errorf("Error decoding data commitment hash: %w", err)
}
code := FilecoinMultihashCode(decoded.Code)
if !ValidFilecoinMultihash(code) {
return nil, FC_UNDEFINED, ErrIncorrectHash

if err := ValidateFilecoinCidSegments(
FilMultiCodec(c.Type()),
FilMultiHash(decoded.Code),
decoded.Digest,
); err != nil {
return nil, FC_UNDEFINED, err
}
return decoded.Digest, code, nil

return decoded.Digest, FilMultiCodec(c.Type()), nil
}

// DataCommitmentV1ToCID converts a raw data commitment to a CID
// by adding:
// - serialization type of raw
// - hashing type of Filecoin unsealed hashing function v1 (0xfc2)
// - codec: cid.FilCommitmentUnsealed
// - hash type: multihash.SHA2_256_TRUNC254_PADDED
func DataCommitmentV1ToCID(commD []byte) (cid.Cid, error) {
return CommitmentToCID(commD, FC_UNSEALED_V1)
return CommitmentToCID(commD, cid.FilCommitmentUnsealed, multihash.SHA2_256_TRUNC254_PADDED)
}

// CIDToDataCommitmentV1 extracts the raw data commitment from a CID
// assuming that it has the correct hashing function and
// serialization types
// after checking for the correct codec and hash types.
func CIDToDataCommitmentV1(c cid.Cid) ([]byte, error) {
commD, hash, err := CIDToCommitment(c)
commD, codec, err := CIDToCommitment(c)
if err != nil {
return nil, err
}
if hash != FC_UNSEALED_V1 {
return nil, ErrIncorrectHash
if codec != cid.FilCommitmentUnsealed {
return nil, ErrIncorrectCodec
}
return commD, nil
}

// ReplicaCommitmentV1ToCID converts a raw data commitment to a CID
// by adding:
// - serialization type of raw
// - hashing type of Filecoin sealed hashing function v1 (0xfc2)
// - codec: cid.FilCommitmentSealed
// - hash type: multihash.POSEIDON_BLS12_381_A1_FC1
func ReplicaCommitmentV1ToCID(commR []byte) cid.Cid {
c, _ := CommitmentToCID(commR, FC_SEALED_V1)
c, _ := CommitmentToCID(commR, cid.FilCommitmentSealed, multihash.POSEIDON_BLS12_381_A1_FC1)
return c
}

// CIDToReplicaCommitmentV1 extracts the raw replica commitment from a CID
// assuming that it has the correct hashing function and
// serialization types
// after checking for the correct codec and hash types.
func CIDToReplicaCommitmentV1(c cid.Cid) ([]byte, error) {
commR, hash, err := CIDToCommitment(c)
commR, codec, err := CIDToCommitment(c)
if err != nil {
return nil, err
}
if hash != FC_SEALED_V1 {
return nil, ErrIncorrectHash
if codec != cid.FilCommitmentSealed {
return nil, ErrIncorrectCodec
}
return commR, nil
}

// ValidateFilecoinCidSegments returns an error if the provided CID parts
// conflict with each other.
func ValidateFilecoinCidSegments(mc FilMultiCodec, mh FilMultiHash, commitment []byte) error {

switch mc {
case cid.FilCommitmentUnsealed:
if mh != multihash.SHA2_256_TRUNC254_PADDED {
return ErrIncorrectHash
}
case cid.FilCommitmentSealed:
if mh != multihash.POSEIDON_BLS12_381_A1_FC1 {
return ErrIncorrectHash
}
default: // neigher of the codecs above: we are not in Fil teritory
return ErrIncorrectCodec
}

if len(commitment) != 32 {
return fmt.Errorf("commitments must be 32 bytes long")
}

return nil
}

// PieceCommitmentV1ToCID converts a commP to a CID
// -- it is just a helper function that is equivalent to
// DataCommitmentV1ToCID.
func PieceCommitmentV1ToCID(commP []byte) (cid.Cid, error) {
return DataCommitmentV1ToCID(commP)
}
var PieceCommitmentV1ToCID = DataCommitmentV1ToCID

// CIDToPieceCommitmentV1 converts a CID to a commP
// -- it is just a helper function that is equivalent to
// CIDToDataCommitmentV1.
func CIDToPieceCommitmentV1(c cid.Cid) ([]byte, error) {
return CIDToDataCommitmentV1(c)
}

func rawMultiHash(code uint64, buf []byte) multihash.Multihash {
newBuf := make([]byte, varint.UvarintSize(code)+varint.UvarintSize(uint64(len(buf)))+len(buf))
n := varint.PutUvarint(newBuf, code)
n += varint.PutUvarint(newBuf[n:], uint64(len(buf)))

copy(newBuf[n:], buf)
return multihash.Multihash(newBuf)
}

// ValidFilecoinMultihash returns true if the given multihash type
// is recognized as belonging to filecoin
func ValidFilecoinMultihash(code FilecoinMultihashCode) bool {
_, ok := FilecoinMultihashNames[code]
return ok
}
var CIDToPieceCommitmentV1 = CIDToDataCommitmentV1
42 changes: 21 additions & 21 deletions commcid_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,11 @@ func TestDataCommitmentToCID(t *testing.T) {
c, err := commcid.DataCommitmentV1ToCID(randBytes)
require.NoError(t, err)

require.Equal(t, c.Prefix().Codec, uint64(cid.Raw))
require.Equal(t, c.Prefix().Codec, uint64(cid.FilCommitmentUnsealed))
mh := c.Hash()
decoded, err := multihash.Decode([]byte(mh))
require.NoError(t, err)
require.Equal(t, decoded.Code, uint64(commcid.FC_UNSEALED_V1))
require.Equal(t, decoded.Code, uint64(multihash.SHA2_256_TRUNC254_PADDED))
require.Equal(t, decoded.Length, len(randBytes))
require.True(t, bytes.Equal(decoded.Digest, randBytes))
}
Expand All @@ -35,10 +35,10 @@ func TestCIDToDataCommitment(t *testing.T) {
require.NoError(t, err)

t.Run("with correct hash format", func(t *testing.T) {
hash := testMultiHash(uint64(commcid.FC_UNSEALED_V1), randBytes, 0)
hash := testMultiHash(multihash.SHA2_256_TRUNC254_PADDED, randBytes, 0)

t.Run("decodes raw commitment hash when correct cid format", func(t *testing.T) {
c := cid.NewCidV1(cid.Raw, hash)
c := cid.NewCidV1(cid.FilCommitmentUnsealed, hash)
decoded, err := commcid.CIDToDataCommitmentV1(c)
require.NoError(t, err)
require.True(t, bytes.Equal(decoded, randBytes))
Expand All @@ -53,8 +53,8 @@ func TestCIDToDataCommitment(t *testing.T) {
})

t.Run("error on incorrectly formatted hash", func(t *testing.T) {
hash := testMultiHash(uint64(commcid.FC_UNSEALED_V1), randBytes, 5)
c := cid.NewCidV1(cid.Raw, hash)
hash := testMultiHash(multihash.SHA2_256_TRUNC254_PADDED, randBytes, 5)
c := cid.NewCidV1(cid.FilCommitmentUnsealed, hash)
decoded, err := commcid.CIDToDataCommitmentV1(c)
require.Error(t, err)
require.Regexp(t, "^Error decoding data commitment hash:", err.Error())
Expand All @@ -63,7 +63,7 @@ func TestCIDToDataCommitment(t *testing.T) {
t.Run("error on wrong hash type", func(t *testing.T) {
encoded, err := multihash.Encode(randBytes, multihash.SHA2_256)
require.NoError(t, err)
c := cid.NewCidV1(cid.Raw, multihash.Multihash(encoded))
c := cid.NewCidV1(cid.FilCommitmentUnsealed, multihash.Multihash(encoded))
decoded, err := commcid.CIDToDataCommitmentV1(c)
require.EqualError(t, err, commcid.ErrIncorrectHash.Error())
require.Nil(t, decoded)
Expand All @@ -77,11 +77,11 @@ func TestReplicaCommitmentToCID(t *testing.T) {

c := commcid.ReplicaCommitmentV1ToCID(randBytes)

require.Equal(t, c.Prefix().Codec, uint64(cid.Raw))
require.Equal(t, c.Prefix().Codec, uint64(cid.FilCommitmentSealed))
mh := c.Hash()
decoded, err := multihash.Decode([]byte(mh))
require.NoError(t, err)
require.Equal(t, decoded.Code, uint64(uint64(commcid.FC_SEALED_V1)))
require.Equal(t, decoded.Code, uint64(multihash.POSEIDON_BLS12_381_A1_FC1))
require.Equal(t, decoded.Length, len(randBytes))
require.True(t, bytes.Equal(decoded.Digest, randBytes))
}
Expand All @@ -92,10 +92,10 @@ func TestCIDToReplicaCommitment(t *testing.T) {
require.NoError(t, err)

t.Run("with correct hash format", func(t *testing.T) {
hash := testMultiHash(uint64(commcid.FC_SEALED_V1), randBytes, 0)
hash := testMultiHash(multihash.POSEIDON_BLS12_381_A1_FC1, randBytes, 0)

t.Run("decodes raw commitment hash when correct cid format", func(t *testing.T) {
c := cid.NewCidV1(cid.Raw, hash)
c := cid.NewCidV1(cid.FilCommitmentSealed, hash)
decoded, err := commcid.CIDToReplicaCommitmentV1(c)
require.NoError(t, err)
require.True(t, bytes.Equal(decoded, randBytes))
Expand All @@ -110,19 +110,19 @@ func TestCIDToReplicaCommitment(t *testing.T) {
})

t.Run("error on incorrectly formatted hash", func(t *testing.T) {
hash := testMultiHash(uint64(commcid.FC_SEALED_V1), randBytes, 5)
hash := testMultiHash(cid.FilCommitmentSealed, randBytes, 0)
c := cid.NewCidV1(cid.Raw, hash)
decoded, err := commcid.CIDToReplicaCommitmentV1(c)
require.Error(t, err)
require.Regexp(t, "^Error decoding data commitment hash:", err.Error())
require.Regexp(t, "^unexpected commitment codec", err.Error())
require.Nil(t, decoded)
})
t.Run("error on wrong hash type", func(t *testing.T) {
encoded, err := multihash.Encode(randBytes, multihash.SHA2_256)
require.NoError(t, err)
c := cid.NewCidV1(cid.Raw, multihash.Multihash(encoded))
decoded, err := commcid.CIDToReplicaCommitmentV1(c)
require.EqualError(t, err, commcid.ErrIncorrectHash.Error())
require.EqualError(t, err, commcid.ErrIncorrectCodec.Error())
require.Nil(t, decoded)
})
}
Expand All @@ -135,11 +135,11 @@ func TestPieceCommitmentToCID(t *testing.T) {
c, err := commcid.PieceCommitmentV1ToCID(randBytes)
require.NoError(t, err)

require.Equal(t, c.Prefix().Codec, uint64(cid.Raw))
require.Equal(t, c.Prefix().Codec, uint64(cid.FilCommitmentUnsealed))
mh := c.Hash()
decoded, err := multihash.Decode([]byte(mh))
require.NoError(t, err)
require.Equal(t, decoded.Code, uint64(commcid.FC_UNSEALED_V1))
require.Equal(t, decoded.Code, uint64(multihash.SHA2_256_TRUNC254_PADDED))
require.Equal(t, decoded.Length, len(randBytes))
require.True(t, bytes.Equal(decoded.Digest, randBytes))
}
Expand All @@ -150,10 +150,10 @@ func TestCIDToPieceCommitment(t *testing.T) {
require.NoError(t, err)

t.Run("with correct hash format", func(t *testing.T) {
hash := testMultiHash(uint64(commcid.FC_UNSEALED_V1), randBytes, 0)
hash := testMultiHash(multihash.SHA2_256_TRUNC254_PADDED, randBytes, 0)

t.Run("decodes raw commitment hash when correct cid format", func(t *testing.T) {
c := cid.NewCidV1(cid.Raw, hash)
c := cid.NewCidV1(cid.FilCommitmentUnsealed, hash)
decoded, err := commcid.CIDToPieceCommitmentV1(c)
require.NoError(t, err)
require.True(t, bytes.Equal(decoded, randBytes))
Expand All @@ -168,8 +168,8 @@ func TestCIDToPieceCommitment(t *testing.T) {
})

t.Run("error on incorrectly formatted hash", func(t *testing.T) {
hash := testMultiHash(uint64(commcid.FC_UNSEALED_V1), randBytes, 5)
c := cid.NewCidV1(cid.Raw, hash)
hash := testMultiHash(multihash.SHA2_256_TRUNC254_PADDED, randBytes, 5)
c := cid.NewCidV1(cid.FilCommitmentUnsealed, hash)
decoded, err := commcid.CIDToPieceCommitmentV1(c)
require.Error(t, err)
require.Regexp(t, "^Error decoding data commitment hash:", err.Error())
Expand All @@ -178,7 +178,7 @@ func TestCIDToPieceCommitment(t *testing.T) {
t.Run("error on wrong hash type", func(t *testing.T) {
encoded, err := multihash.Encode(randBytes, multihash.SHA2_256)
require.NoError(t, err)
c := cid.NewCidV1(cid.Raw, multihash.Multihash(encoded))
c := cid.NewCidV1(cid.FilCommitmentUnsealed, multihash.Multihash(encoded))
decoded, err := commcid.CIDToPieceCommitmentV1(c)
require.EqualError(t, err, commcid.ErrIncorrectHash.Error())
require.Nil(t, decoded)
Expand Down
Loading

0 comments on commit 96d863c

Please sign in to comment.