From 95920ea543ad9120b932e72671f48e01717b92a8 Mon Sep 17 00:00:00 2001 From: Maxim Vladimirskiy Date: Thu, 1 Jul 2021 13:35:32 +0300 Subject: [PATCH] Replace byte hash with string hash --- config.go | 12 ++++++------ region_picker.go | 4 ++-- replicated_hash.go | 16 ++++++++-------- replicated_hash_test.go | 20 ++++++++++---------- 4 files changed, 26 insertions(+), 26 deletions(-) diff --git a/config.go b/config.go index 85aa877f..14650ce1 100644 --- a/config.go +++ b/config.go @@ -116,7 +116,7 @@ func (c *Config) SetDefaults() error { setter.SetDefault(&c.Behaviors.MultiRegionBatchLimit, maxBatchSize) setter.SetDefault(&c.Behaviors.MultiRegionSyncWait, time.Second) - setter.SetDefault(&c.LocalPicker, NewReplicatedConsistentHash(nil, DefaultReplicas)) + setter.SetDefault(&c.LocalPicker, NewReplicatedConsistentHash(nil, defaultReplicas)) setter.SetDefault(&c.RegionPicker, NewRegionPicker(nil)) setter.SetDefault(&c.Cache, NewLRUCache(0)) @@ -335,12 +335,12 @@ func SetupDaemonConfig(logger *logrus.Logger, configFile string) (DaemonConfig, switch pp { case "replicated-hash": - setter.SetDefault(&replicas, getEnvInteger(log, "GUBER_REPLICATED_HASH_REPLICAS"), DefaultReplicas) + setter.SetDefault(&replicas, getEnvInteger(log, "GUBER_REPLICATED_HASH_REPLICAS"), defaultReplicas) conf.Picker = NewReplicatedConsistentHash(nil, replicas) setter.SetDefault(&hash, os.Getenv("GUBER_PEER_PICKER_HASH"), "fnv1a") - hashFuncs := map[string]HashFunc64{ - "fnv1a": fnv1a.HashBytes64, - "fnv1": fnv1.HashBytes64, + hashFuncs := map[string]HashString64{ + "fnv1a": fnv1a.HashString64, + "fnv1": fnv1.HashString64, } fn, ok := hashFuncs[hash] if !ok { @@ -528,7 +528,7 @@ func validClientAuthTypes(m map[string]tls.ClientAuthType) string { return strings.Join(rs, ",") } -func validHash64Keys(m map[string]HashFunc64) string { +func validHash64Keys(m map[string]HashString64) string { var rs []string for k, _ := range m { rs = append(rs, k) diff --git a/region_picker.go b/region_picker.go index 408636e9..ec025b9c 100644 --- a/region_picker.go +++ b/region_picker.go @@ -25,11 +25,11 @@ type RegionPicker struct { reqQueue chan *RateLimitReq } -func NewRegionPicker(fn HashFunc64) *RegionPicker { +func NewRegionPicker(fn HashString64) *RegionPicker { rp := &RegionPicker{ regions: make(map[string]PeerPicker), reqQueue: make(chan *RateLimitReq, 0), - ReplicatedConsistentHash: NewReplicatedConsistentHash(fn, DefaultReplicas), + ReplicatedConsistentHash: NewReplicatedConsistentHash(fn, defaultReplicas), } return rp } diff --git a/replicated_hash.go b/replicated_hash.go index 9c29b7a9..0aebebcd 100644 --- a/replicated_hash.go +++ b/replicated_hash.go @@ -28,15 +28,15 @@ import ( "github.com/segmentio/fasthash/fnv1" ) -const DefaultReplicas = 512 +const defaultReplicas = 512 -type HashFunc64 func(data []byte) uint64 +type HashString64 func(data string) uint64 -var DefaultHash64 HashFunc64 = fnv1.HashBytes64 +var defaultHashString64 HashString64 = fnv1.HashString64 // Implements PeerPicker type ReplicatedConsistentHash struct { - hashFunc HashFunc64 + hashFunc HashString64 peerKeys []peerInfo peers map[string]*PeerClient replicas int @@ -47,7 +47,7 @@ type peerInfo struct { peer *PeerClient } -func NewReplicatedConsistentHash(fn HashFunc64, replicas int) *ReplicatedConsistentHash { +func NewReplicatedConsistentHash(fn HashString64, replicas int) *ReplicatedConsistentHash { ch := &ReplicatedConsistentHash{ hashFunc: fn, peers: make(map[string]*PeerClient), @@ -55,7 +55,7 @@ func NewReplicatedConsistentHash(fn HashFunc64, replicas int) *ReplicatedConsist } if ch.hashFunc == nil { - ch.hashFunc = DefaultHash64 + ch.hashFunc = defaultHashString64 } return ch } @@ -82,7 +82,7 @@ func (ch *ReplicatedConsistentHash) Add(peer *PeerClient) { key := fmt.Sprintf("%x", md5.Sum([]byte(peer.Info().GRPCAddress))) for i := 0; i < ch.replicas; i++ { - hash := ch.hashFunc(strToBytesUnsafe(strconv.Itoa(i) + key)) + hash := ch.hashFunc(strconv.Itoa(i) + key) ch.peerKeys = append(ch.peerKeys, peerInfo{ hash: hash, peer: peer, @@ -107,7 +107,7 @@ func (ch *ReplicatedConsistentHash) Get(key string) (*PeerClient, error) { if ch.Size() == 0 { return nil, errors.New("unable to pick a peer; pool is empty") } - hash := ch.hashFunc(strToBytesUnsafe(key)) + hash := ch.hashFunc(key) // Binary search for appropriate peer idx := sort.Search(len(ch.peerKeys), func(i int) bool { return ch.peerKeys[i].hash >= hash }) diff --git a/replicated_hash_test.go b/replicated_hash_test.go index b8284b65..c87ad67e 100644 --- a/replicated_hash_test.go +++ b/replicated_hash_test.go @@ -13,7 +13,7 @@ func TestReplicatedConsistentHash(t *testing.T) { hosts := []string{"a.svc.local", "b.svc.local", "c.svc.local"} t.Run("Size", func(t *testing.T) { - hash := NewReplicatedConsistentHash(nil, DefaultReplicas) + hash := NewReplicatedConsistentHash(nil, defaultReplicas) for _, h := range hosts { hash.Add(&PeerClient{conf: PeerConfig{Info: PeerInfo{GRPCAddress: h}}}) @@ -23,7 +23,7 @@ func TestReplicatedConsistentHash(t *testing.T) { }) t.Run("Host", func(t *testing.T) { - hash := NewReplicatedConsistentHash(nil, DefaultReplicas) + hash := NewReplicatedConsistentHash(nil, defaultReplicas) hostMap := map[string]*PeerClient{} for _, h := range hosts { @@ -46,7 +46,7 @@ func TestReplicatedConsistentHash(t *testing.T) { for _, tc := range []struct { name string - inHashFunc HashFunc64 + inHashFunc HashString64 outDistribution map[string]int }{{ name: "default", @@ -55,19 +55,19 @@ func TestReplicatedConsistentHash(t *testing.T) { }, }, { name: "fasthash/fnv1a", - inHashFunc: fnv1a.HashBytes64, + inHashFunc: fnv1a.HashString64, outDistribution: map[string]int{ "a.svc.local": 3110, "b.svc.local": 3856, "c.svc.local": 3034, }, }, { name: "fasthash/fnv1", - inHashFunc: fnv1.HashBytes64, + inHashFunc: fnv1.HashString64, outDistribution: map[string]int{ "a.svc.local": 2948, "b.svc.local": 3592, "c.svc.local": 3460, }, }} { t.Run(tc.name, func(t *testing.T) { - hash := NewReplicatedConsistentHash(tc.inHashFunc, DefaultReplicas) + hash := NewReplicatedConsistentHash(tc.inHashFunc, defaultReplicas) distribution := make(map[string]int) for _, h := range hosts { @@ -87,9 +87,9 @@ func TestReplicatedConsistentHash(t *testing.T) { } func BenchmarkReplicatedConsistantHash(b *testing.B) { - hashFuncs := map[string]HashFunc64{ - "fasthash/fnv1a": fnv1a.HashBytes64, - "fasthash/fnv1": fnv1.HashBytes64, + hashFuncs := map[string]HashString64{ + "fasthash/fnv1a": fnv1a.HashString64, + "fasthash/fnv1": fnv1.HashString64, } for name, hashFunc := range hashFuncs { @@ -99,7 +99,7 @@ func BenchmarkReplicatedConsistantHash(b *testing.B) { ips[i] = net.IPv4(byte(i>>24), byte(i>>16), byte(i>>8), byte(i)).String() } - hash := NewReplicatedConsistentHash(hashFunc, DefaultReplicas) + hash := NewReplicatedConsistentHash(hashFunc, defaultReplicas) hosts := []string{"a.svc.local", "b.svc.local", "c.svc.local"} for _, h := range hosts { hash.Add(&PeerClient{conf: PeerConfig{Info: PeerInfo{GRPCAddress: h}}})