init: 1.0.0
This commit is contained in:
Chang lue Tsen
@@ -0,0 +1,186 @@
|
||||
package hash
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"sync"
|
||||
|
||||
"github.com/perfect-panel/ppanel-server/pkg/lang"
|
||||
)
|
||||
|
||||
const (
|
||||
// TopWeight is the top weight that one entry might set.
|
||||
TopWeight = 100
|
||||
|
||||
minReplicas = 100
|
||||
prime = 16777619
|
||||
)
|
||||
|
||||
type (
|
||||
// Func defines the hash method.
|
||||
Func func(data []byte) uint64
|
||||
|
||||
// A ConsistentHash is a ring hash implementation.
|
||||
ConsistentHash struct {
|
||||
hashFunc Func
|
||||
replicas int
|
||||
keys []uint64
|
||||
ring map[uint64][]any
|
||||
nodes map[string]lang.PlaceholderType
|
||||
lock sync.RWMutex
|
||||
}
|
||||
)
|
||||
|
||||
// NewConsistentHash returns a ConsistentHash.
|
||||
func NewConsistentHash() *ConsistentHash {
|
||||
return NewCustomConsistentHash(minReplicas, Hash)
|
||||
}
|
||||
|
||||
// NewCustomConsistentHash returns a ConsistentHash with given replicas and hash func.
|
||||
func NewCustomConsistentHash(replicas int, fn Func) *ConsistentHash {
|
||||
if replicas < minReplicas {
|
||||
replicas = minReplicas
|
||||
}
|
||||
|
||||
if fn == nil {
|
||||
fn = Hash
|
||||
}
|
||||
|
||||
return &ConsistentHash{
|
||||
hashFunc: fn,
|
||||
replicas: replicas,
|
||||
ring: make(map[uint64][]any),
|
||||
nodes: make(map[string]lang.PlaceholderType),
|
||||
}
|
||||
}
|
||||
|
||||
// Add adds the node with the number of h.replicas,
|
||||
// the later call will overwrite the replicas of the former calls.
|
||||
func (h *ConsistentHash) Add(node any) {
|
||||
h.AddWithReplicas(node, h.replicas)
|
||||
}
|
||||
|
||||
// AddWithReplicas adds the node with the number of replicas,
|
||||
// replicas will be truncated to h.replicas if it's larger than h.replicas,
|
||||
// the later call will overwrite the replicas of the former calls.
|
||||
func (h *ConsistentHash) AddWithReplicas(node any, replicas int) {
|
||||
h.Remove(node)
|
||||
|
||||
if replicas > h.replicas {
|
||||
replicas = h.replicas
|
||||
}
|
||||
|
||||
nodeRepr := repr(node)
|
||||
h.lock.Lock()
|
||||
defer h.lock.Unlock()
|
||||
h.addNode(nodeRepr)
|
||||
|
||||
for i := 0; i < replicas; i++ {
|
||||
hash := h.hashFunc([]byte(nodeRepr + strconv.Itoa(i)))
|
||||
h.keys = append(h.keys, hash)
|
||||
h.ring[hash] = append(h.ring[hash], node)
|
||||
}
|
||||
|
||||
sort.Slice(h.keys, func(i, j int) bool {
|
||||
return h.keys[i] < h.keys[j]
|
||||
})
|
||||
}
|
||||
|
||||
// AddWithWeight adds the node with weight, the weight can be 1 to 100, indicates the percent,
|
||||
// the later call will overwrite the replicas of the former calls.
|
||||
func (h *ConsistentHash) AddWithWeight(node any, weight int) {
|
||||
// don't need to make sure weight not larger than TopWeight,
|
||||
// because AddWithReplicas makes sure replicas cannot be larger than h.replicas
|
||||
replicas := h.replicas * weight / TopWeight
|
||||
h.AddWithReplicas(node, replicas)
|
||||
}
|
||||
|
||||
// Get returns the corresponding node from h base on the given v.
|
||||
func (h *ConsistentHash) Get(v any) (any, bool) {
|
||||
h.lock.RLock()
|
||||
defer h.lock.RUnlock()
|
||||
|
||||
if len(h.ring) == 0 {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
hash := h.hashFunc([]byte(repr(v)))
|
||||
index := sort.Search(len(h.keys), func(i int) bool {
|
||||
return h.keys[i] >= hash
|
||||
}) % len(h.keys)
|
||||
|
||||
nodes := h.ring[h.keys[index]]
|
||||
switch len(nodes) {
|
||||
case 0:
|
||||
return nil, false
|
||||
case 1:
|
||||
return nodes[0], true
|
||||
default:
|
||||
innerIndex := h.hashFunc([]byte(innerRepr(v)))
|
||||
pos := int(innerIndex % uint64(len(nodes)))
|
||||
return nodes[pos], true
|
||||
}
|
||||
}
|
||||
|
||||
// Remove removes the given node from h.
|
||||
func (h *ConsistentHash) Remove(node any) {
|
||||
nodeRepr := repr(node)
|
||||
|
||||
h.lock.Lock()
|
||||
defer h.lock.Unlock()
|
||||
|
||||
if !h.containsNode(nodeRepr) {
|
||||
return
|
||||
}
|
||||
|
||||
for i := 0; i < h.replicas; i++ {
|
||||
hash := h.hashFunc([]byte(nodeRepr + strconv.Itoa(i)))
|
||||
index := sort.Search(len(h.keys), func(i int) bool {
|
||||
return h.keys[i] >= hash
|
||||
})
|
||||
if index < len(h.keys) && h.keys[index] == hash {
|
||||
h.keys = append(h.keys[:index], h.keys[index+1:]...)
|
||||
}
|
||||
h.removeRingNode(hash, nodeRepr)
|
||||
}
|
||||
|
||||
h.removeNode(nodeRepr)
|
||||
}
|
||||
|
||||
func (h *ConsistentHash) removeRingNode(hash uint64, nodeRepr string) {
|
||||
if nodes, ok := h.ring[hash]; ok {
|
||||
newNodes := nodes[:0]
|
||||
for _, x := range nodes {
|
||||
if repr(x) != nodeRepr {
|
||||
newNodes = append(newNodes, x)
|
||||
}
|
||||
}
|
||||
if len(newNodes) > 0 {
|
||||
h.ring[hash] = newNodes
|
||||
} else {
|
||||
delete(h.ring, hash)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (h *ConsistentHash) addNode(nodeRepr string) {
|
||||
h.nodes[nodeRepr] = lang.Placeholder
|
||||
}
|
||||
|
||||
func (h *ConsistentHash) containsNode(nodeRepr string) bool {
|
||||
_, ok := h.nodes[nodeRepr]
|
||||
return ok
|
||||
}
|
||||
|
||||
func (h *ConsistentHash) removeNode(nodeRepr string) {
|
||||
delete(h.nodes, nodeRepr)
|
||||
}
|
||||
|
||||
func innerRepr(node any) string {
|
||||
return fmt.Sprintf("%d:%v", prime, node)
|
||||
}
|
||||
|
||||
func repr(node any) string {
|
||||
return lang.Repr(node)
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
package hash
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
const (
|
||||
keySize = 20
|
||||
requestSize = 1000
|
||||
)
|
||||
|
||||
func BenchmarkConsistentHashGet(b *testing.B) {
|
||||
ch := NewConsistentHash()
|
||||
for i := 0; i < keySize; i++ {
|
||||
ch.Add("localhost:" + strconv.Itoa(i))
|
||||
}
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
ch.Get(i)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConsistentHashIncrementalTransfer(t *testing.T) {
|
||||
prefix := "anything"
|
||||
create := func() *ConsistentHash {
|
||||
ch := NewConsistentHash()
|
||||
for i := 0; i < keySize; i++ {
|
||||
ch.Add(prefix + strconv.Itoa(i))
|
||||
}
|
||||
return ch
|
||||
}
|
||||
|
||||
originCh := create()
|
||||
keys := make(map[int]string, requestSize)
|
||||
for i := 0; i < requestSize; i++ {
|
||||
key, ok := originCh.Get(requestSize + i)
|
||||
assert.True(t, ok)
|
||||
assert.NotNil(t, key)
|
||||
keys[i] = key.(string)
|
||||
}
|
||||
|
||||
node := fmt.Sprintf("%s%d", prefix, keySize)
|
||||
for i := 0; i < 10; i++ {
|
||||
laterCh := create()
|
||||
laterCh.AddWithWeight(node, 10*(i+1))
|
||||
|
||||
for j := 0; j < requestSize; j++ {
|
||||
key, ok := laterCh.Get(requestSize + j)
|
||||
assert.True(t, ok)
|
||||
assert.NotNil(t, key)
|
||||
value := key.(string)
|
||||
assert.True(t, value == keys[j] || value == node)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestConsistentHashTransferOnFailure(t *testing.T) {
|
||||
index := 41
|
||||
keys, newKeys := getKeysBeforeAndAfterFailure(t, "localhost:", index)
|
||||
var transferred int
|
||||
for k, v := range newKeys {
|
||||
if v != keys[k] {
|
||||
transferred++
|
||||
}
|
||||
}
|
||||
|
||||
ratio := float32(transferred) / float32(requestSize)
|
||||
assert.True(t, ratio < 2.5/float32(keySize), fmt.Sprintf("%d: %f", index, ratio))
|
||||
}
|
||||
|
||||
func TestConsistentHashLeastTransferOnFailure(t *testing.T) {
|
||||
prefix := "localhost:"
|
||||
index := 41
|
||||
keys, newKeys := getKeysBeforeAndAfterFailure(t, prefix, index)
|
||||
for k, v := range keys {
|
||||
newV := newKeys[k]
|
||||
if v != prefix+strconv.Itoa(index) {
|
||||
assert.Equal(t, v, newV)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestConsistentHash_Remove(t *testing.T) {
|
||||
ch := NewConsistentHash()
|
||||
ch.Add("first")
|
||||
ch.Add("second")
|
||||
ch.Remove("first")
|
||||
for i := 0; i < 100; i++ {
|
||||
val, ok := ch.Get(i)
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, "second", val)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConsistentHash_RemoveInterface(t *testing.T) {
|
||||
const key = "any"
|
||||
ch := NewConsistentHash()
|
||||
node1 := newMockNode(key, 1)
|
||||
node2 := newMockNode(key, 2)
|
||||
ch.AddWithWeight(node1, 80)
|
||||
ch.AddWithWeight(node2, 50)
|
||||
assert.Equal(t, 1, len(ch.nodes))
|
||||
node, ok := ch.Get(1)
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, key, node.(*mockNode).addr)
|
||||
assert.Equal(t, 2, node.(*mockNode).id)
|
||||
}
|
||||
|
||||
func getKeysBeforeAndAfterFailure(t *testing.T, prefix string, index int) (map[int]string, map[int]string) {
|
||||
ch := NewConsistentHash()
|
||||
for i := 0; i < keySize; i++ {
|
||||
ch.Add(prefix + strconv.Itoa(i))
|
||||
}
|
||||
|
||||
keys := make(map[int]string, requestSize)
|
||||
for i := 0; i < requestSize; i++ {
|
||||
key, ok := ch.Get(requestSize + i)
|
||||
assert.True(t, ok)
|
||||
assert.NotNil(t, key)
|
||||
keys[i] = key.(string)
|
||||
}
|
||||
|
||||
remove := fmt.Sprintf("%s%d", prefix, index)
|
||||
ch.Remove(remove)
|
||||
newKeys := make(map[int]string, requestSize)
|
||||
for i := 0; i < requestSize; i++ {
|
||||
key, ok := ch.Get(requestSize + i)
|
||||
assert.True(t, ok)
|
||||
assert.NotNil(t, key)
|
||||
assert.NotEqual(t, remove, key)
|
||||
newKeys[i] = key.(string)
|
||||
}
|
||||
|
||||
return keys, newKeys
|
||||
}
|
||||
|
||||
type mockNode struct {
|
||||
addr string
|
||||
id int
|
||||
}
|
||||
|
||||
func newMockNode(addr string, id int) *mockNode {
|
||||
return &mockNode{
|
||||
addr: addr,
|
||||
id: id,
|
||||
}
|
||||
}
|
||||
|
||||
func (n *mockNode) String() string {
|
||||
return n.addr
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package hash
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"fmt"
|
||||
|
||||
"github.com/spaolacci/murmur3"
|
||||
)
|
||||
|
||||
// Hash returns the hash value of data.
|
||||
func Hash(data []byte) uint64 {
|
||||
return murmur3.Sum64(data)
|
||||
}
|
||||
|
||||
// Md5 returns the md5 bytes of data.
|
||||
func Md5(data []byte) []byte {
|
||||
digest := md5.New()
|
||||
digest.Write(data)
|
||||
return digest.Sum(nil)
|
||||
}
|
||||
|
||||
// Md5Hex returns the md5 hex string of data.
|
||||
func Md5Hex(data []byte) string {
|
||||
return fmt.Sprintf("%x", Md5(data))
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
package hash
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
"math/big"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
const (
|
||||
text = "hello, world!\n"
|
||||
md5Digest = "910c8bc73110b0cd1bc5d2bcae782511"
|
||||
)
|
||||
|
||||
func TestMd5(t *testing.T) {
|
||||
actual := fmt.Sprintf("%x", Md5([]byte(text)))
|
||||
assert.Equal(t, md5Digest, actual)
|
||||
}
|
||||
|
||||
func TestMd5Hex(t *testing.T) {
|
||||
actual := Md5Hex([]byte(text))
|
||||
assert.Equal(t, md5Digest, actual)
|
||||
}
|
||||
|
||||
func BenchmarkHashFnv(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
h := fnv.New32()
|
||||
new(big.Int).SetBytes(h.Sum([]byte(text))).Int64()
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkHashMd5(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
h := md5.New()
|
||||
bytes := h.Sum([]byte(text))
|
||||
new(big.Int).SetBytes(bytes).Int64()
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkMurmur3(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
Hash([]byte(text))
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user