|
@ -1,4 +1,5 @@ |
|
|
require "./ring.cr" |
|
|
require "./ring.cr" |
|
|
|
|
|
require "json" |
|
|
require "mutex" |
|
|
require "mutex" |
|
|
require "openssl" |
|
|
require "openssl" |
|
|
|
|
|
|
|
@ -12,6 +13,18 @@ module Crystal::Scatter |
|
|
end |
|
|
end |
|
|
end |
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
class SliceInfo |
|
|
|
|
|
JSON.mapping( |
|
|
|
|
|
slices: Array(Array(Slice)), |
|
|
|
|
|
last: Int32, |
|
|
|
|
|
shards: UInt32 |
|
|
|
|
|
) |
|
|
|
|
|
|
|
|
|
|
|
def initialize(@slices, @last, @shards) |
|
|
|
|
|
|
|
|
|
|
|
end |
|
|
|
|
|
end |
|
|
|
|
|
|
|
|
class MetaRing |
|
|
class MetaRing |
|
|
getter rings : Array(Ring) |
|
|
getter rings : Array(Ring) |
|
|
getter ring_graph : RingGraph |
|
|
getter ring_graph : RingGraph |
|
@ -44,12 +57,31 @@ module Crystal::Scatter |
|
|
return tag |
|
|
return tag |
|
|
end |
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
private def hash_and_split(data) |
|
|
|
|
|
h = hash_impl data |
|
|
|
|
|
space=UInt64::MAX/@shards |
|
|
|
|
|
targets = Array(UInt64).new |
|
|
|
|
|
(1..@shards).each do |
|
|
|
|
|
targets << h |
|
|
|
|
|
h+=space |
|
|
|
|
|
end |
|
|
|
|
|
return targets |
|
|
|
|
|
end |
|
|
|
|
|
|
|
|
private def get_slice_from_hash(ring : Ring, h : UInt64) |
|
|
private def get_slice_from_hash(ring : Ring, h : UInt64) |
|
|
value = ring.find do |slice| |
|
|
value = ring.find do |slice| |
|
|
h>=slice.s_begin && slice.s_end>=h |
|
|
h>=slice.s_begin && slice.s_end>=h |
|
|
end |
|
|
end |
|
|
return value.not_nil! |
|
|
return value.not_nil! |
|
|
end |
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
private def unfold_ring(ring : Ring, targets : Array(UInt64)) |
|
|
|
|
|
slices = Array(Slice).new |
|
|
|
|
|
targets.each do |h| |
|
|
|
|
|
slices << get_slice_from_hash(ring, h) |
|
|
|
|
|
end |
|
|
|
|
|
return slices |
|
|
|
|
|
end |
|
|
|
|
|
|
|
|
def add(element : Daemon) |
|
|
def add(element : Daemon) |
|
|
@lock.synchronize do |
|
|
@lock.synchronize do |
|
@ -78,44 +110,25 @@ module Crystal::Scatter |
|
|
ring? = @rings.last |
|
|
ring? = @rings.last |
|
|
end |
|
|
end |
|
|
ring = ring?.not_nil! |
|
|
ring = ring?.not_nil! |
|
|
h = hash_impl data |
|
|
|
|
|
space=UInt64::MAX/@shards |
|
|
|
|
|
targets = Array(UInt64).new |
|
|
|
|
|
(1..@shards).each do |
|
|
|
|
|
targets << h |
|
|
|
|
|
h+=space |
|
|
|
|
|
end |
|
|
|
|
|
slices = Array(Slice).new |
|
|
|
|
|
targets.each do |h| |
|
|
|
|
|
slices << get_slice_from_hash(ring, h) |
|
|
|
|
|
end |
|
|
|
|
|
|
|
|
targets = hash_and_split(data) |
|
|
|
|
|
slices = unfold_ring(ring,targets) |
|
|
return slices |
|
|
return slices |
|
|
end |
|
|
end |
|
|
|
|
|
|
|
|
def get_slices_for(data : String, n_last o">= 1 : Int32) |
|
|
|
|
|
|
|
|
def get_slices_for(data : String, n_last : UInt32) |
|
|
n_rings = Array(Ring).new |
|
|
n_rings = Array(Ring).new |
|
|
|
|
|
sllast = 0 |
|
|
@lock.synchronize do |
|
|
@lock.synchronize do |
|
|
|
|
|
sllast = @rings.size - 1 |
|
|
t = @rings.size-n_last |
|
|
t = @rings.size-n_last |
|
|
n_rings = @rings.skip( t > 0 ? t : 1 ) |
|
|
|
|
|
|
|
|
n_rings = @rings.skip( t >= 0 ? t : 1 ) |
|
|
end |
|
|
end |
|
|
h = hash_impl data |
|
|
|
|
|
space=UInt64::MAX/@shards |
|
|
|
|
|
targets = Array(UInt64).new |
|
|
|
|
|
(1..@shards).each do |
|
|
|
|
|
targets << h |
|
|
|
|
|
h+=space |
|
|
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
targets = hash_and_split(data) |
|
|
ret = Array(Array(Slice)).new |
|
|
ret = Array(Array(Slice)).new |
|
|
n_rings.each do |ring| |
|
|
n_rings.each do |ring| |
|
|
slices = Array(Slice).new |
|
|
|
|
|
targets.each do |h| |
|
|
|
|
|
slices << get_slice_from_hash(ring, h) |
|
|
|
|
|
end |
|
|
|
|
|
ret << slices |
|
|
|
|
|
|
|
|
ret << unfold_ring(ring,targets) |
|
|
end |
|
|
end |
|
|
return ret |
|
|
|
|
|
|
|
|
return SliceInfo.new ret, sllast, @shards |
|
|
end |
|
|
end |
|
|
end |
|
|
end |
|
|
|
|
|
|
|
|
end |
|
|
end |