Browse Source

Added slice finding and ring piling

master
Archivist 6 years ago
parent
commit
e9437ae23d
3 changed files with 106 additions and 12 deletions
  1. +1
    -0
      .gitignore
  2. +61
    -10
      spec/crystal-scatter_spec.cr
  3. +44
    -2
      src/crystal-scatter/metaring.cr

+ 1
- 0
.gitignore View File

@ -7,3 +7,4 @@
# Libraries don't need dependency lock
# Dependencies will be locked in application that uses them
/shard.lock
crystal-scatter

+ 61
- 10
spec/crystal-scatter_spec.cr View File

@ -3,14 +3,65 @@ require "./spec_helper"
describe Crystal::Scatter do
# TODO: Write tests
it "add elements" do
rg = Crystal::Scatter::RingGraph.new
rg.add(UInt64.new(1),[0,0,0,0,0,0],"OSD0")
rg.add(UInt64.new(1),[0,0,0,0,0,1],"OSD1")
rg.add(UInt64.new(1),[0,0,0,0,1,0],"OSD2")
rg.get_weight.should eq(3)
rg.generate_ring(BigRational.new(1))
rg[0][0][0][0][0][0].range_effector.not_nil!.[0].should eq(0)
rg[0][0][0][0][1][0].range_effector.not_nil!.[1].should eq(UInt64::MAX)
end
it "add elements to RingGraphs" do
rg = Crystal::Scatter::RingGraph.new
rg.add(UInt64.new(1),[0,0,0,0,0,0],"OSD0")
rg.add(UInt64.new(1),[0,0,0,0,0,1],"OSD1")
rg.add(UInt64.new(1),[0,0,0,0,1,0],"OSD2")
rg.get_weight.should eq(3)
rg.generate_ring(BigRational.new(1))
rg[0][0][0][0][0][0].range_effector.not_nil!.[0].should eq(0)
rg[0][0][0][0][1][0].range_effector.not_nil!.[1].should eq(UInt64::MAX)
end
it "hashes data fast" do
v = Crystal::Scatter::MetaRing.new(3)
time = Time.now
(1..100).each do |numb|
v.hash(numb.to_s)
end
t = (Time.now - time)
(t.milliseconds <= 1).should be_true
end
it "hashes data good" do
v = Crystal::Scatter::MetaRing.new(3)
hash_nb =1000000
store = Hash(UInt64, Int32).new
(1..hash_nb).each do |numb|
h = v.hash(numb.to_s)
nb=store.fetch(h,0)
store[h]=nb+1
end
conflicts = hash_nb-store.size
(conflicts <= 100).should be_true
mod_array = Array(Int32).new(1000,0)
slice_array = Array(Int32).new(1000,0)
store.each_key do |key|
key_mod = key%1000
mod_array[key_mod]+=1
key_slice = key/(UInt64::MAX/1000)
slice_array[key_slice]+=1
end
good_mod = true
good_slice = true
mod_array.each do |v|
good_mod&=(v > (hash_nb/1000*4/5))
end
slice_array.each do |v|
good_slice&=(v > (hash_nb/1000*4/5))
end
good_mod.should be_true
good_slice.should be_true
end
end

+ 44
- 2
src/crystal-scatter/metaring.cr View File

@ -1,5 +1,6 @@
require "./ring.cr"
require "mutex"
require "openssl"
module Crystal::Scatter
@ -15,9 +16,9 @@ module Crystal::Scatter
getter rings : Array(Ring)
getter ring_graph : RingGraph
def initialize
def initialize(@shards : UInt32)
@rings = Array(Ring).new
@ring_graph = b">Array(RingGraph).new
@ring_graph = RingGraph.new
@lock = Mutex.new
end
@ -34,6 +35,19 @@ module Crystal::Scatter
@rings.push ring
end
private def hash_impl(data : String)
ctx = OpenSSL::Digest.new("md4")
ctx << data
digest = ctx.digest
tag = IO::ByteFormat::BigEndian.decode(UInt64, digest)
tag ^= IO::ByteFormat::BigEndian.decode(UInt64, digest+8)
return tag
end
private def get_slice_from_hash(h : UInt64)
end
def add(element : Daemon)
@lock.synchronize do
add_impl element
@ -49,6 +63,34 @@ module Crystal::Scatter
update_impl
end
end
def hash(data : String)
hash_impl data
end
def get_slices_for(data : String)
ret = Array.new(@shards)
ring? : Ring | Nil
@lock.synchronize do
ring? = @rings.last
end
ring = ring?.not_nil!
h = hash_impl data
space=UInt64::MAX/@shards
targets = Array(UInt64).new
(1..shards).each do ||
targets << h
h+=space
end
slices = Array(Slice).new
targets.each do |h|
value = ring.find do |slice|
h>=slice.s_begin && slice.s_end>=h
end
slices << value.not_nil!
end
return slices
end
end
end

Loading…
Cancel
Save