From e9437ae23d7c87c65ad3068793d62597a15c361e Mon Sep 17 00:00:00 2001
From: Archivist <qnekomata@gmail.com>
Date: Sat, 11 Aug 2018 11:43:15 +0200
Subject: [PATCH] Added slice finding and ring piling

---
 .gitignore                      |  1 +
 spec/crystal-scatter_spec.cr    | 71 ++++++++++++++++++++++++++++-----
 src/crystal-scatter/metaring.cr | 46 ++++++++++++++++++++-
 3 files changed, 106 insertions(+), 12 deletions(-)

diff --git a/.gitignore b/.gitignore
index e29dae7..6107a16 100644
--- a/.gitignore
+++ b/.gitignore
@@ -7,3 +7,4 @@
 # Libraries don't need dependency lock
 # Dependencies will be locked in application that uses them
 /shard.lock
+crystal-scatter
diff --git a/spec/crystal-scatter_spec.cr b/spec/crystal-scatter_spec.cr
index 1916f59..e759233 100644
--- a/spec/crystal-scatter_spec.cr
+++ b/spec/crystal-scatter_spec.cr
@@ -3,14 +3,65 @@ require "./spec_helper"
 describe Crystal::Scatter do
   # TODO: Write tests
 
-  it "add elements" do
-    rg = Crystal::Scatter::RingGraph.new 
-    rg.add(UInt64.new(1),[0,0,0,0,0,0],"OSD0")
-    rg.add(UInt64.new(1),[0,0,0,0,0,1],"OSD1")
-    rg.add(UInt64.new(1),[0,0,0,0,1,0],"OSD2")
-    rg.get_weight.should eq(3)
-    rg.generate_ring(BigRational.new(1))
-    rg[0][0][0][0][0][0].range_effector.not_nil!.[0].should eq(0)
-    rg[0][0][0][0][1][0].range_effector.not_nil!.[1].should eq(UInt64::MAX)
-  end
+    it "add elements to RingGraphs" do
+        rg = Crystal::Scatter::RingGraph.new 
+        rg.add(UInt64.new(1),[0,0,0,0,0,0],"OSD0")
+        rg.add(UInt64.new(1),[0,0,0,0,0,1],"OSD1")
+        rg.add(UInt64.new(1),[0,0,0,0,1,0],"OSD2")
+        rg.get_weight.should eq(3)
+        rg.generate_ring(BigRational.new(1))
+        rg[0][0][0][0][0][0].range_effector.not_nil!.[0].should eq(0)
+        rg[0][0][0][0][1][0].range_effector.not_nil!.[1].should eq(UInt64::MAX)
+    end
+
+    it "hashes data fast" do
+        v = Crystal::Scatter::MetaRing.new(3)
+        time = Time.now
+        (1..100).each do |numb|
+            v.hash(numb.to_s)
+        end
+        t = (Time.now - time)
+        (t.milliseconds <= 1).should be_true
+    end
+
+    it "hashes data good" do
+        v = Crystal::Scatter::MetaRing.new(3)
+        hash_nb =1000000
+        store = Hash(UInt64, Int32).new
+        (1..hash_nb).each do |numb|
+            h = v.hash(numb.to_s)
+            nb=store.fetch(h,0)
+            store[h]=nb+1
+        end
+        
+        
+        conflicts = hash_nb-store.size
+        (conflicts <= 100).should be_true
+
+
+        mod_array = Array(Int32).new(1000,0)
+        slice_array = Array(Int32).new(1000,0)
+
+        store.each_key do |key|
+            key_mod = key%1000
+            mod_array[key_mod]+=1
+
+            key_slice = key/(UInt64::MAX/1000)
+            slice_array[key_slice]+=1
+        end
+
+        good_mod = true
+        good_slice = true
+
+        mod_array.each do |v|
+            good_mod&=(v > (hash_nb/1000*4/5))
+        end
+
+        slice_array.each do |v|
+            good_slice&=(v > (hash_nb/1000*4/5))
+        end
+
+        good_mod.should be_true
+        good_slice.should be_true
+    end
 end
diff --git a/src/crystal-scatter/metaring.cr b/src/crystal-scatter/metaring.cr
index 34ba732..aa7ba27 100644
--- a/src/crystal-scatter/metaring.cr
+++ b/src/crystal-scatter/metaring.cr
@@ -1,5 +1,6 @@
 require "./ring.cr"
 require "mutex"
+require "openssl"
 
 module Crystal::Scatter 
 
@@ -15,9 +16,9 @@ module Crystal::Scatter
         getter rings : Array(Ring)
         getter ring_graph : RingGraph
 
-        def initialize
+        def initialize(@shards : UInt32)
             @rings = Array(Ring).new
-            @ring_graph = Array(RingGraph).new
+            @ring_graph = RingGraph.new
             @lock = Mutex.new
         end
 
@@ -34,6 +35,19 @@ module Crystal::Scatter
             @rings.push ring
         end
 
+        private def hash_impl(data : String)
+            ctx = OpenSSL::Digest.new("md4")
+            ctx << data
+            digest = ctx.digest
+            tag = IO::ByteFormat::BigEndian.decode(UInt64, digest)
+            tag ^= IO::ByteFormat::BigEndian.decode(UInt64, digest+8)
+            return tag
+        end
+
+        private def get_slice_from_hash(h : UInt64)
+            
+        end
+
         def add(element : Daemon)
             @lock.synchronize do
                 add_impl element
@@ -49,6 +63,34 @@ module Crystal::Scatter
                 update_impl
             end
         end
+
+        def hash(data : String)
+            hash_impl data
+        end
+
+        def get_slices_for(data : String)
+            ret = Array.new(@shards)
+            ring? : Ring | Nil
+            @lock.synchronize do
+                ring? = @rings.last
+            end
+            ring = ring?.not_nil!
+            h = hash_impl data
+            space=UInt64::MAX/@shards
+            targets = Array(UInt64).new
+            (1..shards).each do ||
+                targets << h
+                h+=space
+            end
+            slices = Array(Slice).new
+            targets.each do |h|
+                value = ring.find do |slice|
+                    h>=slice.s_begin && slice.s_end>=h
+                end
+                slices << value.not_nil!
+            end
+            return slices
+        end
     end
 
 end
\ No newline at end of file