From 18dcf0e48b99c5d535f5ccdbc308f1624074ef38 Mon Sep 17 00:00:00 2001 From: Ludovic 'Archivist' Lagouardette Date: Mon, 17 Sep 2018 12:32:47 +0200 Subject: [PATCH] tests pass, Staging a 1.0.0-rc1 --- .editorconfig | 4 +- README.md | 35 ++++++++-------- shard.yml | 8 +++- spec/crystal-scatter_spec.cr | 6 +-- src/crystal-scatter/metaring.cr | 6 +-- src/crystal-scatter/ring.cr | 5 ++- src/crystal-scatter/ring_graph.cr | 70 ++++++++++++++++++++----------- src/crystal-scatter/version.cr | 2 +- 8 files changed, 82 insertions(+), 54 deletions(-) diff --git a/.editorconfig b/.editorconfig index 163eb75..e75be6c 100644 --- a/.editorconfig +++ b/.editorconfig @@ -4,6 +4,6 @@ root = true charset = utf-8 end_of_line = lf insert_final_newline = true -indent_style = space -indent_size = 2 +indent_style = tabs +indent_size = 4 trim_trailing_whitespace = true diff --git a/README.md b/README.md index 6e0e66b..d1203ac 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # crystal-scatter -Release candidate for v0.1 "John Scott" +Release candidate for v1.0 "David Robert Jones" ## Installation @@ -10,7 +10,7 @@ Add this to your application's `shard.yml`: dependencies: crystal-scatter: git: https://git.nekoit.xyz/Archivist/crystal-scatter - version: ~> 0.1.0-rc2 + version: ~> 1.0.0-rc1 ``` ## Usage @@ -27,11 +27,11 @@ client_ip = "185.124.235.1" ring = Crystal::Scatter::MetaRing.new(3) ring.add [ - Crystal::Scatter::Daemon (8 , [0,0,0,0,0,0], "https://api01.myservice.com"), - Crystal::Scatter::Daemon (16, [0,0,0,0,1,0], "https://api02.myservice.com"), - Crystal::Scatter::Daemon (12, [1,0,0,0,0,0], "https://api03.myservice.com"), - Crystal::Scatter::Daemon (12, [1,0,0,0,1,0], "https://api04.myservice.com"), - Crystal::Scatter::Daemon (12, [2,0,0,0,0,0], "https://api05.myservice.com") + Crystal::Scatter::Daemon (8 , uuid01, [0,0,0,0,0,0], "https://api01.myservice.com"), + Crystal::Scatter::Daemon (16, uuid02, [0,0,0,0,1,0], "https://api02.myservice.com"), + Crystal::Scatter::Daemon (12, uuid03, [1,0,0,0,0,0], "https://api03.myservice.com"), + Crystal::Scatter::Daemon (12, uuid04, [1,0,0,0,1,0], "https://api04.myservice.com"), + Crystal::Scatter::Daemon (12, uuid05, [2,0,0,0,0,0], "https://api05.myservice.com") ] ring.get_slices_for(client_ip) @@ -47,16 +47,17 @@ The output will provide you information to contact the service in a balanced way ### In details ```crystal -Crystal::Scatter::Daemon (12, [2,0,0,0,0,0], "https://api05.myservice.com") - ^ ^ ^ ^ ^ ^ ^ ^ - | | | | | | | | - | | | | | | | L Real target - | | | | | | L Serving daemon - | | | | | L Serving server - | | | | L Serving PSU - | | | L Serving rack - | | L Serving room - | L Serving datacenter +Crystal::Scatter::Daemon (12, uuid, [2,0,0,0,0], "https://api05.myservice.com") + ^ ^ ^ ^ ^ ^ ^ ^ ^ + | | | | | | | | | + | | | | | | | | L Real target + | | | | | | | L Serving daemon + | | | | | | L Serving server + | | | | | L Serving PSU + | | | | L Serving rack + | | | L Serving room + | | L Serving datacenter + | L UUID provided by the HARP wrapper L Weight ``` diff --git a/shard.yml b/shard.yml index aab8963..7e2b7eb 100644 --- a/shard.yml +++ b/shard.yml @@ -1,9 +1,13 @@ name: crystal-scatter -version: 0.1.0-rc2 +version: 1.0.0-rc1 authors: - Ludovic 'Archivist' Lagouardette -crystal: 0.25.0 +crystal: 0.26.1 + +dependencies: + crystalline: + path: ../crystalline license: MIT diff --git a/spec/crystal-scatter_spec.cr b/spec/crystal-scatter_spec.cr index 57b869e..9a42107 100644 --- a/spec/crystal-scatter_spec.cr +++ b/spec/crystal-scatter_spec.cr @@ -3,14 +3,14 @@ require "./spec_helper" describe Crystal::Scatter do it "add elements to RingGraphs" do - rg = Crystal::Scatter::RingGraph.new + rg = Crystal::Scatter::RingGraph.new 0 rg.add(UInt64.new(1),[0,0,0,0,0,0],"OSD0") rg.add(UInt64.new(1),[0,0,0,0,0,1],"OSD1") rg.add(UInt64.new(1),[0,0,0,0,1,0],"OSD2") rg.get_weight.should eq(3) rg.generate_ring(BigRational.new(1)) - rg[0][0][0][0][0][0].range_effector.not_nil!.[0].should eq(0) - rg[0][0][0][0][1][0].range_effector.not_nil!.[1].should eq(UInt64::MAX) + rg.first.first.first.first.first.first.range_effector.not_nil!.[0].should eq(0) + rg.last.last.last.last.last.last.range_effector.not_nil!.[1].should eq(UInt64::MAX) end it "slices data into multiple pods" do diff --git a/src/crystal-scatter/metaring.cr b/src/crystal-scatter/metaring.cr index 804157d..4bdd8a6 100644 --- a/src/crystal-scatter/metaring.cr +++ b/src/crystal-scatter/metaring.cr @@ -8,7 +8,7 @@ module Crystal::Scatter class Daemon getter weight : UInt64 getter location : Array(Int32) - getter url : String + getter url : String def initialize(@weight : UInt64, @location : Array(Int32), @url : String) end end @@ -31,7 +31,7 @@ module Crystal::Scatter def initialize(@shards : UInt32, @split_ratio : BigRational = BigRational.new(1,2)) @rings = Array(Ring).new - @ring_graph = RingGraph.new + @ring_graph = RingGraph.new 0 @lock = Mutex.new end @@ -43,7 +43,7 @@ module Crystal::Scatter @ring_graph.generate_ring @split_ratio ring = Ring.new @ring_graph.snapshot do |daemon| - ring.push Slice.new(daemon.range_effector.not_nil!.[0],daemon.range_effector.not_nil!.[1],daemon.reach_at) + ring.push Slice.new(daemon.range_effector.not_nil!.[0],daemon.range_effector.not_nil!.[1],daemon.reach_at,daemon.self_id) end @rings.push ring end diff --git a/src/crystal-scatter/ring.cr b/src/crystal-scatter/ring.cr index 5308719..b93d2c7 100644 --- a/src/crystal-scatter/ring.cr +++ b/src/crystal-scatter/ring.cr @@ -6,10 +6,11 @@ module Crystal::Scatter JSON.mapping( s_begin: UInt64, s_end: UInt64, - url: String + url: String, + id: Int32 ) - def initialize(@s_begin,@s_end,@url) + def initialize(@s_begin,@s_end,@url,@id) end end diff --git a/src/crystal-scatter/ring_graph.cr b/src/crystal-scatter/ring_graph.cr index f97402c..4ef5b9f 100644 --- a/src/crystal-scatter/ring_graph.cr +++ b/src/crystal-scatter/ring_graph.cr @@ -1,4 +1,5 @@ require "big" +require "crystalline" module Crystal::Scatter @@ -9,9 +10,10 @@ module Crystal::Scatter class Daemon(Ign,T) getter range_effector : Nil | Tuple(UInt64,UInt64) - getter reach_at : T + getter reach_at : T + getter self_id : Int32 - def initialize(@weight : UInt64 = 1) + def initialize(@self_id, @weight : UInt64 = 1) @reach_at = T.new end @@ -27,7 +29,11 @@ module Crystal::Scatter @weight end - def add(@weight, elems,@reach_at) + def <=>(other : Daemon(Ign,T)) + @self_id<=>other.self_id + end + + def add(@weight,elems,@reach_at) end def set_end @@ -39,38 +45,55 @@ module Crystal::Scatter end end - class Container(T,U) < Array(T) + class Container(T,U,SelfT) + @contents : Crystalline::Containers::RBTreeSet(T) + getter self_id : Int32 + + def initialize(@self_id : Int32) + @contents = SelfT.new + end + + def <=>(other : Container(T,U,SelfT)) + @self_id<=>other.self_id + end + def set_effectors(last_set : UInt64, split_rate : BigRational, total_weight : UInt64, split_ratio : BigRational) : UInt64 - split_rate *= size - each do |elem| + split_rate *= @contents.size + @contents.each do |elem| last_set = elem.set_effectors(last_set, split_rate,total_weight,split_ratio) end return last_set - end + end + + def first + @contents.first.not_nil! + end + + def last + @contents.last.not_nil! + end def set_end - last.set_end + @contents.last.not_nil!.set_end end def get_weight : UInt64 total : UInt64 = 0 - each do |elem| + @contents.each do |elem| total += elem.get_weight end return total end def add(nweight, elems, reach_at) - if elems[0]==size - push T.new - elsif elems[0]>size - raise ArgumentError.new("Tried to insert in an invalid position") - end - at(elems[0]).add(nweight, elems.skip(1), reach_at) + if(!(@contents.has_key?(T.new(elems[0])))) + @contents << T.new(elems[0]) + end + @contents.get(T.new(elems[0])).not_nil!.add(nweight, elems.skip(1), reach_at) end def snapshot - each do |elem| + @contents.each do |elem| elem.snapshot do |x| yield x end @@ -78,24 +101,23 @@ module Crystal::Scatter end end - class Server(U) < Container(Daemon(U,U),U) end + class Server(U) < Container(Daemon(U,U),U,Crystalline::Containers::RBTreeSet(Daemon(U,U))) end - class PSU(U) < Container(Server(U),U) end + class PSU(U) < Container(Server(U),U,Crystalline::Containers::RBTreeSet(Server(U))) end - class Rack(U) < Container(PSU(U),U) end + class Rack(U) < Container(PSU(U),U,Crystalline::Containers::RBTreeSet(PSU(U))) end - class Room(U) < Container(Rack(U),U) end + class Room(U) < Container(Rack(U),U,Crystalline::Containers::RBTreeSet(Rack(U))) end - class Datacenter(U) < Container(Room(U),U) end + class Datacenter(U) < Container(Room(U),U,Crystalline::Containers::RBTreeSet(Room(U))) end - class Root(U) < Container(Datacenter(U),U) end + class Root(U) < Container(Datacenter(U),U,Crystalline::Containers::RBTreeSet(Datacenter(U))) end end - class RingGraph < Servers::Root(String) + class RingGraph < Servers::Root(String) def generate_ring(split_ratio : BigRational) set_effectors(UInt64::MAX, BigRational.new(1), get_weight, split_ratio) set_end end end - end \ No newline at end of file diff --git a/src/crystal-scatter/version.cr b/src/crystal-scatter/version.cr index 34bceb9..6022a2b 100644 --- a/src/crystal-scatter/version.cr +++ b/src/crystal-scatter/version.cr @@ -1,3 +1,3 @@ module Crystal::Scatter - VERSION = "0.1.0-rc2" + VERSION = "1.0.0-rc1" end