Browse Source

tests pass, Staging a 1.0.0-rc1

master
Ludovic 'Archivist' Lagouardette 6 years ago
parent
commit
18dcf0e48b
8 changed files with 82 additions and 54 deletions
  1. +2
    -2
      .editorconfig
  2. +18
    -17
      README.md
  3. +6
    -2
      shard.yml
  4. +3
    -3
      spec/crystal-scatter_spec.cr
  5. +3
    -3
      src/crystal-scatter/metaring.cr
  6. +3
    -2
      src/crystal-scatter/ring.cr
  7. +46
    -24
      src/crystal-scatter/ring_graph.cr
  8. +1
    -1
      src/crystal-scatter/version.cr

+ 2
- 2
.editorconfig View File

@ -4,6 +4,6 @@ root = true
charset = utf-8 charset = utf-8
end_of_line = lf end_of_line = lf
insert_final_newline = true insert_final_newline = true
indent_style = space
indent_size = 2
indent_style = tabs
indent_size = 4
trim_trailing_whitespace = true trim_trailing_whitespace = true

+ 18
- 17
README.md View File

@ -1,6 +1,6 @@
# crystal-scatter # crystal-scatter
Release candidate for v0.1 "John Scott"
Release candidate for v1.0 "David Robert Jones"
## Installation ## Installation
@ -10,7 +10,7 @@ Add this to your application's `shard.yml`:
dependencies: dependencies:
crystal-scatter: crystal-scatter:
git: https://git.nekoit.xyz/Archivist/crystal-scatter git: https://git.nekoit.xyz/Archivist/crystal-scatter
version: ~> 0.1.0-rc2
version: ~> 1.0.0-rc1
``` ```
## Usage ## Usage
@ -27,11 +27,11 @@ client_ip = "185.124.235.1"
ring = Crystal::Scatter::MetaRing.new(3) ring = Crystal::Scatter::MetaRing.new(3)
ring.add [ ring.add [
Crystal::Scatter::Daemon (8 , [0,0,0,0,0,0], "https://api01.myservice.com"),
Crystal::Scatter::Daemon (16, [0,0,0,0,1,0], "https://api02.myservice.com"),
Crystal::Scatter::Daemon (12, [1,0,0,0,0,0], "https://api03.myservice.com"),
Crystal::Scatter::Daemon (12, [1,0,0,0,1,0], "https://api04.myservice.com"),
Crystal::Scatter::Daemon (12, [2,0,0,0,0,0], "https://api05.myservice.com")
Crystal::Scatter::Daemon (8 , uuid01, [0,0,0,0,0,0], "https://api01.myservice.com"),
Crystal::Scatter::Daemon (16, uuid02, [0,0,0,0,1,0], "https://api02.myservice.com"),
Crystal::Scatter::Daemon (12, uuid03, [1,0,0,0,0,0], "https://api03.myservice.com"),
Crystal::Scatter::Daemon (12, uuid04, [1,0,0,0,1,0], "https://api04.myservice.com"),
Crystal::Scatter::Daemon (12, uuid05, [2,0,0,0,0,0], "https://api05.myservice.com")
] ]
ring.get_slices_for(client_ip) ring.get_slices_for(client_ip)
@ -47,16 +47,17 @@ The output will provide you information to contact the service in a balanced way
### In details ### In details
```crystal ```crystal
Crystal::Scatter::Daemon (12, [2,0,0,0,0,0], "https://api05.myservice.com")
^ ^ ^ ^ ^ ^ ^ ^
| | | | | | | |
| | | | | | | L Real target
| | | | | | L Serving daemon
| | | | | L Serving server
| | | | L Serving PSU
| | | L Serving rack
| | L Serving room
| L Serving datacenter
Crystal::Scatter::Daemon (12, uuid, [2,0,0,0,0], "https://api05.myservice.com")
^ ^ ^ ^ ^ ^ ^ ^ ^
| | | | | | | | |
| | | | | | | | L Real target
| | | | | | | L Serving daemon
| | | | | | L Serving server
| | | | | L Serving PSU
| | | | L Serving rack
| | | L Serving room
| | L Serving datacenter
| L UUID provided by the HARP wrapper
L Weight L Weight
``` ```

+ 6
- 2
shard.yml View File

@ -1,9 +1,13 @@
name: crystal-scatter name: crystal-scatter
version: 0.1.0-rc2
version: 1.0.0-rc1
authors: authors:
- Ludovic 'Archivist' Lagouardette <lagouardette.ludovic@gmail.com> - Ludovic 'Archivist' Lagouardette <lagouardette.ludovic@gmail.com>
crystal: 0.25.0
crystal: 0.26.1
dependencies:
crystalline:
path: ../crystalline
license: MIT license: MIT

+ 3
- 3
spec/crystal-scatter_spec.cr View File

@ -3,14 +3,14 @@ require "./spec_helper"
describe Crystal::Scatter do describe Crystal::Scatter do
it "add elements to RingGraphs" do it "add elements to RingGraphs" do
rg = Crystal::Scatter::RingGraph.new
rg = Crystal::Scatter::RingGraph.new 0
rg.add(UInt64.new(1),[0,0,0,0,0,0],"OSD0") rg.add(UInt64.new(1),[0,0,0,0,0,0],"OSD0")
rg.add(UInt64.new(1),[0,0,0,0,0,1],"OSD1") rg.add(UInt64.new(1),[0,0,0,0,0,1],"OSD1")
rg.add(UInt64.new(1),[0,0,0,0,1,0],"OSD2") rg.add(UInt64.new(1),[0,0,0,0,1,0],"OSD2")
rg.get_weight.should eq(3) rg.get_weight.should eq(3)
rg.generate_ring(BigRational.new(1)) rg.generate_ring(BigRational.new(1))
rg[0][0][0][0][0][0].range_effector.not_nil!.[0].should eq(0)
rg[0][0][0][0][1][0].range_effector.not_nil!.[1].should eq(UInt64::MAX)
rg.first.first.first.first.first.first.range_effector.not_nil!.[0].should eq(0)
rg.last.last.last.last.last.last.range_effector.not_nil!.[1].should eq(UInt64::MAX)
end end
it "slices data into multiple pods" do it "slices data into multiple pods" do

+ 3
- 3
src/crystal-scatter/metaring.cr View File

@ -8,7 +8,7 @@ module Crystal::Scatter
class Daemon class Daemon
getter weight : UInt64 getter weight : UInt64
getter location : Array(Int32) getter location : Array(Int32)
getter url : String
getter url : String
def initialize(@weight : UInt64, @location : Array(Int32), @url : String) def initialize(@weight : UInt64, @location : Array(Int32), @url : String)
end end
end end
@ -31,7 +31,7 @@ module Crystal::Scatter
def initialize(@shards : UInt32, @split_ratio : BigRational = BigRational.new(1,2)) def initialize(@shards : UInt32, @split_ratio : BigRational = BigRational.new(1,2))
@rings = Array(Ring).new @rings = Array(Ring).new
@ring_graph = RingGraph.new
@ring_graph = RingGraph.new 0
@lock = Mutex.new @lock = Mutex.new
end end
@ -43,7 +43,7 @@ module Crystal::Scatter
@ring_graph.generate_ring @split_ratio @ring_graph.generate_ring @split_ratio
ring = Ring.new ring = Ring.new
@ring_graph.snapshot do |daemon| @ring_graph.snapshot do |daemon|
ring.push Slice.new(daemon.range_effector.not_nil!.[0],daemon.range_effector.not_nil!.[1],daemon.reach_at)
ring.push Slice.new(daemon.range_effector.not_nil!.[0],daemon.range_effector.not_nil!.[1],daemon.reach_at,daemon.self_id)
end end
@rings.push ring @rings.push ring
end end

+ 3
- 2
src/crystal-scatter/ring.cr View File

@ -6,10 +6,11 @@ module Crystal::Scatter
JSON.mapping( JSON.mapping(
s_begin: UInt64, s_begin: UInt64,
s_end: UInt64, s_end: UInt64,
url: String
url: String,
id: Int32
) )
def initialize(@s_begin,@s_end,@url)
def initialize(@s_begin,@s_end,@url,@id)
end end
end end

+ 46
- 24
src/crystal-scatter/ring_graph.cr View File

@ -1,4 +1,5 @@
require "big" require "big"
require "crystalline"
module Crystal::Scatter module Crystal::Scatter
@ -9,9 +10,10 @@ module Crystal::Scatter
class Daemon(Ign,T) class Daemon(Ign,T)
getter range_effector : Nil | Tuple(UInt64,UInt64) getter range_effector : Nil | Tuple(UInt64,UInt64)
getter reach_at : T
getter reach_at : T
getter self_id : Int32
def initialize(@weight : UInt64 = 1)
def initialize(@self_id, @weight : UInt64 = 1)
@reach_at = T.new @reach_at = T.new
end end
@ -27,7 +29,11 @@ module Crystal::Scatter
@weight @weight
end end
def add(@weight, elems,@reach_at)
def <=>(other : Daemon(Ign,T))
@self_id<=>other.self_id
end
def add(@weight,elems,@reach_at)
end end
def set_end def set_end
@ -39,38 +45,55 @@ module Crystal::Scatter
end end
end end
class Container(T,U) < Array(T)
class Container(T,U,SelfT)
@contents : Crystalline::Containers::RBTreeSet(T)
getter self_id : Int32
def initialize(@self_id : Int32)
@contents = SelfT.new
end
def <=>(other : Container(T,U,SelfT))
@self_id<=>other.self_id
end
def set_effectors(last_set : UInt64, split_rate : BigRational, total_weight : UInt64, split_ratio : BigRational) : UInt64 def set_effectors(last_set : UInt64, split_rate : BigRational, total_weight : UInt64, split_ratio : BigRational) : UInt64
split_rate *= size
each do |elem|
split_rate *= vi">@contents.size
vi">@contents.each do |elem|
last_set = elem.set_effectors(last_set, split_rate,total_weight,split_ratio) last_set = elem.set_effectors(last_set, split_rate,total_weight,split_ratio)
end end
return last_set return last_set
end
end
def first
@contents.first.not_nil!
end
def last
@contents.last.not_nil!
end
def set_end def set_end
last.set_end
vi">@contents.last.not_nil!.set_end
end end
def get_weight : UInt64 def get_weight : UInt64
total : UInt64 = 0 total : UInt64 = 0
each do |elem|
vi">@contents.each do |elem|
total += elem.get_weight total += elem.get_weight
end end
return total return total
end end
def add(nweight, elems, reach_at) def add(nweight, elems, reach_at)
if elems[0]==size
push T.new
elsif elems[0]>size
raise ArgumentError.new("Tried to insert in an invalid position")
end
at(elems[0]).add(nweight, elems.skip(1), reach_at)
if(!(@contents.has_key?(T.new(elems[0]))))
@contents << T.new(elems[0])
end
@contents.get(T.new(elems[0])).not_nil!.add(nweight, elems.skip(1), reach_at)
end end
def snapshot def snapshot
each do |elem|
vi">@contents.each do |elem|
elem.snapshot do |x| elem.snapshot do |x|
yield x yield x
end end
@ -78,24 +101,23 @@ module Crystal::Scatter
end end
end end
class Server(U) < Container(Daemon(U,U),U) end
class Server(U) < Container(Daemon(U,U),U,Crystalline::Containers::RBTreeSet(Daemon(U,U))) end
class PSU(U) < Container(Server(U),U) end
class PSU(U) < Container(Server(U),U,Crystalline::Containers::RBTreeSet(Server(U))) end
class Rack(U) < Container(PSU(U),U) end
class Rack(U) < Container(PSU(U),U,Crystalline::Containers::RBTreeSet(PSU(U))) end
class Room(U) < Container(Rack(U),U) end
class Room(U) < Container(Rack(U),U,Crystalline::Containers::RBTreeSet(Rack(U))) end
class Datacenter(U) < Container(Room(U),U) end
class Datacenter(U) < Container(Room(U),U,Crystalline::Containers::RBTreeSet(Room(U))) end
class Root(U) < Container(Datacenter(U),U) end
class Root(U) < Container(Datacenter(U),U,Crystalline::Containers::RBTreeSet(Datacenter(U))) end
end end
class RingGraph < Servers::Root(String)
class RingGraph < Servers::Root(String)
def generate_ring(split_ratio : BigRational) def generate_ring(split_ratio : BigRational)
set_effectors(UInt64::MAX, BigRational.new(1), get_weight, split_ratio) set_effectors(UInt64::MAX, BigRational.new(1), get_weight, split_ratio)
set_end set_end
end end
end end
end end

+ 1
- 1
src/crystal-scatter/version.cr View File

@ -1,3 +1,3 @@
module Crystal::Scatter module Crystal::Scatter
VERSION = "0.1.0-rc2"
VERSION = "1.0.0-rc1"
end end

Loading…
Cancel
Save