From 597ba118c64a88d6f9985e5cb06748811575afc9 Mon Sep 17 00:00:00 2001 From: Ludovic 'Archivist' Lagouardette Date: Mon, 3 Feb 2020 17:22:59 +0100 Subject: [PATCH] Quotient filter and advancements to RC6 --- include/gp/quotient_filter.hpp | 208 +++++++++++++++++++++++++++++++++ include/gp_config.hpp | 6 +- include/rc6_generic.hpp | 29 ++--- tests.cpp | 1 + tests/quotient_filter.cpp | 127 ++++++++++++++++++++ tests/rc6_generic.cpp | 2 +- 6 files changed, 357 insertions(+), 16 deletions(-) create mode 100644 include/gp/quotient_filter.hpp create mode 100644 tests/quotient_filter.cpp diff --git a/include/gp/quotient_filter.hpp b/include/gp/quotient_filter.hpp new file mode 100644 index 0000000..9bb35b8 --- /dev/null +++ b/include/gp/quotient_filter.hpp @@ -0,0 +1,208 @@ +#pragma once +#include +#include +#include +#include +#include + + +namespace gp { + + + struct linear_skipstone { + constexpr static size_t ccbits = 1; + constexpr static size_t cclast = 1; + size_t ret = 0; + void next() { + ++ret; + } + operator size_t() { + return ret; + } + }; + + struct stepcache_skipstone { + constexpr static size_t ccbits = 7; + constexpr static size_t cclast = 127; + + constexpr static size_t list[128] = { + 0, 1, 2, 3, + 8, 9, 10, 11, + 20, 21, 22, 23, + 36, 37, 38, 39, + 56, 57, 58, 59, + 80, 81, 82, 83, + 108, 109, 110, 111, + 140, 141, 142, 143, + 176, 177, 178, 179, + 216, 217, 218, 219, + 260, 261, 262, 263, + 308, 309, 310, 311, + 360, 361, 362, 363, + 416, 417, 418, 419, + 476, 477, 478, 479, + 540, 541, 542, 543, + 608, 609, 610, 611, + 680, 681, 682, 683, + 756, 757, 758, 759, + 836, 837, 838, 839, + 920, 921, 922, 923, + 1008, 1009, 1010, 1011, + 1100, 1101, 1102, 1103, + 1196, 1197, 1198, 1199, + 1296, 1297, 1298, 1299, + 1400, 1401, 1402, 1403, + 1508, 1509, 1510, 1511, + 1736, 1737, 1738, 1739, + 1856, 1857, 1858, 1859, + 1980, 1981, 1982, 1983, + 2108, 2109, 2110, 2111, + 2240, 2241, 2242, 2243 + }; + size_t ret = 0; + void next() { + ++ret; + } + operator size_t() { + [[unlikely]] if ( ret >= 128 ) + return ret+list[127]; + return list[ret]; + } + }; + + template + class quotient_filter { + constexpr static size_t phys_size = (1 << magnitude) / 32; + using rem_t = typename gp::either<(remainder<=16), uint16_t, uint32_t>::type; + struct node{ + //rem_t ccount : skipstone_t::ccbits; + rem_t is_occupied : 1; + rem_t is_deleted : 1; + rem_t r : remainder; + }; + + gp::array< + node, + 1 << magnitude + > data; + + void skipstone_killswitch(size_t q, size_t skip) + { + if(q == (q+skip)%data.size()) + { + if constexpr (gp_config::has_exceptions) + { + throw gp::runtime_error("infinite skiploop detected"); + } + else + { + exit((int)gp_errorcodes::infinite_skipstone); + } + } + } + + public: + void set_hash(hash_type v) + { + const size_t q = v & ((1 << magnitude)-1); + const size_t r = (v >> magnitude) & ((1 << remainder)-1); + skipstone_t skip; + + for(;;) + { + auto& slot = data[(q+skip)%data.size()]; + if(slot.is_occupied) + { + if(slot.is_deleted) + { + slot.r = r; + slot.is_deleted = false; + for(;;) + { + skip.next(); + slot = data[(q+skip)%data.size()]; + if(!slot.is_occupied) + { + return; + } + else if(slot.r == r) + { + slot.is_deleted = true; + } + skipstone_killswitch(q, skip); + } + } + if(slot.r == r) + { + return; + } + skip.next(); + } + else + { + slot.r = r; + slot.is_occupied = true; + slot.is_deleted = false; + return; + } + skipstone_killswitch(q, skip); + } + } + + bool test_hash(hash_type v) + { + const size_t q = v & ((1 << magnitude)-1); + const size_t r = (v >> magnitude) & ((1 << remainder)-1); + skipstone_t skip; + + for(;;) + { + auto& slot = data[(q+skip)%data.size()]; + if(slot.is_occupied) + { + if(!slot.is_deleted) + { + if(slot.r == r) + { + return true; + } + } + skip.next(); + } + else + { + return false; + } + skipstone_killswitch(q, skip); + } + } + + void remove_hash(hash_type v) + { + const size_t q = v & ((1 << magnitude)-1); + const size_t r = (v >> magnitude) & ((1 << remainder)-1); + skipstone_t skip; + + for(;;) + { + auto& slot = data[(q+skip)%data.size()]; + if(slot.is_occupied) + { + if(!slot.is_deleted) + { + if(slot.r == r) + { + slot.is_deleted = true; + } + } + skip.next(); + } + else + { + return; + } + skipstone_killswitch(q, skip); + } + } + }; +} \ No newline at end of file diff --git a/include/gp_config.hpp b/include/gp_config.hpp index ccc3589..d832b4c 100644 --- a/include/gp_config.hpp +++ b/include/gp_config.hpp @@ -61,4 +61,8 @@ namespace gp_config{ else std::exit(-1);}; */ -} \ No newline at end of file +} + +enum class gp_errorcodes : int { + infinite_skipstone = 3000 +}; \ No newline at end of file diff --git a/include/rc6_generic.hpp b/include/rc6_generic.hpp index 21b5dcf..a5f2ce3 100644 --- a/include/rc6_generic.hpp +++ b/include/rc6_generic.hpp @@ -54,6 +54,7 @@ size_t lg(uint64_t v) template class RC6 { + static constexpr size_t word_size = 8*sizeof(word_t); static word_t r_l(const word_t& w, size_t v) { return (w << v) | ( w >> (sizeof(w)-v)); } @@ -64,30 +65,30 @@ class RC6 { class RC6_KeySched { public: - static constexpr size_t c = b/sizeof(word_t)/8; + static constexpr size_t c = (b+word_size-1)/word_size; static constexpr size_t v_3 = std::max(c, 2*r+4); static constexpr size_t v = v_3*3; private: - std::array S; + std::array S; public: RC6_KeySched(std::array L) { S[0] = P; - for(size_t i = 1; i < 2*r+3; ++i) + for(auto it = S.begin()+1; it < S.end(); ++it) { - S[i] = S[i - 1] + Q; + *it = *(it-1) + Q; } word_t A = 0; word_t B = 0; word_t i = 0; word_t j = 0; - for(size_t s = 1; s < v; ++s) + for(size_t s = 0; s < v; ++s) { + i = s % (2*r+4); + j = s % c; A = S[i] = r_l( S[i] + A + B, 3 ); - B = L[j] = r_l( L[j] + A + B, (A + B)%(8*sizeof(word_t))); - i = (i + 1) % (2*r+4); - j = (j + 1) % c; + B = L[j] = r_l( L[j] + A + B, (A + B)%(word_size)); } } @@ -116,17 +117,17 @@ public: B += S[0]; D += S[1]; - for(size_t i = 1; i < r; ++i) + for(size_t i = 1; i <= r; ++i) { auto t = r_l( B * ( 2 * B + 1 ), 5); auto u = r_l( D * ( 2 * D + 1 ), 5); - A = ((A ^ t) << u%(8*sizeof(word_t))) + S[2*i]; - C = ((C ^ u) << t%(8*sizeof(word_t))) + S[2*i+1]; - std::rotate(plaintext.begin(), plaintext.begin()+1, plaintext.end()); + A = r_l((A ^ t), u % word_size) + S[ 2 * i ]; + C = r_l((C ^ u), t % word_size) + S[ 2 * i + 1 ]; + auto tmp = A; A = B; B = C; C = D; D = tmp; } - A += S[2*r+3]; - C += S[2*r+2]; + A += S[2*r+2]; + C += S[2*r+3]; } }; \ No newline at end of file diff --git a/tests.cpp b/tests.cpp index f40a712..83a6e8c 100644 --- a/tests.cpp +++ b/tests.cpp @@ -4,6 +4,7 @@ #include "rc6_generic.cpp" #include "gp_test.cpp" #include "bloomfilter.cpp" +#include "quotient_filter.cpp" #include int main() diff --git a/tests/quotient_filter.cpp b/tests/quotient_filter.cpp new file mode 100644 index 0000000..9661de2 --- /dev/null +++ b/tests/quotient_filter.cpp @@ -0,0 +1,127 @@ +#include "test_scaffold.h" +#include +#include + +#include "gp/quotient_filter.hpp" + +typedef std::linear_congruential_engine cheap_rand; +typedef std::linear_congruential_engine cheap_rand_bis; + +struct qfilter_test : public test_scaffold { + uint32_t seed; + + qfilter_test() { + seed = std::random_device{}(); + name = __FILE__ ":1_seed"; + name += std::to_string(seed); + } + + virtual int run() { + + cheap_rand setter(seed); + cheap_rand getter(seed); + + gp::quotient_filter test_filter; + + for(int a = 0 ; a < 100; a++) + { + test_filter.set_hash(setter()); + } + + bool result = true; + for(int a = 0 ; a < 100; a++) + { + result *= test_filter.test_hash(getter()); + } + + return !result; + } +}; + +append_test dummy_rshyr43(new qfilter_test{}); + +struct qfilter2_test : public test_scaffold { + uint32_t seedA; + uint32_t seedB; + + qfilter2_test() { + seedA = std::random_device{}(); + seedB = std::random_device{}(); + name = __FILE__ ":2_seedA"; + name += std::to_string(seedA); + name += "&seedB"; + name += std::to_string(seedB); + } + + virtual int run() { + + cheap_rand setter(seedA); + cheap_rand deleter(seedA); + cheap_rand_bis getter(seedB); + + gp::quotient_filter test_filter; + + for(int a = 0 ; a < 100000; a++) + { + test_filter.set_hash(setter()); + } + + int interference = 0; + for(int a = 0 ; a < 100000; a++) + { + interference += test_filter.test_hash(getter()); + } + + for(int a = 0 ; a < 100000; a++) + { + test_filter.remove_hash(deleter()); + } + + return interference > 25; + } +}; + +append_test dummy_kegdflu3(new qfilter2_test{}); + +struct qfilter3_test : public test_scaffold { + uint32_t seedA; + uint32_t seedB; + + qfilter3_test() { + seedA = std::random_device{}(); + seedB = std::random_device{}(); + name = __FILE__ ":3_seedA"; + name += std::to_string(seedA); + name += "&seedB"; + name += std::to_string(seedB); + } + + virtual int run() { + + cheap_rand setter(seedA); + cheap_rand deleter(seedA); + cheap_rand_bis getter(seedB); + + gp::quotient_filter test_filter; + + for(int a = 0 ; a < 100000; a++) + { + test_filter.set_hash(setter()); + } + + int interference = 0; + for(int a = 0 ; a < 100000; a++) + { + interference += test_filter.test_hash(getter()); + } + + for(int a = 0 ; a < 100000; a++) + { + test_filter.remove_hash(deleter()); + } + + return interference > 25; + } +}; + +append_test dummy_kjdflu3(new qfilter3_test{}); \ No newline at end of file diff --git a/tests/rc6_generic.cpp b/tests/rc6_generic.cpp index c44901e..466ba53 100644 --- a/tests/rc6_generic.cpp +++ b/tests/rc6_generic.cpp @@ -9,7 +9,7 @@ struct RC6test : public test_scaffold { } virtual int run() { - using rc = RC6; + using rc = RC6<>; rc::key_type key = {0,0,0,0}; rc::block_type plaintext = {0,0,0,0};