Quotient filter and advancements to RC6

include/gp/quotient_filter.hpp

@@ -0,0 +1,208 @@
+#pragma once
+#include <stdint.h>
+#include <atomic>
+#include <gp/array.hpp>
+#include <gp/algorithm/tmp_manip.hpp>
+#include <gp/exception.hpp>
+
+
+namespace gp {
+
+
+	struct linear_skipstone {
+		constexpr static size_t ccbits = 1;
+		constexpr static size_t cclast = 1;
+		size_t ret = 0;
+		void next() {
+			++ret;
+		}
+		operator size_t() {
+			return ret;
+		}
+	};
+
+	struct stepcache_skipstone {
+		constexpr static size_t ccbits = 7;
+		constexpr static size_t cclast = 127;
+
+		constexpr static size_t list[128] = {
+			0,    1,    2,    3,
+			8,    9,   10,   11,
+			20,   21,   22,   23,
+			36,   37,   38,   39,
+			56,   57,   58,   59,
+			80,   81,   82,   83,
+			108,  109,  110,  111,
+			140,  141,  142,  143,
+			176,  177,  178,  179,
+			216,  217,  218,  219,
+			260,  261,  262,  263,
+			308,  309,  310,  311,
+			360,  361,  362,  363,
+			416,  417,  418,  419,
+			476,  477,  478,  479,
+			540,  541,  542,  543,
+			608,  609,  610,  611,
+			680,  681,  682,  683,
+			756,  757,  758,  759,
+			836,  837,  838,  839,
+			920,  921,  922,  923,
+			1008, 1009, 1010, 1011,
+			1100, 1101, 1102, 1103,
+			1196, 1197, 1198, 1199,
+			1296, 1297, 1298, 1299,
+			1400, 1401, 1402, 1403,
+			1508, 1509, 1510, 1511,
+			1736, 1737, 1738, 1739,
+			1856, 1857, 1858, 1859,
+			1980, 1981, 1982, 1983,
+			2108, 2109, 2110, 2111,
+			2240, 2241, 2242, 2243
+		};
+		size_t ret = 0;
+		void next() {
+			++ret;
+		}
+		operator size_t() {
+			[[unlikely]] if ( ret >= 128 )
+				return ret+list[127];
+			return list[ret];
+		}
+	};
+
+	template<typename hash_type = uint32_t, uint8_t magnitude = 20, uint8_t remainder = (8*sizeof(hash_type))-magnitude, typename skipstone_t = linear_skipstone>
+	class quotient_filter {
+		constexpr static size_t phys_size = (1 << magnitude) / 32;
+		using rem_t = typename gp::either<(remainder<=16), uint16_t, uint32_t>::type;
+		struct node{
+			//rem_t ccount : skipstone_t::ccbits;
+			rem_t is_occupied : 1;
+			rem_t is_deleted : 1;
+			rem_t r : remainder;
+		};
+
+		gp::array<
+			node,
+			1 << magnitude
+		> data;
+
+		void skipstone_killswitch(size_t q, size_t skip)
+		{
+			if(q == (q+skip)%data.size())
+			{
+				if constexpr (gp_config::has_exceptions)
+				{
+					throw gp::runtime_error("infinite skiploop detected");
+				}
+				else
+				{
+					exit((int)gp_errorcodes::infinite_skipstone);
+				}
+			}
+		}
+
+	public:
+		void set_hash(hash_type v)
+		{
+			const size_t q  = v & ((1 << magnitude)-1);
+			const size_t r  = (v >> magnitude) & ((1 << remainder)-1);
+			skipstone_t skip;
+
+			for(;;)
+			{
+				auto& slot = data[(q+skip)%data.size()];
+				if(slot.is_occupied)
+				{
+					if(slot.is_deleted)
+					{
+						slot.r = r;
+						slot.is_deleted = false;
+						for(;;)
+						{
+							skip.next();
+							slot = data[(q+skip)%data.size()];
+							if(!slot.is_occupied)
+							{
+								return;
+							}
+							else if(slot.r == r)
+							{
+								slot.is_deleted = true;
+							}
+							skipstone_killswitch(q, skip);
+						}
+					}
+					if(slot.r == r)
+					{
+						return;
+					}
+					skip.next();
+				}
+				else
+				{
+					slot.r = r;
+					slot.is_occupied = true;
+					slot.is_deleted = false;
+					return;
+				}
+				skipstone_killswitch(q, skip);
+			}
+		}
+
+		bool test_hash(hash_type v)
+		{
+			const size_t q  = v & ((1 << magnitude)-1);
+			const size_t r  = (v >> magnitude) & ((1 << remainder)-1);
+			skipstone_t skip;
+
+			for(;;)
+			{
+				auto& slot = data[(q+skip)%data.size()];
+				if(slot.is_occupied)
+				{
+					if(!slot.is_deleted)
+					{
+						if(slot.r == r)
+						{
+							return true;
+						}
+					}
+					skip.next();
+				}
+				else
+				{
+					return false;
+				}
+				skipstone_killswitch(q, skip);
+			}
+		}
+
+		void remove_hash(hash_type v)
+		{
+			const size_t q  = v & ((1 << magnitude)-1);
+			const size_t r  = (v >> magnitude) & ((1 << remainder)-1);
+			skipstone_t skip;
+
+			for(;;)
+			{
+				auto& slot = data[(q+skip)%data.size()];
+				if(slot.is_occupied)
+				{
+					if(!slot.is_deleted)
+					{
+						if(slot.r == r)
+						{
+							slot.is_deleted = true;
+						}
+					}
+					skip.next();
+				}
+				else
+				{
+					return;
+				}
+				skipstone_killswitch(q, skip);
+			}
+		}
+	};
+}

include/gp_config.hpp

@@ -61,4 +61,8 @@ namespace gp_config{
 		else
 			std::exit(-1);};
 	*/
-}
+}
+
+enum class gp_errorcodes : int {
+	infinite_skipstone = 3000
+};

include/rc6_generic.hpp

@@ -54,6 +54,7 @@ size_t lg(uint64_t v)
 
 template<typename word_t = uint32_t, size_t r = 20, size_t b = 128, word_t P = 0xB7E15163, word_t Q = 0x9E3779B9>
 class RC6 {
+	static constexpr size_t word_size = 8*sizeof(word_t);
 	static word_t r_l(const word_t& w, size_t v) {
 		return (w << v) | ( w >> (sizeof(w)-v));
 	}
@@ -64,30 +65,30 @@ class RC6 {
 
 	class RC6_KeySched {
 	public:
-		static constexpr size_t c = n">b/sizeof(word_t)/8;
+		static constexpr size_t c = p">(b+word_size-1)/word_size;
 		static constexpr size_t v_3 = std::max(c, 2*r+4);
 		static constexpr size_t v = v_3*3;
 	private:
-		std::array<word_t, n">v_3> S;
+		std::array<word_t, mi">2*r+4> S;
 	public:
 		RC6_KeySched(std::array<word_t, c> L)
 		{
 			S[0] = P;
-			for(n">size_t i = 1; i < 2*r+3; ++i)
+			for(k">auto it = S.begin()+1; it < S.end(); ++it)
 			{
-				n">S[i] = S[i - 1] + Q;
+				o">*it = *(it-1) + Q;
 			}
 			word_t A = 0;
 			word_t B = 0;
 			word_t i = 0;
 			word_t j = 0;
 
-			for(size_t s = 1; s < v; ++s)
+			for(size_t s = 0; s < v; ++s)
 			{
+				i = s % (2*r+4);
+				j = s % c;
 				A = S[i] = r_l( S[i] + A + B, 3 );
-				B = L[j] = r_l( L[j] + A + B, (A + B)%(8*sizeof(word_t)));
-				i = (i + 1) % (2*r+4);
-				j = (j + 1) % c;
+				B = L[j] = r_l( L[j] + A + B, (A + B)%(word_size));
 			}
 		}
 
@@ -116,17 +117,17 @@ public:
 		B += S[0];
 		D += S[1];
 
-		for(size_t i = 1; i < r; ++i)
+		for(size_t i = 1; i <= r; ++i)
 		{
 			auto t = r_l( B * ( 2 * B + 1 ), 5);
 			auto u = r_l( D * ( 2 * D + 1 ), 5);
-			A = p">((A ^ t) << u%(8*sizeof(word_t))) + S[2*i];
-			C = p">((C ^ u) << t%(8*sizeof(word_t))) + S[2*i+1];
-			n">std::rotate(plaintext.begin(), plaintext.begin()+1, plaintext.end());
+			A = n">r_l((A ^ t), u % word_size) + S[ 2 * i ];
+			C = n">r_l((C ^ u), t % word_size) + S[ 2 * i + 1 ];
+			k">auto tmp = A; A = B; B = C; C = D; D = tmp;
 		}
 
-		A += S[2*r+3];
-		C += S[2*r+2];
+		A += S[2*r+2];
+		C += S[2*r+3];
 	}
 
 };

tests.cpp

@@ -4,6 +4,7 @@
 #include "rc6_generic.cpp"
 #include "gp_test.cpp"
 #include "bloomfilter.cpp"
+#include "quotient_filter.cpp"
 #include <iostream>
 
 int main()

tests/quotient_filter.cpp

@@ -0,0 +1,127 @@
+#include "test_scaffold.h"
+#include <random>
+#include <string>
+
+#include "gp/quotient_filter.hpp"
+
+typedef std::linear_congruential_engine<uint_fast64_t, 128273, 13, 2147483647> cheap_rand;
+typedef std::linear_congruential_engine<uint_fast64_t, 69857, 87541, 2147483647> cheap_rand_bis;
+
+struct qfilter_test : public test_scaffold {
+	uint32_t seed;
+
+	qfilter_test() {
+		seed = std::random_device{}();
+		name = __FILE__ ":1_seed";
+		name += std::to_string(seed);
+	}
+
+	virtual int run() {
+		
+		cheap_rand setter(seed);
+		cheap_rand getter(seed);
+
+		gp::quotient_filter test_filter;
+
+		for(int a = 0 ; a < 100; a++)
+		{
+			test_filter.set_hash(setter());
+		}
+
+		bool result = true;
+		for(int a = 0 ; a < 100; a++)
+		{
+			result *= test_filter.test_hash(getter());
+		}
+
+		return !result;
+	}
+};
+
+append_test dummy_rshyr43(new qfilter_test{});
+
+struct qfilter2_test : public test_scaffold {
+	uint32_t seedA;
+	uint32_t seedB;
+
+	qfilter2_test() {
+		seedA = std::random_device{}();
+		seedB = std::random_device{}();
+		name = __FILE__ ":2_seedA";
+		name += std::to_string(seedA);
+		name += "&seedB";
+		name += std::to_string(seedB);
+	}
+
+	virtual int run() {
+		
+		cheap_rand setter(seedA);
+		cheap_rand deleter(seedA);
+		cheap_rand_bis getter(seedB);
+
+		gp::quotient_filter test_filter;
+
+		for(int a = 0 ; a < 100000; a++)
+		{
+			test_filter.set_hash(setter());
+		}
+
+		int interference = 0;
+		for(int a = 0 ; a < 100000; a++)
+		{
+			interference += test_filter.test_hash(getter());
+		}
+
+		for(int a = 0 ; a < 100000; a++)
+		{
+			test_filter.remove_hash(deleter());
+		}
+
+		return interference > 25;
+	}
+};
+
+append_test dummy_kegdflu3(new qfilter2_test{});
+
+struct qfilter3_test : public test_scaffold {
+	uint32_t seedA;
+	uint32_t seedB;
+
+	qfilter3_test() {
+		seedA = std::random_device{}();
+		seedB = std::random_device{}();
+		name = __FILE__ ":3_seedA";
+		name += std::to_string(seedA);
+		name += "&seedB";
+		name += std::to_string(seedB);
+	}
+
+	virtual int run() {
+		
+		cheap_rand setter(seedA);
+		cheap_rand deleter(seedA);
+		cheap_rand_bis getter(seedB);
+
+		gp::quotient_filter<uint32_t, 20,12, gp::stepcache_skipstone> test_filter;
+
+		for(int a = 0 ; a < 100000; a++)
+		{
+			test_filter.set_hash(setter());
+		}
+
+		int interference = 0;
+		for(int a = 0 ; a < 100000; a++)
+		{
+			interference += test_filter.test_hash(getter());
+		}
+
+		for(int a = 0 ; a < 100000; a++)
+		{
+			test_filter.remove_hash(deleter());
+		}
+
+		return interference > 25;
+	}
+};
+
+append_test dummy_kjdflu3(new qfilter3_test{});

tests/rc6_generic.cpp

@@ -9,7 +9,7 @@ struct RC6test : public test_scaffold {
 	}
 
 	virtual int run() {
-		using rc = RC6<kt">uint32_t, 20, 128>;
+		using rc = RC6<>;
 
 		rc::key_type key = {0,0,0,0};
 		rc::block_type plaintext = {0,0,0,0};