You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
416 lines
15 KiB
416 lines
15 KiB
require "openssl"
|
|
require "benchmark"
|
|
require "digest"
|
|
require "big"
|
|
|
|
def to_hex(decimal)
|
|
decimal_remainder = decimal % 16
|
|
hexchars = "0123456789abcdef"
|
|
div = decimal / 16
|
|
if div == 0
|
|
hexchars[decimal_remainder].to_s
|
|
else
|
|
to_hex(div) + hexchars[decimal_remainder].to_s
|
|
end
|
|
end
|
|
|
|
class CRC
|
|
|
|
# The initial value of the CRC checksum
|
|
INIT_CRC = 0x00
|
|
|
|
# The XOR mask to apply to the resulting CRC checksum
|
|
XOR_MASK = 0x00
|
|
|
|
# The bit width of the CRC checksum
|
|
WIDTH = 0
|
|
|
|
# Default place holder CRC table
|
|
TABLE = ([] of UInt64).freeze
|
|
|
|
#
|
|
# Calculates the CRC checksum.
|
|
#
|
|
# @param [String] data
|
|
# The given data.
|
|
#
|
|
# @return [Integer]
|
|
# The CRC checksum.
|
|
#
|
|
def self.checksum(data)
|
|
crc = self.new
|
|
crc << data
|
|
|
|
return crc.checksum
|
|
end
|
|
|
|
#
|
|
# Packs the given CRC checksum.
|
|
#
|
|
# @return [String]
|
|
# The packed CRC checksum.
|
|
#
|
|
def self.pack(crc)
|
|
'\0'
|
|
end
|
|
|
|
#
|
|
# Initializes the CRC checksum.
|
|
#
|
|
def initialize
|
|
end
|
|
|
|
#
|
|
# The input block length.
|
|
#
|
|
# @return [1]
|
|
#
|
|
def block_length
|
|
1
|
|
end
|
|
|
|
#
|
|
# The length of the digest.
|
|
#
|
|
# @return [Integer]
|
|
# The length in bytes.
|
|
#
|
|
def digest_length
|
|
(@width / 8.0).ceil
|
|
end
|
|
|
|
#
|
|
# Updates the CRC checksum with the given data.
|
|
#
|
|
# @param [String] data
|
|
# The data to update the CRC checksum with.
|
|
#
|
|
def update(data)
|
|
end
|
|
|
|
#
|
|
# @see {#update}
|
|
#
|
|
def <<(data)
|
|
update(data)
|
|
return self
|
|
end
|
|
|
|
#
|
|
# Resets the CRC checksum.
|
|
#
|
|
# @return [Integer]
|
|
# The default value of the CRC checksum.
|
|
#
|
|
def reset
|
|
@crc = @@INIT_CRC
|
|
end
|
|
|
|
#
|
|
# The resulting CRC checksum.
|
|
#
|
|
# @return [Integer]
|
|
# The resulting CRC checksum.
|
|
#
|
|
def checksum
|
|
@crc ^ @@XOR_MASK
|
|
end
|
|
|
|
#
|
|
# Finishes the CRC checksum calculation.
|
|
#
|
|
# @see {pack}
|
|
#
|
|
def finish
|
|
self.class.pack(checksum)
|
|
end
|
|
|
|
end
|
|
|
|
class CRC64 < CRC
|
|
|
|
@@WIDTH = 64_u64
|
|
|
|
@@INIT_CRC : UInt64 = 0x0000000000000000_u64
|
|
|
|
@@XOR_MASK : UInt64 = 0x0000000000000000_u64
|
|
|
|
# Generated by `./pycrc.py --algorithm=table-driven --model=crc-64 --generate=c`
|
|
@@TABLE : Array(UInt64) = [
|
|
0x0000000000000000_u64, 0x01b0000000000000_u64, 0x0360000000000000_u64, 0x02d0000000000000_u64,
|
|
0x06c0000000000000_u64, 0x0770000000000000_u64, 0x05a0000000000000_u64, 0x0410000000000000_u64,
|
|
0x0d80000000000000_u64, 0x0c30000000000000_u64, 0x0ee0000000000000_u64, 0x0f50000000000000_u64,
|
|
0x0b40000000000000_u64, 0x0af0000000000000_u64, 0x0820000000000000_u64, 0x0990000000000000_u64,
|
|
0x1b00000000000000_u64, 0x1ab0000000000000_u64, 0x1860000000000000_u64, 0x19d0000000000000_u64,
|
|
0x1dc0000000000000_u64, 0x1c70000000000000_u64, 0x1ea0000000000000_u64, 0x1f10000000000000_u64,
|
|
0x1680000000000000_u64, 0x1730000000000000_u64, 0x15e0000000000000_u64, 0x1450000000000000_u64,
|
|
0x1040000000000000_u64, 0x11f0000000000000_u64, 0x1320000000000000_u64, 0x1290000000000000_u64,
|
|
0x3600000000000000_u64, 0x37b0000000000000_u64, 0x3560000000000000_u64, 0x34d0000000000000_u64,
|
|
0x30c0000000000000_u64, 0x3170000000000000_u64, 0x33a0000000000000_u64, 0x3210000000000000_u64,
|
|
0x3b80000000000000_u64, 0x3a30000000000000_u64, 0x38e0000000000000_u64, 0x3950000000000000_u64,
|
|
0x3d40000000000000_u64, 0x3cf0000000000000_u64, 0x3e20000000000000_u64, 0x3f90000000000000_u64,
|
|
0x2d00000000000000_u64, 0x2cb0000000000000_u64, 0x2e60000000000000_u64, 0x2fd0000000000000_u64,
|
|
0x2bc0000000000000_u64, 0x2a70000000000000_u64, 0x28a0000000000000_u64, 0x2910000000000000_u64,
|
|
0x2080000000000000_u64, 0x2130000000000000_u64, 0x23e0000000000000_u64, 0x2250000000000000_u64,
|
|
0x2640000000000000_u64, 0x27f0000000000000_u64, 0x2520000000000000_u64, 0x2490000000000000_u64,
|
|
0x6c00000000000000_u64, 0x6db0000000000000_u64, 0x6f60000000000000_u64, 0x6ed0000000000000_u64,
|
|
0x6ac0000000000000_u64, 0x6b70000000000000_u64, 0x69a0000000000000_u64, 0x6810000000000000_u64,
|
|
0x6180000000000000_u64, 0x6030000000000000_u64, 0x62e0000000000000_u64, 0x6350000000000000_u64,
|
|
0x6740000000000000_u64, 0x66f0000000000000_u64, 0x6420000000000000_u64, 0x6590000000000000_u64,
|
|
0x7700000000000000_u64, 0x76b0000000000000_u64, 0x7460000000000000_u64, 0x75d0000000000000_u64,
|
|
0x71c0000000000000_u64, 0x7070000000000000_u64, 0x72a0000000000000_u64, 0x7310000000000000_u64,
|
|
0x7a80000000000000_u64, 0x7b30000000000000_u64, 0x79e0000000000000_u64, 0x7850000000000000_u64,
|
|
0x7c40000000000000_u64, 0x7df0000000000000_u64, 0x7f20000000000000_u64, 0x7e90000000000000_u64,
|
|
0x5a00000000000000_u64, 0x5bb0000000000000_u64, 0x5960000000000000_u64, 0x58d0000000000000_u64,
|
|
0x5cc0000000000000_u64, 0x5d70000000000000_u64, 0x5fa0000000000000_u64, 0x5e10000000000000_u64,
|
|
0x5780000000000000_u64, 0x5630000000000000_u64, 0x54e0000000000000_u64, 0x5550000000000000_u64,
|
|
0x5140000000000000_u64, 0x50f0000000000000_u64, 0x5220000000000000_u64, 0x5390000000000000_u64,
|
|
0x4100000000000000_u64, 0x40b0000000000000_u64, 0x4260000000000000_u64, 0x43d0000000000000_u64,
|
|
0x47c0000000000000_u64, 0x4670000000000000_u64, 0x44a0000000000000_u64, 0x4510000000000000_u64,
|
|
0x4c80000000000000_u64, 0x4d30000000000000_u64, 0x4fe0000000000000_u64, 0x4e50000000000000_u64,
|
|
0x4a40000000000000_u64, 0x4bf0000000000000_u64, 0x4920000000000000_u64, 0x4890000000000000_u64,
|
|
0xd800000000000000_u64, 0xd9b0000000000000_u64, 0xdb60000000000000_u64, 0xdad0000000000000_u64,
|
|
0xdec0000000000000_u64, 0xdf70000000000000_u64, 0xdda0000000000000_u64, 0xdc10000000000000_u64,
|
|
0xd580000000000000_u64, 0xd430000000000000_u64, 0xd6e0000000000000_u64, 0xd750000000000000_u64,
|
|
0xd340000000000000_u64, 0xd2f0000000000000_u64, 0xd020000000000000_u64, 0xd190000000000000_u64,
|
|
0xc300000000000000_u64, 0xc2b0000000000000_u64, 0xc060000000000000_u64, 0xc1d0000000000000_u64,
|
|
0xc5c0000000000000_u64, 0xc470000000000000_u64, 0xc6a0000000000000_u64, 0xc710000000000000_u64,
|
|
0xce80000000000000_u64, 0xcf30000000000000_u64, 0xcde0000000000000_u64, 0xcc50000000000000_u64,
|
|
0xc840000000000000_u64, 0xc9f0000000000000_u64, 0xcb20000000000000_u64, 0xca90000000000000_u64,
|
|
0xee00000000000000_u64, 0xefb0000000000000_u64, 0xed60000000000000_u64, 0xecd0000000000000_u64,
|
|
0xe8c0000000000000_u64, 0xe970000000000000_u64, 0xeba0000000000000_u64, 0xea10000000000000_u64,
|
|
0xe380000000000000_u64, 0xe230000000000000_u64, 0xe0e0000000000000_u64, 0xe150000000000000_u64,
|
|
0xe540000000000000_u64, 0xe4f0000000000000_u64, 0xe620000000000000_u64, 0xe790000000000000_u64,
|
|
0xf500000000000000_u64, 0xf4b0000000000000_u64, 0xf660000000000000_u64, 0xf7d0000000000000_u64,
|
|
0xf3c0000000000000_u64, 0xf270000000000000_u64, 0xf0a0000000000000_u64, 0xf110000000000000_u64,
|
|
0xf880000000000000_u64, 0xf930000000000000_u64, 0xfbe0000000000000_u64, 0xfa50000000000000_u64,
|
|
0xfe40000000000000_u64, 0xfff0000000000000_u64, 0xfd20000000000000_u64, 0xfc90000000000000_u64,
|
|
0xb400000000000000_u64, 0xb5b0000000000000_u64, 0xb760000000000000_u64, 0xb6d0000000000000_u64,
|
|
0xb2c0000000000000_u64, 0xb370000000000000_u64, 0xb1a0000000000000_u64, 0xb010000000000000_u64,
|
|
0xb980000000000000_u64, 0xb830000000000000_u64, 0xbae0000000000000_u64, 0xbb50000000000000_u64,
|
|
0xbf40000000000000_u64, 0xbef0000000000000_u64, 0xbc20000000000000_u64, 0xbd90000000000000_u64,
|
|
0xaf00000000000000_u64, 0xaeb0000000000000_u64, 0xac60000000000000_u64, 0xadd0000000000000_u64,
|
|
0xa9c0000000000000_u64, 0xa870000000000000_u64, 0xaaa0000000000000_u64, 0xab10000000000000_u64,
|
|
0xa280000000000000_u64, 0xa330000000000000_u64, 0xa1e0000000000000_u64, 0xa050000000000000_u64,
|
|
0xa440000000000000_u64, 0xa5f0000000000000_u64, 0xa720000000000000_u64, 0xa690000000000000_u64,
|
|
0x8200000000000000_u64, 0x83b0000000000000_u64, 0x8160000000000000_u64, 0x80d0000000000000_u64,
|
|
0x84c0000000000000_u64, 0x8570000000000000_u64, 0x87a0000000000000_u64, 0x8610000000000000_u64,
|
|
0x8f80000000000000_u64, 0x8e30000000000000_u64, 0x8ce0000000000000_u64, 0x8d50000000000000_u64,
|
|
0x8940000000000000_u64, 0x88f0000000000000_u64, 0x8a20000000000000_u64, 0x8b90000000000000_u64,
|
|
0x9900000000000000_u64, 0x98b0000000000000_u64, 0x9a60000000000000_u64, 0x9bd0000000000000_u64,
|
|
0x9fc0000000000000_u64, 0x9e70000000000000_u64, 0x9ca0000000000000_u64, 0x9d10000000000000_u64,
|
|
0x9480000000000000_u64, 0x9530000000000000_u64, 0x97e0000000000000_u64, 0x9650000000000000_u64,
|
|
0x9240000000000000_u64, 0x93f0000000000000_u64, 0x9120000000000000_u64, 0x9090000000000000_u64
|
|
]
|
|
|
|
@crc : UInt64
|
|
|
|
def initialize
|
|
@crc = @@INIT_CRC
|
|
|
|
reset
|
|
end
|
|
|
|
#
|
|
# Packs the CRC64 checksum.
|
|
#
|
|
# @param [Integer] crc
|
|
# The checksum to pack.
|
|
#
|
|
# @return [String]
|
|
# The packed checksum.
|
|
#
|
|
def self.pack(crc)
|
|
buffer = ""
|
|
|
|
buffer << ((crc & 0xff00000000000000) >> 56).chr
|
|
buffer << ((crc & 0xff000000000000) >> 48).chr
|
|
buffer << ((crc & 0xff0000000000) >> 40).chr
|
|
buffer << ((crc & 0xff00000000) >> 32).chr
|
|
buffer << ((crc & 0xff000000) >> 24).chr
|
|
buffer << ((crc & 0xff0000) >> 16).chr
|
|
buffer << ((crc & 0xff00) >> 8).chr
|
|
buffer << (crc & 0xff).chr
|
|
|
|
buffer
|
|
end
|
|
|
|
#
|
|
# Updates the CRC64 checksum.
|
|
#
|
|
# @param [String] data
|
|
# The data to update the checksum with.
|
|
#
|
|
def update(data)
|
|
data.each_byte do |b|
|
|
@crc = ((@@TABLE[(@crc ^ b) & 0xff] ^ (@crc >> 8)) & 0xffffffffffffffff)
|
|
end
|
|
|
|
return self
|
|
end
|
|
|
|
def digest
|
|
v = @crc
|
|
ptr = pointerof(v).as(UInt8*)
|
|
slice = Slice(UInt8).new(ptr,8)
|
|
return IO::ByteFormat::BigEndian.decode(UInt64, slice)
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
def hash_ripemd160(data : String)
|
|
ctx = OpenSSL::Digest.new("ripemd160")
|
|
ctx << data
|
|
digest = ctx.digest
|
|
tag = IO::ByteFormat::BigEndian.decode(UInt64, digest)
|
|
tag ^= IO::ByteFormat::BigEndian.decode(UInt64, digest+8)
|
|
return tag
|
|
end
|
|
|
|
|
|
def hash_sha1(data : String)
|
|
ctx = OpenSSL::Digest.new("sha1")
|
|
ctx << data
|
|
digest = ctx.digest
|
|
tag = IO::ByteFormat::BigEndian.decode(UInt64, digest)
|
|
tag ^= IO::ByteFormat::BigEndian.decode(UInt64, digest+8)
|
|
return tag
|
|
end
|
|
def hash_sha224(data : String)
|
|
ctx = OpenSSL::Digest.new("sha224")
|
|
ctx << data
|
|
digest = ctx.digest
|
|
tag = IO::ByteFormat::BigEndian.decode(UInt64, digest)
|
|
tag ^= IO::ByteFormat::BigEndian.decode(UInt64, digest+8)
|
|
return tag
|
|
end
|
|
def hash_sha256(data : String)
|
|
ctx = OpenSSL::Digest.new("sha256")
|
|
ctx << data
|
|
digest = ctx.digest
|
|
tag = IO::ByteFormat::BigEndian.decode(UInt64, digest)
|
|
tag ^= IO::ByteFormat::BigEndian.decode(UInt64, digest+8)
|
|
return tag
|
|
end
|
|
def hash_sha384(data : String)
|
|
ctx = OpenSSL::Digest.new("sha384")
|
|
ctx << data
|
|
digest = ctx.digest
|
|
tag = IO::ByteFormat::BigEndian.decode(UInt64, digest)
|
|
tag ^= IO::ByteFormat::BigEndian.decode(UInt64, digest+8)
|
|
return tag
|
|
end
|
|
def hash_sha512(data : String)
|
|
ctx = OpenSSL::Digest.new("sha512")
|
|
ctx << data
|
|
digest = ctx.digest
|
|
tag = IO::ByteFormat::BigEndian.decode(UInt64, digest)
|
|
tag ^= IO::ByteFormat::BigEndian.decode(UInt64, digest+8)
|
|
return tag
|
|
end
|
|
|
|
def hash_md4(data : String)
|
|
ctx = OpenSSL::Digest.new("md4")
|
|
ctx << data
|
|
digest = ctx.digest
|
|
tag = IO::ByteFormat::BigEndian.decode(UInt64, digest)
|
|
tag ^= IO::ByteFormat::BigEndian.decode(UInt64, digest+8)
|
|
return tag
|
|
end
|
|
def hash_md5(data : String)
|
|
ctx = OpenSSL::Digest.new("md5")
|
|
ctx << data
|
|
digest = ctx.digest
|
|
tag = IO::ByteFormat::BigEndian.decode(UInt64, digest)
|
|
tag ^= IO::ByteFormat::BigEndian.decode(UInt64, digest+8)
|
|
return tag
|
|
end
|
|
def hash_crc64(data : String)
|
|
ctx = CRC64.new
|
|
ctx << data
|
|
return ctx.digest
|
|
end
|
|
|
|
if to_hex(hash_crc64("azertyuiopqsdfghjklmwxcvbn123456"))!="c9292a94ae08e4b2"
|
|
puts "CRITICAL CRC64 ERROR : got "+to_hex(hash_crc64("azertyuiopqsdfghjklmwxcvbn123456"))+"\n"
|
|
end
|
|
|
|
store = Hash(UInt64,Int32).new(10000000)
|
|
s3 = Set(UInt64).new
|
|
(5000000000_u64..5020000000_u64).each do |n|
|
|
h = hash_crc64(to_hex(n))
|
|
store[h] = store.fetch(h,0)+1
|
|
end
|
|
|
|
store.each do |k,v|
|
|
if v>1
|
|
puts "Got #{v} duplicates for key #{to_hex(k)}"
|
|
end
|
|
s3 << k
|
|
end
|
|
sum = BigRational.new(0)
|
|
store.each do |k,v|
|
|
sum+=k
|
|
end
|
|
|
|
mean = sum/store.size
|
|
|
|
|
|
sumalt = BigRational.new(0)
|
|
store.each do |k,v|
|
|
sumalt+=((-mean+k)*(-mean+k))
|
|
end
|
|
stddev=Math.sqrt(sumalt/store.size)
|
|
|
|
puts "Mean of CRCs: "+mean.to_f.to_s
|
|
puts "StdDev of CRCs: "+stddev.to_f.to_s
|
|
|
|
|
|
|
|
|
|
puts "32 bytes key ->"
|
|
|
|
Benchmark.ips do |x|
|
|
x.report("\tCRC64 32 bytes") { hash_crc64("azertyuiopqsdfghjklmwxcvbn123456") }
|
|
x.report("\tripemd160 32 bytes") { hash_ripemd160("azertyuiopqsdfghjklmwxcvbn123456") }
|
|
x.report("\tMD4 32 bytes") { hash_md4("azertyuiopqsdfghjklmwxcvbn123456") }
|
|
x.report("\tMD5 32 bytes") { hash_md5("azertyuiopqsdfghjklmwxcvbn123456") }
|
|
x.report("\tSHA1 32 bytes") { hash_sha1("azertyuiopqsdfghjklmwxcvbn123456") }
|
|
x.report("\tSHA224 32 bytes") { hash_sha224("azertyuiopqsdfghjklmwxcvbn123456") }
|
|
x.report("\tSHA256 32 bytes") { hash_sha256("azertyuiopqsdfghjklmwxcvbn123456") }
|
|
x.report("\tSHA384 32 bytes") { hash_sha384("azertyuiopqsdfghjklmwxcvbn123456") }
|
|
x.report("\tSHA512 32 bytes") { hash_sha512("azertyuiopqsdfghjklmwxcvbn123456") }
|
|
end
|
|
|
|
puts "128 bytes key ->"
|
|
|
|
Benchmark.ips do |x|
|
|
str = "azertyuiopqsdfghjklmwxcvbn123456azertyuiopqsdfghjklmwxcvbn123456azertyuiopqsdfghjklmwxcvbn123456azertyuiopqsdfghjklmwxcvbn123456"
|
|
x.report("\tCRC64 128 bytes") { hash_crc64(str) }
|
|
x.report("\tripemd160 128 bytes") { hash_ripemd160(str) }
|
|
x.report("\tMD4 128 bytes") { hash_md4(str) }
|
|
x.report("\tMD5 128 bytes") { hash_md5(str) }
|
|
x.report("\tSHA1 128 bytes") { hash_sha1(str) }
|
|
x.report("\tSHA224 128 bytes") { hash_sha224(str) }
|
|
x.report("\tSHA256 128 bytes") { hash_sha256(str) }
|
|
x.report("\tSHA384 128 bytes") { hash_sha384(str) }
|
|
x.report("\tSHA512 128 bytes") { hash_sha512(str) }
|
|
end
|
|
|
|
puts "1024 bytes key ->"
|
|
|
|
Benchmark.ips do |x|
|
|
str = "azertyuiopqsdfghjklmwxcvbn123456azertyuiopqsdfghjklmwxcvbn123456azertyuiopqsdfghjklmwxcvbn123456azertyuiopqsdfghjklmwxcvbn123456"*8
|
|
x.report("\tCRC64 1024 bytes") { hash_crc64(str) }
|
|
x.report("\tripemd160 1024 bytes") { hash_ripemd160(str) }
|
|
x.report("\tMD4 1024 bytes") { hash_md4(str) }
|
|
x.report("\tMD5 1024 bytes") { hash_md5(str) }
|
|
x.report("\tSHA1 1024 bytes") { hash_sha1(str) }
|
|
x.report("\tSHA224 1024 bytes") { hash_sha224(str) }
|
|
x.report("\tSHA256 1024 bytes") { hash_sha256(str) }
|
|
x.report("\tSHA384 1024 bytes") { hash_sha384(str) }
|
|
x.report("\tSHA512 1024 bytes") { hash_sha512(str) }
|
|
end
|
|
|