SmallTests/digest.cr

require "openssl"
require "benchmark"
require "digest"
require "big"

def to_hex(decimal)
    decimal_remainder = decimal % 16
    hexchars = "0123456789abcdef"
    div = decimal / 16
    if div == 0
      hexchars[decimal_remainder].to_s
    else
      to_hex(div) + hexchars[decimal_remainder].to_s
    end
  end

class CRC

    # The initial value of the CRC checksum
    INIT_CRC = 0x00

    # The XOR mask to apply to the resulting CRC checksum
    XOR_MASK = 0x00

    # The bit width of the CRC checksum
    WIDTH = 0

    # Default place holder CRC table
    TABLE = ([] of UInt64).freeze

    #
    # Calculates the CRC checksum.
    #
    # @param [String] data
    #   The given data.
    #
    # @return [Integer]
    #   The CRC checksum.
    #
    def self.checksum(data)
      crc = self.new
      crc << data

      return crc.checksum
    end

    #
    # Packs the given CRC checksum.
    #
    # @return [String]
    #   The packed CRC checksum.
    #
    def self.pack(crc)
      '\0'
    end

    #
    # Initializes the CRC checksum.
    #
    def initialize
    end

    #
    # The input block length.
    #
    # @return [1]
    #
    def block_length
      1
    end

    #
    # The length of the digest.
    #
    # @return [Integer]
    #   The length in bytes.
    #
    def digest_length
      (@width / 8.0).ceil
    end

    #
    # Updates the CRC checksum with the given data.
    #
    # @param [String] data
    #   The data to update the CRC checksum with.
    #
    def update(data)
    end

    #
    # @see {#update}
    #
    def <<(data)
      update(data)
      return self
    end

    #
    # Resets the CRC checksum.
    #
    # @return [Integer]
    #   The default value of the CRC checksum.
    #
    def reset
      @crc = @@INIT_CRC
    end

    #
    # The resulting CRC checksum.
    #
    # @return [Integer]
    #   The resulting CRC checksum.
    #
    def checksum
      @crc ^ @@XOR_MASK
    end

    #
    # Finishes the CRC checksum calculation.
    #
    # @see {pack}
    #
    def finish
      self.class.pack(checksum)
    end

end

class CRC64 < CRC

    @@WIDTH = 64_u64

    @@INIT_CRC : UInt64 = 0x0000000000000000_u64

    @@XOR_MASK : UInt64 = 0x0000000000000000_u64

    # Generated by `./pycrc.py --algorithm=table-driven --model=crc-64 --generate=c`
    @@TABLE : Array(UInt64) = [
      0x0000000000000000_u64, 0x01b0000000000000_u64, 0x0360000000000000_u64, 0x02d0000000000000_u64,
      0x06c0000000000000_u64, 0x0770000000000000_u64, 0x05a0000000000000_u64, 0x0410000000000000_u64,
      0x0d80000000000000_u64, 0x0c30000000000000_u64, 0x0ee0000000000000_u64, 0x0f50000000000000_u64,
      0x0b40000000000000_u64, 0x0af0000000000000_u64, 0x0820000000000000_u64, 0x0990000000000000_u64,
      0x1b00000000000000_u64, 0x1ab0000000000000_u64, 0x1860000000000000_u64, 0x19d0000000000000_u64,
      0x1dc0000000000000_u64, 0x1c70000000000000_u64, 0x1ea0000000000000_u64, 0x1f10000000000000_u64,
      0x1680000000000000_u64, 0x1730000000000000_u64, 0x15e0000000000000_u64, 0x1450000000000000_u64,
      0x1040000000000000_u64, 0x11f0000000000000_u64, 0x1320000000000000_u64, 0x1290000000000000_u64,
      0x3600000000000000_u64, 0x37b0000000000000_u64, 0x3560000000000000_u64, 0x34d0000000000000_u64,
      0x30c0000000000000_u64, 0x3170000000000000_u64, 0x33a0000000000000_u64, 0x3210000000000000_u64,
      0x3b80000000000000_u64, 0x3a30000000000000_u64, 0x38e0000000000000_u64, 0x3950000000000000_u64,
      0x3d40000000000000_u64, 0x3cf0000000000000_u64, 0x3e20000000000000_u64, 0x3f90000000000000_u64,
      0x2d00000000000000_u64, 0x2cb0000000000000_u64, 0x2e60000000000000_u64, 0x2fd0000000000000_u64,
      0x2bc0000000000000_u64, 0x2a70000000000000_u64, 0x28a0000000000000_u64, 0x2910000000000000_u64,
      0x2080000000000000_u64, 0x2130000000000000_u64, 0x23e0000000000000_u64, 0x2250000000000000_u64,
      0x2640000000000000_u64, 0x27f0000000000000_u64, 0x2520000000000000_u64, 0x2490000000000000_u64,
      0x6c00000000000000_u64, 0x6db0000000000000_u64, 0x6f60000000000000_u64, 0x6ed0000000000000_u64,
      0x6ac0000000000000_u64, 0x6b70000000000000_u64, 0x69a0000000000000_u64, 0x6810000000000000_u64,
      0x6180000000000000_u64, 0x6030000000000000_u64, 0x62e0000000000000_u64, 0x6350000000000000_u64,
      0x6740000000000000_u64, 0x66f0000000000000_u64, 0x6420000000000000_u64, 0x6590000000000000_u64,
      0x7700000000000000_u64, 0x76b0000000000000_u64, 0x7460000000000000_u64, 0x75d0000000000000_u64,
      0x71c0000000000000_u64, 0x7070000000000000_u64, 0x72a0000000000000_u64, 0x7310000000000000_u64,
      0x7a80000000000000_u64, 0x7b30000000000000_u64, 0x79e0000000000000_u64, 0x7850000000000000_u64,
      0x7c40000000000000_u64, 0x7df0000000000000_u64, 0x7f20000000000000_u64, 0x7e90000000000000_u64,
      0x5a00000000000000_u64, 0x5bb0000000000000_u64, 0x5960000000000000_u64, 0x58d0000000000000_u64,
      0x5cc0000000000000_u64, 0x5d70000000000000_u64, 0x5fa0000000000000_u64, 0x5e10000000000000_u64,
      0x5780000000000000_u64, 0x5630000000000000_u64, 0x54e0000000000000_u64, 0x5550000000000000_u64,
      0x5140000000000000_u64, 0x50f0000000000000_u64, 0x5220000000000000_u64, 0x5390000000000000_u64,
      0x4100000000000000_u64, 0x40b0000000000000_u64, 0x4260000000000000_u64, 0x43d0000000000000_u64,
      0x47c0000000000000_u64, 0x4670000000000000_u64, 0x44a0000000000000_u64, 0x4510000000000000_u64,
      0x4c80000000000000_u64, 0x4d30000000000000_u64, 0x4fe0000000000000_u64, 0x4e50000000000000_u64,
      0x4a40000000000000_u64, 0x4bf0000000000000_u64, 0x4920000000000000_u64, 0x4890000000000000_u64,
      0xd800000000000000_u64, 0xd9b0000000000000_u64, 0xdb60000000000000_u64, 0xdad0000000000000_u64,
      0xdec0000000000000_u64, 0xdf70000000000000_u64, 0xdda0000000000000_u64, 0xdc10000000000000_u64,
      0xd580000000000000_u64, 0xd430000000000000_u64, 0xd6e0000000000000_u64, 0xd750000000000000_u64,
      0xd340000000000000_u64, 0xd2f0000000000000_u64, 0xd020000000000000_u64, 0xd190000000000000_u64,
      0xc300000000000000_u64, 0xc2b0000000000000_u64, 0xc060000000000000_u64, 0xc1d0000000000000_u64,
      0xc5c0000000000000_u64, 0xc470000000000000_u64, 0xc6a0000000000000_u64, 0xc710000000000000_u64,
      0xce80000000000000_u64, 0xcf30000000000000_u64, 0xcde0000000000000_u64, 0xcc50000000000000_u64,
      0xc840000000000000_u64, 0xc9f0000000000000_u64, 0xcb20000000000000_u64, 0xca90000000000000_u64,
      0xee00000000000000_u64, 0xefb0000000000000_u64, 0xed60000000000000_u64, 0xecd0000000000000_u64,
      0xe8c0000000000000_u64, 0xe970000000000000_u64, 0xeba0000000000000_u64, 0xea10000000000000_u64,
      0xe380000000000000_u64, 0xe230000000000000_u64, 0xe0e0000000000000_u64, 0xe150000000000000_u64,
      0xe540000000000000_u64, 0xe4f0000000000000_u64, 0xe620000000000000_u64, 0xe790000000000000_u64,
      0xf500000000000000_u64, 0xf4b0000000000000_u64, 0xf660000000000000_u64, 0xf7d0000000000000_u64,
      0xf3c0000000000000_u64, 0xf270000000000000_u64, 0xf0a0000000000000_u64, 0xf110000000000000_u64,
      0xf880000000000000_u64, 0xf930000000000000_u64, 0xfbe0000000000000_u64, 0xfa50000000000000_u64,
      0xfe40000000000000_u64, 0xfff0000000000000_u64, 0xfd20000000000000_u64, 0xfc90000000000000_u64,
      0xb400000000000000_u64, 0xb5b0000000000000_u64, 0xb760000000000000_u64, 0xb6d0000000000000_u64,
      0xb2c0000000000000_u64, 0xb370000000000000_u64, 0xb1a0000000000000_u64, 0xb010000000000000_u64,
      0xb980000000000000_u64, 0xb830000000000000_u64, 0xbae0000000000000_u64, 0xbb50000000000000_u64,
      0xbf40000000000000_u64, 0xbef0000000000000_u64, 0xbc20000000000000_u64, 0xbd90000000000000_u64,
      0xaf00000000000000_u64, 0xaeb0000000000000_u64, 0xac60000000000000_u64, 0xadd0000000000000_u64,
      0xa9c0000000000000_u64, 0xa870000000000000_u64, 0xaaa0000000000000_u64, 0xab10000000000000_u64,
      0xa280000000000000_u64, 0xa330000000000000_u64, 0xa1e0000000000000_u64, 0xa050000000000000_u64,
      0xa440000000000000_u64, 0xa5f0000000000000_u64, 0xa720000000000000_u64, 0xa690000000000000_u64,
      0x8200000000000000_u64, 0x83b0000000000000_u64, 0x8160000000000000_u64, 0x80d0000000000000_u64,
      0x84c0000000000000_u64, 0x8570000000000000_u64, 0x87a0000000000000_u64, 0x8610000000000000_u64,
      0x8f80000000000000_u64, 0x8e30000000000000_u64, 0x8ce0000000000000_u64, 0x8d50000000000000_u64,
      0x8940000000000000_u64, 0x88f0000000000000_u64, 0x8a20000000000000_u64, 0x8b90000000000000_u64,
      0x9900000000000000_u64, 0x98b0000000000000_u64, 0x9a60000000000000_u64, 0x9bd0000000000000_u64,
      0x9fc0000000000000_u64, 0x9e70000000000000_u64, 0x9ca0000000000000_u64, 0x9d10000000000000_u64,
      0x9480000000000000_u64, 0x9530000000000000_u64, 0x97e0000000000000_u64, 0x9650000000000000_u64,
      0x9240000000000000_u64, 0x93f0000000000000_u64, 0x9120000000000000_u64, 0x9090000000000000_u64
    ]
 
    @crc : UInt64     

    def initialize
        @crc        = @@INIT_CRC
  
        reset
    end

    #
    # Packs the CRC64 checksum.
    #
    # @param [Integer] crc
    #   The checksum to pack.
    #
    # @return [String]
    #   The packed checksum.
    #
    def self.pack(crc)
      buffer = ""

      buffer << ((crc & 0xff00000000000000) >> 56).chr
      buffer << ((crc & 0xff000000000000) >> 48).chr
      buffer << ((crc & 0xff0000000000) >> 40).chr
      buffer << ((crc & 0xff00000000) >> 32).chr
      buffer << ((crc & 0xff000000) >> 24).chr
      buffer << ((crc & 0xff0000) >> 16).chr
      buffer << ((crc & 0xff00) >> 8).chr
      buffer << (crc & 0xff).chr

      buffer
    end

    #
    # Updates the CRC64 checksum.
    #
    # @param [String] data
    #   The data to update the checksum with.
    #
    def update(data)
      data.each_byte do |b|
        @crc = ((@@TABLE[(@crc ^ b) & 0xff] ^ (@crc >> 8)) & 0xffffffffffffffff)
      end

      return self
    end

    def digest
        v = @crc
        ptr = pointerof(v).as(UInt8*)
        slice = Slice(UInt8).new(ptr,8)
        return IO::ByteFormat::BigEndian.decode(UInt64, slice)
    end

end



def hash_ripemd160(data : String)
    ctx = OpenSSL::Digest.new("ripemd160")
    ctx << data
    digest = ctx.digest
    tag = IO::ByteFormat::BigEndian.decode(UInt64, digest)
    tag ^= IO::ByteFormat::BigEndian.decode(UInt64, digest+8)
    return tag
end


def hash_sha1(data : String)
    ctx = OpenSSL::Digest.new("sha1")
    ctx << data
    digest = ctx.digest
    tag = IO::ByteFormat::BigEndian.decode(UInt64, digest)
    tag ^= IO::ByteFormat::BigEndian.decode(UInt64, digest+8)
    return tag
end
def hash_sha224(data : String)
    ctx = OpenSSL::Digest.new("sha224")
    ctx << data
    digest = ctx.digest
    tag = IO::ByteFormat::BigEndian.decode(UInt64, digest)
    tag ^= IO::ByteFormat::BigEndian.decode(UInt64, digest+8)
    return tag
end
def hash_sha256(data : String)
    ctx = OpenSSL::Digest.new("sha256")
    ctx << data
    digest = ctx.digest
    tag = IO::ByteFormat::BigEndian.decode(UInt64, digest)
    tag ^= IO::ByteFormat::BigEndian.decode(UInt64, digest+8)
    return tag
end
def hash_sha384(data : String)
    ctx = OpenSSL::Digest.new("sha384")
    ctx << data
    digest = ctx.digest
    tag = IO::ByteFormat::BigEndian.decode(UInt64, digest)
    tag ^= IO::ByteFormat::BigEndian.decode(UInt64, digest+8)
    return tag
end
def hash_sha512(data : String)
    ctx = OpenSSL::Digest.new("sha512")
    ctx << data
    digest = ctx.digest
    tag = IO::ByteFormat::BigEndian.decode(UInt64, digest)
    tag ^= IO::ByteFormat::BigEndian.decode(UInt64, digest+8)
    return tag
end

def hash_md4(data : String)
    ctx = OpenSSL::Digest.new("md4")
    ctx << data
    digest = ctx.digest
    tag = IO::ByteFormat::BigEndian.decode(UInt64, digest)
    tag ^= IO::ByteFormat::BigEndian.decode(UInt64, digest+8)
    return tag
end
def hash_md5(data : String)
    ctx = OpenSSL::Digest.new("md5")
    ctx << data
    digest = ctx.digest
    tag = IO::ByteFormat::BigEndian.decode(UInt64, digest)
    tag ^= IO::ByteFormat::BigEndian.decode(UInt64, digest+8)
    return tag
end
def hash_crc64(data : String)
    ctx = CRC64.new
    ctx << data
    return ctx.digest
end

if to_hex(hash_crc64("azertyuiopqsdfghjklmwxcvbn123456"))!="c9292a94ae08e4b2"
    puts "CRITICAL CRC64 ERROR : got "+to_hex(hash_crc64("azertyuiopqsdfghjklmwxcvbn123456"))+"\n"
end

store = Hash(UInt64,Int32).new(10000000)
s3 = Set(UInt64).new
(5000000000_u64..5020000000_u64).each do |n|
    h = hash_crc64(to_hex(n))
    store[h] = store.fetch(h,0)+1
end

store.each do |k,v| 
    if v>1
        puts "Got #{v} duplicates for key #{to_hex(k)}"
    end
    s3 << k
end
sum = BigRational.new(0)
store.each do |k,v|
    sum+=k
end

mean = sum/store.size


sumalt = BigRational.new(0)
store.each do |k,v|
    sumalt+=((-mean+k)*(-mean+k))
end
stddev=Math.sqrt(sumalt/store.size)

puts "Mean of CRCs: "+mean.to_f.to_s
puts "StdDev of CRCs: "+stddev.to_f.to_s




puts "32 bytes key ->"

Benchmark.ips do |x|
    x.report("\tCRC64 32 bytes") { hash_crc64("azertyuiopqsdfghjklmwxcvbn123456") }
    x.report("\tripemd160 32 bytes") { hash_ripemd160("azertyuiopqsdfghjklmwxcvbn123456") }
    x.report("\tMD4 32 bytes") { hash_md4("azertyuiopqsdfghjklmwxcvbn123456") }
    x.report("\tMD5 32 bytes") { hash_md5("azertyuiopqsdfghjklmwxcvbn123456") }
    x.report("\tSHA1 32 bytes") { hash_sha1("azertyuiopqsdfghjklmwxcvbn123456") }
    x.report("\tSHA224 32 bytes") { hash_sha224("azertyuiopqsdfghjklmwxcvbn123456") }
    x.report("\tSHA256 32 bytes") { hash_sha256("azertyuiopqsdfghjklmwxcvbn123456") }
    x.report("\tSHA384 32 bytes") { hash_sha384("azertyuiopqsdfghjklmwxcvbn123456") }
    x.report("\tSHA512 32 bytes") { hash_sha512("azertyuiopqsdfghjklmwxcvbn123456") }
end

puts "128 bytes key ->"

Benchmark.ips do |x|
    str = "azertyuiopqsdfghjklmwxcvbn123456azertyuiopqsdfghjklmwxcvbn123456azertyuiopqsdfghjklmwxcvbn123456azertyuiopqsdfghjklmwxcvbn123456"
    x.report("\tCRC64 128 bytes") { hash_crc64(str) }
    x.report("\tripemd160 128 bytes") { hash_ripemd160(str) }
    x.report("\tMD4 128 bytes") { hash_md4(str) }
    x.report("\tMD5 128 bytes") { hash_md5(str) }
    x.report("\tSHA1 128 bytes") { hash_sha1(str) }
    x.report("\tSHA224 128 bytes") { hash_sha224(str) }
    x.report("\tSHA256 128 bytes") { hash_sha256(str) }
    x.report("\tSHA384 128 bytes") { hash_sha384(str) }
    x.report("\tSHA512 128 bytes") { hash_sha512(str) }
end

puts "1024 bytes key ->"

Benchmark.ips do |x|
    str = "azertyuiopqsdfghjklmwxcvbn123456azertyuiopqsdfghjklmwxcvbn123456azertyuiopqsdfghjklmwxcvbn123456azertyuiopqsdfghjklmwxcvbn123456"*8
    x.report("\tCRC64 1024 bytes") { hash_crc64(str) }
    x.report("\tripemd160 1024 bytes") { hash_ripemd160(str) }
    x.report("\tMD4 1024 bytes") { hash_md4(str) }
    x.report("\tMD5 1024 bytes") { hash_md5(str) }
    x.report("\tSHA1 1024 bytes") { hash_sha1(str) }
    x.report("\tSHA224 1024 bytes") { hash_sha224(str) }
    x.report("\tSHA256 1024 bytes") { hash_sha256(str) }
    x.report("\tSHA384 1024 bytes") { hash_sha384(str) }
    x.report("\tSHA512 1024 bytes") { hash_sha512(str) }
end