Add HashCompressor to combine hashes together into a single, fixed-width hash. Fixes #7.

Author: kernelmethod

Manifest.toml

@@ -14,18 +14,18 @@ version = "0.4.0"
 
 [[Arpack_jll]]
 deps = ["Libdl", "OpenBLAS_jll", "Pkg"]
-git-tree-sha1 = "68a90a692ddc0eb72d69a6993ca26e2a923bf195"
+git-tree-sha1 = "e214a9b9bd1b4e1b4f15b22c0994862b66af7ff7"
 uuid = "68821587-b530-5797-8361-c406ea357684"
-version = "3.5.0+2"
+version = "3.5.0+3"
 
 [[Base64]]
 uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 
 [[CompilerSupportLibraries_jll]]
 deps = ["Libdl", "Pkg"]
-git-tree-sha1 = "aa832564f930a7fc9290972526908d01a35aefac"
+git-tree-sha1 = "7c4f882c41faa72118841185afc58a2eb00ef612"
 uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae"
-version = "0.3.0+0"
+version = "0.3.3+0"
 
 [[DataAPI]]
 git-tree-sha1 = "674b67f344687a88310213ddfa8a2b3c76cc4252"
@@ -34,9 +34,9 @@ version = "1.1.0"
 
 [[DataStructures]]
 deps = ["InteractiveUtils", "OrderedCollections"]
-git-tree-sha1 = "5a431d46abf2ef2a4d5d00bd0ae61f651cf854c8"
+git-tree-sha1 = "73eb18320fe3ba58790c8b8f6f89420f0a622773"
 uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
-version = "0.17.10"
+version = "0.17.11"
 
 [[Dates]]
 deps = ["Printf"]
@@ -48,9 +48,9 @@ uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b"
 
 [[Distributions]]
 deps = ["FillArrays", "LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "SpecialFunctions", "Statistics", "StatsBase", "StatsFuns"]
-git-tree-sha1 = "46e716ee1f5d14e64cbf55b7db5621a860e7fc32"
+git-tree-sha1 = "c4ed10355637fcb0725dc6a27060f74df24f13cd"
 uuid = "31c24e10-a181-5473-b8eb-7969acd0382f"
-version = "0.23.0"
+version = "0.23.2"
 
 [[DocStringExtensions]]
 deps = ["LibGit2", "Markdown", "Pkg", "Test"]
@@ -60,9 +60,9 @@ version = "0.8.1"
 
 [[Documenter]]
 deps = ["Base64", "Dates", "DocStringExtensions", "InteractiveUtils", "JSON", "LibGit2", "Logging", "Markdown", "REPL", "Test", "Unicode"]
-git-tree-sha1 = "3bacd94d853a6bccaee1d0104d8b06d29a7506ac"
+git-tree-sha1 = "646ebc3db49889ffeb4c36f89e5d82c6a26295ff"
 uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
-version = "0.24.6"
+version = "0.24.7"
 
 [[FFTW]]
 deps = ["AbstractFFTs", "FFTW_jll", "IntelOpenMP_jll", "Libdl", "LinearAlgebra", "MKL_jll", "Reexport"]
@@ -72,15 +72,15 @@ version = "1.2.0"
 
 [[FFTW_jll]]
 deps = ["Libdl", "Pkg"]
-git-tree-sha1 = "ddb57f4cf125243b4aa4908c94d73a805f3cbf2c"
+git-tree-sha1 = "6c975cd606128d45d1df432fb812d6eb10fee00b"
 uuid = "f5851436-0d7a-5f13-b9de-f02708fd171a"
-version = "3.3.9+4"
+version = "3.3.9+5"
 
 [[FillArrays]]
 deps = ["LinearAlgebra", "Random", "SparseArrays"]
-git-tree-sha1 = "85c6b57e2680fa28d5c8adc798967377646fbf66"
+git-tree-sha1 = "51cc2f9bc4eb9c6c0e81ec2f779d1085583cc956"
 uuid = "1a297f60-69ca-5386-bcde-b61e274b549b"
-version = "0.8.5"
+version = "0.8.7"
 
 [[IntelOpenMP_jll]]
 deps = ["Libdl", "Pkg"]
@@ -99,6 +99,7 @@ uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 version = "0.21.0"
 
 [[LibGit2]]
+deps = ["Printf"]
 uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"
 
 [[Libdl]]
@@ -131,10 +132,10 @@ version = "0.4.3"
 uuid = "a63ad114-7e13-5084-954f-fe012c677804"
 
 [[OpenBLAS_jll]]
-deps = ["Libdl", "Pkg"]
-git-tree-sha1 = "858f107d79a016d9511e34186fe2af11566ba762"
+deps = ["CompilerSupportLibraries_jll", "Libdl", "Pkg"]
+git-tree-sha1 = "2ee3e636e94b9fd95fa8364d5cba2e20dae16609"
 uuid = "4536629a-c528-5b80-bd46-f80d51c5b363"
-version = "0.3.7+7"
+version = "0.3.9+2"
 
 [[OpenSpecFun_jll]]
 deps = ["CompilerSupportLibraries_jll", "Libdl", "Pkg"]
@@ -156,12 +157,12 @@ version = "0.9.12"
 
 [[Parsers]]
 deps = ["Dates", "Test"]
-git-tree-sha1 = "0c16b3179190d3046c073440d94172cfc3bb0553"
+git-tree-sha1 = "75d07cb840c300084634b4991761886d0d762724"
 uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
-version = "0.3.12"
+version = "1.0.1"
 
 [[Pkg]]
-deps = ["Dates", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Test", "UUIDs"]
+deps = ["Dates", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"]
 uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 
 [[Printf]]
@@ -231,9 +232,9 @@ uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 
 [[StatsBase]]
 deps = ["DataAPI", "DataStructures", "LinearAlgebra", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics"]
-git-tree-sha1 = "19bfcb46245f69ff4013b3df3b977a289852c3a1"
+git-tree-sha1 = "a6102b1f364befdb05746f386b67c6b7e3262c45"
 uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
-version = "0.32.2"
+version = "0.33.0"
 
 [[StatsFuns]]
 deps = ["Rmath", "SpecialFunctions"]

Project.toml

@@ -11,6 +11,7 @@ LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a"
 QuadGK = "1fd47b50-473d-5c70-9696-f719f8f3bcdc"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
 SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 

src/LSHBase.jl

@@ -58,8 +58,11 @@ function LSHFunction end
 function lsh_family end
 
 @doc """
-    collision_probability(hashfn::H, sim;
-                          n_hashes::Union{Symbol,Integer}=:auto) where {H <: LSHFunction}
+    collision_probability(
+        hashfn::H,
+        sim;
+        n_hashes::Union{Symbol,Integer}=:auto
+    ) where {H <: LSHFunction}
 
 Compute the probability of hash collision between two inputs with similarity `sim` for an [`LSHFunction`](@ref) of type `H`. This function returns the probability that `n_hashes` hashes simultaneously collide.
 

src/LSHFunctions.jl

@@ -3,10 +3,11 @@ module LSHFunctions
 using Distributions, LinearAlgebra, SparseArrays
 
 #========================
-Common types/utilities used through the LSH module
+Common types/utilities used throughout the module
 ========================#
 
-include("utils.jl")
+include(joinpath("utils", "hash_compression.jl"))
+include(joinpath("utils", "vecops.jl"))
 include("LSHBase.jl")
 include("intervals.jl")
 include("similarities.jl")
@@ -47,6 +48,6 @@ export SimHash, L1Hash, L2Hash, MIPSHash, SignALSH, MinHash,
 
 # Helper / utility functions for LSHFunctions
 export index_hash, query_hash, n_hashes, hashtype, similarity, lsh_family,
-       embedded_similarity, collision_probability, @interval
+       embedded_similarity, collision_probability, @interval, HashCompressor
 
 end # module

src/utils.jl

@@ -0,0 +1,89 @@
+#================================================================
+
+Utilities for compressing hashes into fixed-size hashes.
+
+================================================================#
+
+using SHA
+
+#========================
+HashCompressor definition and constructors
+========================#
+
+@doc """
+    struct HashCompressor
+
+A compressor for converting variable-width hashes generated by LSHFunctions
+into fixed-width hashes. HashCompressor works by taking an array of hashes
+generated by an LSHFunction, and using SHA-256 to convert it into a fixed-width
+hash.
+"""
+struct HashCompressor
+    n_bytes :: Int64
+    salt :: Vector{UInt8}
+end
+
+@doc """
+    function HashCompressor(
+        n_bytes :: Integer = 32,
+        salt :: Union{Vector{UInt8}} = Vector{UInt8}(undef,0)
+    )
+
+Construct a new HashCompressor that compresses a hash returned by an LSHFunction
+into `n_bytes` bytes.
+
+# Keyword arguments
+- `n_bytes::Integer` (default: `32`): the number of bytes to compress hashes into.
+- `salt::Vector{UInt8}` (default: `Vector{UInt8}(undef,0)`: a salt to prepend to hashes before compression using SHA-256.
+
+# Examples
+```jldoctest; setup = :(using LSHFunctions)
+julia> compressor = HashCompressor(n_bytes=4);
+
+julia> compressor([1, 4, 2, 9])
+4-element Array{UInt8,1}:
+ 0xb8
+ 0xdd
+ 0x5a
+ 0x5e
+```
+"""
+function HashCompressor(
+    ;
+    n_bytes :: Integer = 32,
+    salt :: Union{Nothing,Vector{UInt8}} = Vector{UInt8}(undef,0)
+)
+    if !(0 <= n_bytes <= 32)
+        "n_bytes must satisfy 0 <= n_bytes <= 32" |>
+        ErrorException |>
+        throw
+    end
+
+    HashCompressor(Int64(n_bytes), salt)
+end
+
+#========================
+Compression functions
+========================#
+(compressor::HashCompressor)(hashes::BitArray{1}) =
+    reinterpret(UInt8, hashes.chunks) |> compressor
+
+(compressor::HashCompressor)(hashes::AbstractVector{I}) where {I <: Integer} =
+    reinterpret(UInt8, hashes) |> compressor
+
+function (compressor::HashCompressor)(hashes::AbstractVector{UInt8})
+    hashes = begin
+        if length(compressor.salt) == 0
+            sha2_256(hashes)
+        else
+            sha2_256([compressor.salt; hashes])
+        end
+    end
+
+    if compressor.n_bytes < 32
+        hashes[1:compressor.n_bytes]
+    else
+        hashes
+    end
+end
+

src/utils/hash_compression.jl

@@ -0,0 +1,29 @@
+#================================================================
+
+Common matrix and vector operations used in multiple locations
+throughout the module.
+
+================================================================#
+
+# Compute the norms of vectors and columns of matrices
+col_norms(x::Union{AbstractVector,AbstractMatrix}) =
+    map(norm, eachcol(x))
+
+col_norms(x::Union{Vector,Matrix}) =
+    map(BLAS.nrm2, eachcol(x))
+
+col_norms(x::SparseVector) =
+    [BLAS.nrm2(x.nzval)]
+
+col_norms(x::SparseMatrixCSC{T}) where {T} = begin
+    output = Vector{T}(undef, size(x,2))
+    @inbounds for ii = 1:size(x,2)
+        result = T(0)
+        start_idx, end_idx = x.colptr[ii], x.colptr[ii+1]-1
+        @simd for idx = start_idx:end_idx
+            result += x.nzval[idx].^2
+        end
+        output[ii] = √result
+    end
+    return output
+end

src/utils/vecops.jl

@@ -28,3 +28,5 @@ include(joinpath("hashes", "test_lshfunction.jl"))
 
 include(joinpath("function_hashing", "test_monte_carlo.jl"))
 include(joinpath("function_hashing", "test_chebhash.jl"))
+
+include(joinpath("utils", "test_hash_compression.jl"))

test/runtests.jl

@@ -0,0 +1,35 @@
+using Test, LSHFunctions
+
+#==================
+Tests
+==================#
+@testset "HashCompressor tests" begin
+    @testset "Can compress Vector{UInt8} hashes" begin
+        compressor = HashCompressor(n_bytes=6)
+        hashes = UInt8[0x01, 0x04, 0x02, 0x08, 0x06, 0x07, 0x08, 0x04]
+
+        @test compressor(hashes) == UInt8[0xce, 0xd8, 0x24, 0x1c, 0xc0, 0x48]
+    end
+
+    @testset "Can compress Vector{Integer} hashes" begin
+        compressor = HashCompressor(n_bytes=4)
+        hashes = [-1, 8, -6, 3, -5, -9, 9, 0]
+
+        @test compressor(hashes) == UInt8[0xb2, 0x7f, 0x8e, 0xb4]
+    end
+
+    @testset "Can compress BitArray{1} hashes" begin
+        compressor = HashCompressor(n_bytes=5)
+        hashes = BitArray([1, 1, 1, 0, 0, 1, 0, 0, 1, 0])
+
+        @test compressor(hashes) == UInt8[0xa2, 0x99, 0xd7, 0x9f, 0x67]
+    end
+
+    @testset "Can salt hashes" begin
+        salt = UInt8[0xcb, 0xe7, 0x12]
+        compressor = HashCompressor(n_bytes=6, salt=salt)
+        hashes = [-1, 8, -6, 3, -5, -9, 9, 0]
+
+        @test compressor(hashes) == UInt8[0x9f, 0x5c, 0xf4, 0x3a, 0x29, 0x22]
+    end
+end