gcs: Implement version 2 filters.

This implements new version 2 filters which have 4 changes as compared
to version 1 filters:

- Support for independently specifying the false positive rate and
  Golomb coding bin size which allows minimizing the filter size
- A faster (incompatible with version 1) reduction function
- A more compact serialization for the number of members in the set
- Deduplication of all hash collisions prior to reducing and serializing
  the deltas

In addition, it adds a full set of tests and updates the benchmarks to
use the new version 2 filters.

The primary motivating factor for these changes is the ability to
minimize the size of the filters, however, the following is a before and
after comparison of version 1 and 2 filters in terms of performance and
allocations.

It is interesting to note the results for attempting to match a single
item is not very representative due to the fact the actual hash value
itself dominates to the point it can significantly vary due to the very
low ns timings involved.  Those differences average out when matching
multiple items, which is the much more realistic scenario, and the
performance increase is in line with the expected values.  It is also
worth nothing that filter construction now takes a bit longer due to the
additional deduplication step.  While the performance numbers for filter
construction are about 25% larger in relative terms, it is only a few ms
difference in practice and therefore is an acceptable trade off for the
size savings provided.

benchmark                      old ns/op    new ns/op    delta
-----------------------------------------------------------------
BenchmarkFilterBuild50000      16194920     20279043     +25.22%
BenchmarkFilterBuild100000     32609930     41629998     +27.66%
BenchmarkFilterMatch           620          593          -4.35%
BenchmarkFilterMatchAny        2687         2302         -14.33%

benchmark                      old allocs   new allocs   delta
-----------------------------------------------------------------
BenchmarkFilterBuild50000      6            17           +183.33%
BenchmarkFilterBuild100000     6            18           +200.00%
BenchmarkFilterMatch           0            0            +0.00%
BenchmarkFilterMatchAny        0            0            +0.00%

benchmark                      old bytes    new bytes    delta
-----------------------------------------------------------------
BenchmarkFilterBuild50000      688366       2074653      +201.39%
BenchmarkFilterBuild100000     1360064      4132627      +203.86%
BenchmarkFilterMatch           0            0            +0.00%
BenchmarkFilterMatchAny        0            0            +0.00%
This commit is contained in:
Dave Collins 2019-08-21 07:06:03 -05:00
parent b8864c39dc
commit 2c3a4e3054
No known key found for this signature in database
GPG Key ID: B8904D9D9C93D1F2
8 changed files with 424 additions and 79 deletions

View File

@ -17,9 +17,16 @@ var (
// globalHashResult is used to ensure the benchmarks do not elide code.
globalHashResult chainhash.Hash
)
// Collision probability for the benchmarks (1/2**20).
P = uint8(20)
const (
// benchB is used as the B parameter when creating new filters throughout
// the benchmarks.
benchB = uint8(19)
// benchM is used as the M parameter when creating new filters throughout
// the benchmarks.
benchM = 784931
)
// genFilterElements generates the given number of elements using the provided
@ -51,7 +58,7 @@ func BenchmarkFilterBuild50000(b *testing.B) {
b.ResetTimer()
var key [KeySize]byte
for i := 0; i < b.N; i++ {
_, err := NewFilterV1(P, key, contents)
_, err := NewFilterV2(benchB, benchM, key, contents)
if err != nil {
b.Fatalf("unable to generate filter: %v", err)
}
@ -71,7 +78,7 @@ func BenchmarkFilterBuild100000(b *testing.B) {
b.ResetTimer()
var key [KeySize]byte
for i := 0; i < b.N; i++ {
_, err := NewFilterV1(P, key, contents)
_, err := NewFilterV2(benchB, benchM, key, contents)
if err != nil {
b.Fatalf("unable to generate filter: %v", err)
}
@ -88,7 +95,7 @@ func BenchmarkFilterMatch(b *testing.B) {
}
var key [KeySize]byte
filter, err := NewFilterV1(P, key, contents)
filter, err := NewFilterV2(benchB, benchM, key, contents)
if err != nil {
b.Fatalf("Failed to build filter")
}
@ -96,7 +103,7 @@ func BenchmarkFilterMatch(b *testing.B) {
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
globalMatch = filter.Match(key, []byte("Nate"))
globalMatch = filter.Match(key, []byte("Something"))
globalMatch = filter.Match(key, []byte("Nates"))
}
}
@ -119,7 +126,7 @@ func BenchmarkFilterMatchAny(b *testing.B) {
}
var key [KeySize]byte
filter, err := NewFilterV1(P, key, contents)
filter, err := NewFilterV2(benchB, benchM, key, contents)
if err != nil {
b.Fatalf("Failed to build filter")
}
@ -141,7 +148,7 @@ func BenchmarkHash(b *testing.B) {
}
var key [KeySize]byte
filter, err := NewFilterV1(P, key, contents)
filter, err := NewFilterV2(benchB, benchM, key, contents)
if err != nil {
b.Fatalf("Failed to build filter")
}

View File

@ -20,6 +20,10 @@ const (
// collision probability.
ErrPTooBig
// ErrBTooBig signifies that the provided Golomb coding bin size is larger
// than the maximum allowed value.
ErrBTooBig
// ErrMisserialized signifies a filter was misserialized and is missing the
// N and/or P parameters of a serialized filter.
ErrMisserialized
@ -32,6 +36,7 @@ const (
var errorCodeStrings = map[ErrorCode]string{
ErrNTooBig: "ErrNTooBig",
ErrPTooBig: "ErrPTooBig",
ErrBTooBig: "ErrBTooBig",
ErrMisserialized: "ErrMisserialized",
}

View File

@ -16,6 +16,7 @@ func TestErrorCodeStringer(t *testing.T) {
}{
{ErrNTooBig, "ErrNTooBig"},
{ErrPTooBig, "ErrPTooBig"},
{ErrBTooBig, "ErrBTooBig"},
{ErrMisserialized, "ErrMisserialized"},
{0xffff, "Unknown ErrorCode (65535)"},
}

39
gcs/fastreduce.go Normal file
View File

@ -0,0 +1,39 @@
// Copyright (c) 2019 The Decred developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.
//+build go1.12
package gcs
import (
"math/bits"
)
// fastReduce calculates a mapping that is more or less equivalent to x mod N.
// However, instead of using a mod operation that can lead to slowness on many
// processors when not using a power of two due to unnecessary division, this
// uses a "multiply-and-shift" trick that eliminates all divisions as described
// in a blog post by Daniel Lemire, located at the following site at the time
// of this writing:
// https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/
//
// Since that link might disappear, the general idea is to multiply by N and
// shift right by log2(N). Since N is a 64-bit integer in this case, it
// becomes:
//
// (x * N) / 2^64 == (x * N) >> 64
//
// This is a fair map since it maps integers in the range [0,2^64) to multiples
// of N in [0, N*2^64) and then divides by 2^64 to map all multiples of N in
// [0,2^64) to 0, all multiples of N in [2^64, 2*2^64) to 1, etc. This results
// in either ceil(2^64/N) or floor(2^64/N) multiples of N.
func fastReduce(x, N uint64) uint64 {
// This uses math/bits to perform the 128-bit multiplication as the compiler
// will replace it with the relevant intrinsic on most architectures.
//
// The high 64 bits in a 128-bit product is the same as shifting the entire
// product right by 64 bits.
hi, _ := bits.Mul64(x, N)
return hi
}

53
gcs/fastreduce_old.go Normal file
View File

@ -0,0 +1,53 @@
// Copyright (c) 2019 The Decred developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.
//+build !go1.12
package gcs
// mul64 returns the 128-bit product of x and y: (hi, lo) = x * y
// with the product bits' upper half returned in hi and the lower
// half returned in lo.
//
// Copied from go1.12 stdlib.
func mul64(x, y uint64) (hi, lo uint64) {
const mask32 = 1<<32 - 1
x0 := x & mask32
x1 := x >> 32
y0 := y & mask32
y1 := y >> 32
w0 := x0 * y0
t := x1*y0 + w0>>32
w1 := t & mask32
w2 := t >> 32
w1 += x0 * y1
hi = x1*y1 + w2 + w1>>32
lo = x * y
return
}
// fastReduce calculates a mapping that is more or less equivalent to x mod N.
// However, instead of using a mod operation that can lead to slowness on many
// processors when not using a power of two due to unnecessary division, this
// uses a "multiply-and-shift" trick that eliminates all divisions as described
// in a blog post by Daniel Lemire, located at the following site at the time
// of this writing:
// https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/
//
// Since that link might disappear, the general idea is to multiply by N and
// shift right by log2(N). Since N is a 64-bit integer in this case, it
// becomes:
//
// (x * N) / 2^64 == (x * N) >> 64
//
// This is a fair map since it maps integers in the range [0,2^64) to multiples
// of N in [0, N*2^64) and then divides by 2^64 to map all multiples of N in
// [0,2^64) to 0, all multiples of N in [2^64, 2*2^64) to 1, etc. This results
// in either ceil(2^64/N) or floor(2^64/N) multiples of N.
func fastReduce(x, N uint64) uint64 {
// The high 64 bits in a 128-bit product is the same as shifting the entire
// product right by 64 bits.
hi, _ := mul64(x, N)
return hi
}

View File

@ -7,6 +7,7 @@
package gcs
import (
"bytes"
"encoding/binary"
"fmt"
"math"
@ -16,8 +17,24 @@ import (
"github.com/dchest/siphash"
"github.com/decred/dcrd/chaincfg/chainhash"
"github.com/decred/dcrd/crypto/blake256"
"github.com/decred/dcrd/wire"
)
// modReduceV1 is the reduction method used in version 1 filters and simply
// consists of x mod N.
func modReduceV1(x, N uint64) uint64 {
return x % N
}
// chooseReduceFunc chooses which reduction method to use based on the filter
// version and returns the appropriate function to perform it.
func chooseReduceFunc(version uint16) func(uint64, uint64) uint64 {
if version == 1 {
return modReduceV1
}
return fastReduce
}
// KeySize is the size of the byte array required for key material for the
// SipHash keyed hash function.
const KeySize = 16
@ -34,7 +51,7 @@ func (s *uint64s) Swap(i, j int) { (*s)[i], (*s)[j] = (*s)[j], (*s)[i] }
//
// It is used internally to implement the exported filter version types.
//
// See FilterV1 for more details.
// See FilterV1 and FilterV2 for more details.
type filter struct {
version uint16
n uint32
@ -75,11 +92,14 @@ func newFilter(version uint16, B uint8, M uint64, key [KeySize]byte, data [][]by
panic(fmt.Sprintf("B value of %d is greater than max allowed 32", B))
}
switch version {
case 1:
case 1, 2:
default:
panic(fmt.Sprintf("version %d filters are not supported", version))
}
// Note that some of the entries might end up hashing to duplicates and
// being removed below, so it is important to perform this check on the
// raw data to maintain consensus.
numEntries := uint64(len(data))
if numEntries > math.MaxInt32 {
str := fmt.Sprintf("unable to create filter with %d entries greater "+
@ -87,6 +107,31 @@ func newFilter(version uint16, B uint8, M uint64, key [KeySize]byte, data [][]by
return nil, makeError(ErrNTooBig, str)
}
// Insert the hash of each data element into a slice while removing any
// duplicates in the data for filters after version 1.
k0 := binary.LittleEndian.Uint64(key[0:8])
k1 := binary.LittleEndian.Uint64(key[8:16])
values := make([]uint64, 0, numEntries)
switch {
case version == 1:
for _, d := range data {
v := siphash.Hash(k0, k1, d)
values = append(values, v)
}
case version > 1:
seen := make(map[uint64]struct{}, numEntries*2)
for _, d := range data {
v := siphash.Hash(k0, k1, d)
if _, ok := seen[v]; ok {
continue
}
seen[v] = struct{}{}
values = append(values, v)
}
}
numEntries = uint64(len(values))
// Create the filter object and insert metadata.
modBMask := uint64(1<<B) - 1
f := filter{
@ -97,18 +142,14 @@ func newFilter(version uint16, B uint8, M uint64, key [KeySize]byte, data [][]by
}
// Nothing to do for an empty filter.
if len(data) == 0 {
if len(values) == 0 {
return &f, nil
}
// Insert the hash of each data element reduced to the range [0,N*M) into
// slice and sort it.
values := make([]uint64, 0, numEntries)
k0 := binary.LittleEndian.Uint64(key[0:8])
k1 := binary.LittleEndian.Uint64(key[8:16])
for _, d := range data {
v := siphash.Hash(k0, k1, d) % f.modulusNM
values = append(values, v)
// Reduce the hash of each data element to the range [0,N*M) and sort it.
reduceFn := chooseReduceFunc(version)
for i, v := range values {
values[i] = reduceFn(v, f.modulusNM)
}
sort.Sort((*uint64s)(&values))
@ -159,12 +200,28 @@ func newFilter(version uint16, B uint8, M uint64, key [KeySize]byte, data [][]by
b.writeNBits(remainder, uint(f.b))
}
// Save the filter data internally as n + filter bytes
ndata := make([]byte, 4+len(b.bytes))
binary.BigEndian.PutUint32(ndata, f.n)
copy(ndata[4:], b.bytes)
f.filterNData = ndata
f.filterData = ndata[4:]
// Save the filter data internally as n + filter bytes along with the raw
// filter data as a slice into it.
switch version {
case 1:
ndata := make([]byte, 4+len(b.bytes))
binary.BigEndian.PutUint32(ndata, f.n)
copy(ndata[4:], b.bytes)
f.filterNData = ndata
f.filterData = ndata[4:]
case 2:
var buf bytes.Buffer
nSize := wire.VarIntSerializeSize(uint64(f.n))
buf.Grow(nSize + len(b.bytes))
// The errors are ignored here since they can't realistically fail due
// to writing into an allocated buffer.
_ = wire.WriteVarInt(&buf, 0, uint64(f.n))
_, _ = buf.Write(b.bytes)
f.filterNData = buf.Bytes()
f.filterData = f.filterNData[nSize:]
}
return &f, nil
}
@ -205,15 +262,15 @@ func (f *filter) Match(key [KeySize]byte, data []byte) bool {
return false
}
// Create a filter bitstream.
b := newBitReader(f.filterData)
// Hash our search term with the same parameters as the filter.
// Hash the search term with the same parameters as the filter.
reduceFn := chooseReduceFunc(f.version)
k0 := binary.LittleEndian.Uint64(key[0:8])
k1 := binary.LittleEndian.Uint64(key[8:16])
term := siphash.Hash(k0, k1, data) % f.modulusNM
term := siphash.Hash(k0, k1, data)
term = reduceFn(term, f.modulusNM)
// Go through the search filter and look for the desired value.
b := newBitReader(f.filterData)
var lastValue uint64
for lastValue <= term {
// Read the difference between previous and new value from
@ -247,10 +304,8 @@ func (f *filter) MatchAny(key [KeySize]byte, data [][]byte) bool {
return false
}
// Create a filter bitstream.
b := newBitReader(f.filterData)
// Create an uncompressed filter of the search values.
reduceFn := chooseReduceFunc(f.version)
var values *[]uint64
if v := matchPool.Get(); v != nil {
values = v.(*[]uint64)
@ -263,7 +318,8 @@ func (f *filter) MatchAny(key [KeySize]byte, data [][]byte) bool {
k0 := binary.LittleEndian.Uint64(key[0:8])
k1 := binary.LittleEndian.Uint64(key[8:16])
for _, d := range data {
v := siphash.Hash(k0, k1, d) % f.modulusNM
v := siphash.Hash(k0, k1, d)
v = reduceFn(v, f.modulusNM)
*values = append(*values, v)
}
sort.Sort((*uint64s)(values))
@ -271,6 +327,7 @@ func (f *filter) MatchAny(key [KeySize]byte, data [][]byte) bool {
// Zip down the filters, comparing values until we either run out of
// values to compare in one of the filters or we reach a matching
// value.
b := newBitReader(f.filterData)
searchSize := len(data)
var searchIdx int
var filterVal uint64
@ -377,6 +434,99 @@ func FromBytesV1(P uint8, d []byte) (*FilterV1, error) {
return &FilterV1{filter: f}, nil
}
// FilterV2 describes an immutable filter that can be built from a set of data
// elements, serialized, deserialized, and queried in a thread-safe manner. The
// serialized form is compressed as a Golomb Coded Set (GCS) along with the
// number of members of the set. The hash function used is SipHash, a keyed
// function. The key used in building the filter is required in order to match
// filter values and is not included in the serialized form.
//
// Version 2 filters differ from version 1 filters in four ways:
//
// 1) Support for independently specifying the false positive rate and Golomb
// coding bin size which allows minimizing the filter size
// 2) A faster (incompatible with version 1) reduction function
// 3) A more compact serialization for the number of members in the set
// 4) Deduplication of all hash collisions prior to reducing and serializing the
// deltas
type FilterV2 struct {
filter
}
// B returns the tunable bits parameter that was used to construct the filter.
// It represents the bin size in the underlying Golomb coding with a value of
// 2^B.
func (f *FilterV2) B() uint8 {
return f.b
}
// NewFilterV2 builds a new GCS filter with the provided tunable parameters that
// contains every item of the passed data as a member of the set.
//
// B is the tunable bits parameter for constructing the filter that is used as
// the bin size in the underlying Golomb coding with a value of 2^B. The
// optimal value of B to minimize the size of the filter for a given false
// positive rate 1/M is floor(log_2(M) - 0.055256). The maximum allowed value
// for B is 32. An error will be returned for larger values.
//
// M is the inverse of the target false positive rate for the filter. The
// optimal value of M to minimize the size of the filter for a given B is
// ceil(1.497137 * 2^B).
//
// key is a key used in the SipHash function used to hash each data element
// prior to inclusion in the filter. This helps thwart would be attackers
// attempting to choose elements that intentionally cause false positives.
//
// The general process for determining optimal parameters for B and M to
// minimize the size of the filter is to start with the desired false positive
// rate and calculate B per the aforementioned formula accordingly. Then, if
// the application permits the false positive rate to be varied, calculate the
// optimal value of M via the formula provided under the description of M.
func NewFilterV2(B uint8, M uint64, key [KeySize]byte, data [][]byte) (*FilterV2, error) {
// Basic sanity check.
if B > 32 {
str := fmt.Sprintf("B value of %d is greater than max allowed 32", B)
return nil, makeError(ErrBTooBig, str)
}
filter, err := newFilter(2, B, M, key, data)
if err != nil {
return nil, err
}
return &FilterV2{filter: *filter}, nil
}
// FromBytesV2 deserializes a version 2 GCS filter from a known B, M, and
// serialized filter as returned by Bytes().
func FromBytesV2(B uint8, M uint64, d []byte) (*FilterV2, error) {
if B > 32 {
str := fmt.Sprintf("B value of %d is greater than max allowed 32", B)
return nil, makeError(ErrBTooBig, str)
}
var n uint64
var filterData []byte
if len(d) > 0 {
var err error
n, err = wire.ReadVarInt(bytes.NewReader(d), 0)
if err != nil {
str := fmt.Sprintf("failed to read number of filter items: %v", err)
return nil, makeError(ErrMisserialized, str)
}
filterData = d[wire.VarIntSerializeSize(n):]
}
f := filter{
version: 2,
n: uint32(n),
b: B,
modulusNM: n * M,
filterNData: d,
filterData: filterData,
}
return &FilterV2{filter: f}, nil
}
// MakeHeaderForFilter makes a filter chain header for a filter, given the
// filter and the previous filter chain header.
func MakeHeaderForFilter(filter *FilterV1, prevHeader *chainhash.Hash) chainhash.Hash {

View File

@ -16,6 +16,13 @@ import (
"github.com/decred/dcrd/chaincfg/chainhash"
)
// filterMatcher allows different versions of the filter types to be used for
// match testing.
type filterMatcher interface {
Match([KeySize]byte, []byte) bool
MatchAny([KeySize]byte, [][]byte) bool
}
// TestFilter ensures the filters and all associated methods work as expected by
// using various known parameters and contents along with random keys for
// matching purposes.
@ -115,6 +122,72 @@ func TestFilter(t *testing.T) {
fixedKey: [KeySize]byte{},
wantBytes: "000000111ca3aafb023074dc5bf2498df791b7d6e846e9f5016006d600",
wantHash: "afa181cd5c4b08eb9c16d1c97c95df1ca7b82e5e444a396cec5e02f2804fbd1a",
}, {
name: "v2 empty filter",
version: 2,
b: 19,
m: 784931,
matchKey: randKey,
contents: nil,
wantMatches: nil,
fixedKey: [KeySize]byte{},
wantBytes: "",
wantHash: "0000000000000000000000000000000000000000000000000000000000000000",
}, {
name: "v2 filter contents1 with B=19, M=784931",
version: 2,
b: 19,
m: 784931,
matchKey: randKey,
contents: contents1,
wantMatches: contents1,
fixedKey: [KeySize]byte{},
wantBytes: "1189af70ad5baf9da83c64e99b18e96a06cd7295a58b324e81f09c85d093f1e33dcd6f40f18cfcbe2aeb771d8390",
wantHash: "b616838c6090d3e732e775cc2f336ce0b836895f3e0f22d6c3ee4485a6ea5018",
}, {
name: "v2 filter contents2 with B=19, M=784931",
version: 2,
b: 19,
m: 784931,
matchKey: randKey,
contents: contents2,
wantMatches: contents2,
fixedKey: [KeySize]byte{},
wantBytes: "118d4be5372d2f4731c7e1681aefd23028be12306b4d90701a46b472ee80ad60f9fa86c4d6430cfb495ced604362",
wantHash: "f3028f42909209120c8bf649fbbc5a70fb907d8997a02c2c1f2eef0e6402cb15",
}, {
name: "v2 filter contents1 with B=20, M=1569862",
version: 2,
b: 20,
m: 1569862,
matchKey: randKey,
contents: contents1,
wantMatches: contents1,
fixedKey: [KeySize]byte{},
wantBytes: "1189af7056adebe769078c9e99b1774b509b35c52b4b0b324f40f83f2174227e3c33dcd67a078c33f2f855d6ef1d8390",
wantHash: "10d6c29ba756301e42b97a103de601f604f1cd3150e44ac7cb8cdf63222d93d3",
}, {
name: "v2 filter contents2 with B=20, M=1569862",
version: 2,
b: 20,
m: 1569862,
matchKey: randKey,
contents: contents2,
wantMatches: contents2,
fixedKey: [KeySize]byte{},
wantBytes: "118d4be69b968bd1cc38fc2d01aefc918142f847e0d69b9070192359fcbba015ac1f9fa86a26b1fc33ed32b9da604361",
wantHash: "cb23a266bf136103cbfedb33041d4f4324106ed1539b9ec5d1cb4e71bc3cbdec",
}, {
name: "v2 filter contents2 with B=10, M=1534",
version: 2,
b: 10,
m: 1534,
matchKey: randKey,
contents: contents2,
wantMatches: contents2,
fixedKey: [KeySize]byte{},
wantBytes: "118d5a6fbd3e3fc1aa472f983790848fcbcd6b9fb89cc34caf6048",
wantHash: "d7c5f91d6490ec4d66e6db825ad1a628ec7367dc9a47c31bcde31eb5a6ba194b",
}}
for _, test := range tests {
@ -140,7 +213,8 @@ func TestFilter(t *testing.T) {
resultN, uint32(len(test.contents)))
continue
}
if test.version == 1 {
switch test.version {
case 1:
v1Filter := &FilterV1{filter: *f}
resultP := v1Filter.P()
if resultP != test.b {
@ -148,6 +222,15 @@ func TestFilter(t *testing.T) {
resultP, test.b)
continue
}
case 2:
v2Filter := &FilterV2{filter: *f}
resultB := v2Filter.B()
if resultB != test.b {
t.Errorf("%q: unexpected B -- got %d, want %d", test.name,
resultB, test.b)
continue
}
}
// Ensure empty data never matches.
@ -250,12 +333,6 @@ func TestFilter(t *testing.T) {
continue
}
// filterMatcher allows different versions of the filter types to be
// used for the match testing below.
type filterMatcher interface {
Match([KeySize]byte, []byte) bool
}
// Deserialize the filter from bytes.
var f2 filterMatcher
switch test.version {
@ -267,6 +344,14 @@ func TestFilter(t *testing.T) {
}
f2 = tf2
case 2:
tf2, err := FromBytesV2(test.b, test.m, wantBytes)
if err != nil {
t.Errorf("%q: unexpected err: %v", test.name, err)
continue
}
f2 = tf2
default:
t.Errorf("%q: unsupported filter version: %d", test.name,
test.version)
@ -283,16 +368,10 @@ func TestFilter(t *testing.T) {
}
}
// TestFilterMisses ensures the filter does not match entries with a rate that
// far exceeds the false positive rate.
func TestFilterMisses(t *testing.T) {
// Create a filter with the lowest supported false positive rate to reduce
// the chances of a false positive as much as possible.
var key [KeySize]byte
f, err := NewFilterV1(32, key, [][]byte{[]byte("entry")})
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
// testFilterMisses ensures the provided filter does not match entries with a
// rate that far exceeds the false positive rate.
func testFilterMisses(t *testing.T, key [KeySize]byte, f filterMatcher) {
t.Helper()
// Since the filter may have false positives, try several queries and track
// how many matches there are. Something is very wrong if the filter
@ -324,10 +403,33 @@ func TestFilterMisses(t *testing.T) {
}
}
// TestFilterMisses ensures the filter does not match entries with a rate that
// far exceeds the false positive rate.
func TestFilterMisses(t *testing.T) {
// Create a version 1 filter with the lowest supported false positive rate
// to reduce the chances of a false positive as much as possible and test
// it for misses.
const b = 32
var key [KeySize]byte
f, err := NewFilterV1(b, key, [][]byte{[]byte("entry")})
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
testFilterMisses(t, key, f)
// Do the same for a version 2 filter.
const m = 6430154453
f2, err := NewFilterV2(b, m, key, [][]byte{[]byte("entry")})
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
testFilterMisses(t, key, f2)
}
// TestFilterCorners ensures a few negative corner cases such as specifying
// parameters that are too large behave as expected.
func TestFilterCorners(t *testing.T) {
// Attempt to construct filter with parameters too large.
// Attempt to construct v1 filter with parameters too large.
const largeP = 33
var key [KeySize]byte
_, err := NewFilterV1(largeP, key, nil)
@ -341,7 +443,21 @@ func TestFilterCorners(t *testing.T) {
err, ErrPTooBig)
}
// Attempt to decode a filter without the N value serialized properly.
// Attempt to construct v2 filter with parameters too large.
const largeB = 33
const smallM = 1 << 10
_, err = NewFilterV2(largeB, smallM, key, nil)
if !IsErrorCode(err, ErrBTooBig) {
t.Fatalf("did not receive expected err for B too big -- got %v, want %v",
err, ErrBTooBig)
}
_, err = FromBytesV2(largeB, smallM, nil)
if !IsErrorCode(err, ErrBTooBig) {
t.Fatalf("did not receive expected err for B too big -- got %v, want %v",
err, ErrBTooBig)
}
// Attempt to decode a v1 filter without the N value serialized properly.
_, err = FromBytesV1(20, []byte{0x00})
if !IsErrorCode(err, ErrMisserialized) {
t.Fatalf("did not receive expected err -- got %v, want %v", err,

View File

@ -16,8 +16,6 @@ github.com/decred/base58 v1.0.0 h1:BVi1FQCThIjZ0ehG+I99NJ51o0xcc9A/fDKhmJxY6+w=
github.com/decred/base58 v1.0.0/go.mod h1:LLY1p5e3g91byL/UO1eiZaYd+uRoVRarybgcoymu9Ks=
github.com/decred/dcrd/blockchain/stake/v2 v2.0.0 h1:+FMrSt5tPicBKlev0k/r/2VsaVwpIUcm1TPw69XgZw0=
github.com/decred/dcrd/blockchain/stake/v2 v2.0.0/go.mod h1:jv/rKMcZ87lhvVkHot/tElxeAYEUJ3mnKPHJ7WPq86U=
github.com/decred/dcrd/chaincfg v1.5.1 h1:u1Xbq0VTnAXIHW5ECqrWe0VYSgf5vWHqpSiwoLBzxAQ=
github.com/decred/dcrd/chaincfg v1.5.1/go.mod h1:FukMzTjkwzjPU+hK7CqDMQe3NMbSZAYU5PAcsx1wlv0=
github.com/decred/dcrd/chaincfg/chainhash v1.0.1 h1:0vG7U9+dSjSCaHQKdoSKURK2pOb47+b+8FK5q4+Je7M=
github.com/decred/dcrd/chaincfg/chainhash v1.0.1/go.mod h1:OVfvaOsNLS/A1y4Eod0Ip/Lf8qga7VXCQjUQLbkY0Go=
github.com/decred/dcrd/chaincfg/v2 v2.0.2 h1:VeGY52lHuYT01tIGbvYj+OO0GaGxGaJmnh+4vGca1+U=
@ -26,36 +24,20 @@ github.com/decred/dcrd/chaincfg/v2 v2.1.0 h1:2S7TL9YWnKDDiH5bTpp3xcBo+1gl1IXFi5K
github.com/decred/dcrd/chaincfg/v2 v2.1.0/go.mod h1:hpKvhLCDAD/xDZ3V1Pqpv9fIKVYYi11DyxETguazyvg=
github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0=
github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc=
github.com/decred/dcrd/database v1.0.1 h1:BSIerNf4RhSA0iDhiE/320RYqD2y9T+SCj99Pv7svgo=
github.com/decred/dcrd/database v1.0.1/go.mod h1:ILCeyOHFew3fZ7K2B9jl+tp5qFOap/pEGoo6Yy6Wk0g=
github.com/decred/dcrd/database/v2 v2.0.0 h1:KWiyZHk+QyNKQvvxm/KpIejhTqYJqH9ssz1+9sT9nVA=
github.com/decred/dcrd/database/v2 v2.0.0/go.mod h1:Sj2lvTRB0mfSu9uD7ObfwCY/eJ954GFU/X+AndJIyfE=
github.com/decred/dcrd/dcrec v0.0.0-20180721005212-59fe2b293f69/go.mod h1:cRAH1SNk8Mi9hKBc/DHbeiWz/fyO8KWZR3H7okrIuOA=
github.com/decred/dcrd/dcrec v0.0.0-20180721031028-5369a485acf6/go.mod h1:cRAH1SNk8Mi9hKBc/DHbeiWz/fyO8KWZR3H7okrIuOA=
github.com/decred/dcrd/dcrec v0.0.0-20180801202239-0761de129164 h1:N5s3yVfjBNW6XNG3gLxYpvt0IUjUsp/FRfC75QpSI+E=
github.com/decred/dcrd/dcrec v0.0.0-20180801202239-0761de129164/go.mod h1:cRAH1SNk8Mi9hKBc/DHbeiWz/fyO8KWZR3H7okrIuOA=
github.com/decred/dcrd/dcrec v1.0.0 h1:W+z6Es+Rai3MXYVoPAxYr5U1DGis0Co33scJ6uH2J6o=
github.com/decred/dcrd/dcrec v1.0.0/go.mod h1:HIaqbEJQ+PDzQcORxnqen5/V1FR3B4VpIfmePklt8Q8=
github.com/decred/dcrd/dcrec/edwards v0.0.0-20180721005212-59fe2b293f69/go.mod h1:+ehP0Hk/mesyZXttxCtBbhPX23BMpZJ1pcVBqUfbmvU=
github.com/decred/dcrd/dcrec/edwards v0.0.0-20180721031028-5369a485acf6/go.mod h1:+ehP0Hk/mesyZXttxCtBbhPX23BMpZJ1pcVBqUfbmvU=
github.com/decred/dcrd/dcrec/edwards v0.0.0-20181208004914-a0816cf4301f h1:NF7vp3nZ4MsAiXswGmE//m83jCN0lDsQrLI7IwLCTlo=
github.com/decred/dcrd/dcrec/edwards v0.0.0-20181208004914-a0816cf4301f/go.mod h1:+ehP0Hk/mesyZXttxCtBbhPX23BMpZJ1pcVBqUfbmvU=
github.com/decred/dcrd/dcrec/edwards v1.0.0 h1:UDcPNzclKiJlWqV3x1Fl8xMCJrolo4PB4X9t8LwKDWU=
github.com/decred/dcrd/dcrec/edwards v1.0.0/go.mod h1:HblVh1OfMt7xSxUL1ufjToaEvpbjpWvvTAUx4yem8BI=
github.com/decred/dcrd/dcrec/secp256k1 v1.0.0/go.mod h1:JPMFscGlgXTV684jxQNDijae2qrh0fLG7pJBimaYotE=
github.com/decred/dcrd/dcrec/secp256k1 v1.0.1 h1:EFWVd1p0t0Y5tnsm/dJujgV0ORogRJ6vo7CMAjLseAc=
github.com/decred/dcrd/dcrec/secp256k1 v1.0.1/go.mod h1:lhu4eZFSfTJWUnR3CFRcpD+Vta0KUAqnhTsTksHXgy0=
github.com/decred/dcrd/dcrec/secp256k1 v1.0.2 h1:awk7sYJ4pGWmtkiGHFfctztJjHMKGLV8jctGQhAbKe0=
github.com/decred/dcrd/dcrec/secp256k1 v1.0.2/go.mod h1:CHTUIVfmDDd0KFVFpNX1pFVCBUegxW387nN0IGwNKR0=
github.com/decred/dcrd/dcrutil v1.1.1 h1:zOkGiumN/JkobhAgpG/zfFgUoolGKVGYT5na1hbYUoE=
github.com/decred/dcrd/dcrutil v1.1.1/go.mod h1:Jsttr0pEvzPAw+qay1kS1/PsbZYPyhluiNwwY6yBJS4=
github.com/decred/dcrd/dcrutil/v2 v2.0.0 h1:HTqn2tZ8eqBF4y3hJwjyKBmJt16y7/HjzpE82E/crhY=
github.com/decred/dcrd/dcrutil/v2 v2.0.0/go.mod h1:gUshVAXpd51DlcEhr51QfWL2HJGkMDM1U8chY+9VvQg=
github.com/decred/dcrd/txscript v1.0.1 h1:IMgxZFCw3AyG4EbKwywE3SDNshOSHsoUK1Wk/5GqWJ0=
github.com/decred/dcrd/txscript v1.0.1/go.mod h1:FqUX07Y+u3cJ1eIGPoyWbJg+Wk1NTllln/TyDpx9KnY=
github.com/decred/dcrd/txscript/v2 v2.0.0 h1:So+NcQY58mDHDN2N2edED5syGZp2ed8Ltxj8mDE5CAs=
github.com/decred/dcrd/txscript/v2 v2.0.0/go.mod h1:WStcyYYJa+PHJB4XjrLDRzV96/Z4thtsu8mZoVrU6C0=
github.com/decred/dcrd/wire v1.1.0/go.mod h1:/JKOsLInOJu6InN+/zH5AyCq3YDIOW/EqcffvU8fJHM=
github.com/decred/dcrd/wire v1.2.0 h1:HqJVB7vcklIguzFWgRXw/WYCQ9cD3bUC5TKj53i1Hng=
github.com/decred/dcrd/wire v1.2.0/go.mod h1:/JKOsLInOJu6InN+/zH5AyCq3YDIOW/EqcffvU8fJHM=
github.com/decred/slog v1.0.0 h1:Dl+W8O6/JH6n2xIFN2p3DNjCmjYwvrXsjlSJTQQ4MhE=
@ -71,20 +53,12 @@ github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/gomega v1.4.1 h1:PZSj/UFNaVp3KxrzHOcS7oyuWA7LoOY/77yCTEFu21U=
github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
golang.org/x/crypto v0.0.0-20180718160520-a2144134853f h1:lRy+hhwk7YT7MsKejxuz0C5Q1gk6p/QoPQYEmKmGFb8=
golang.org/x/crypto v0.0.0-20180718160520-a2144134853f/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8 h1:1wopBVtVdWnn03fZelqdXTqk7U7zPQCb+T4rbU9ZEoU=
golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180808004115-f9ce57c11b24 h1:mEsFm194MmS9vCwxFy+zwu0EU7ZkxxMD1iH++vmGdUY=
golang.org/x/net v0.0.0-20180808004115-f9ce57c11b24/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180816055513-1c9583448a9c h1:uHnKXcvx6SNkuwC+nrzxkJ+TpPwZOtumbhWrrOYN5YA=
golang.org/x/sys v0.0.0-20180816055513-1c9583448a9c/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=