flip/dcrd
1
0
Fork 0
mirror of https://github.com/FlipsideCrypto/dcrd.git synced 2026-02-06 10:56:47 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

gcs: Add filter version support.

Browse Source 
This refactors the primary gcs filter logic into an internal struct with
a version parameter in in order to pave the way for supporting v2
filters which will have a different serialization that makes them
incompatible with v1 filters while still retaining the ability to work
with v1 filters in the interim.

The exported type is renamed to FilterV1 and the new internal struct is
embedded so its methods are externally available.

The tests and all callers in the repo have been updated accordingly.
This commit is contained in:
Dave Collins 2019-08-20 08:49:37 -05:00
parent 150b54aa0f
commit 90d2deb420
No known key found for this signature in database
GPG Key ID: B8904D9D9C93D1F2
9 changed files with 125 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
blockchain/go.mod
View File

@ -3,6 +3,7 @@ module github.com/decred/dcrd/blockchain/v2
go 1.11
require (
github.com/dchest/blake256 v1.1.0 // indirect
github.com/decred/dcrd/blockchain/stake/v2 v2.0.1
github.com/decred/dcrd/blockchain/standalone v1.0.0
github.com/decred/dcrd/chaincfg/chainhash v1.0.2

2
blockchain/go.sum
View File

@ -10,6 +10,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dchest/blake256 v1.0.0 h1:6gUgI5MHdz9g0TdrgKqXsoDX+Zjxmm1Sc6OsoGru50I=
github.com/dchest/blake256 v1.0.0/go.mod h1:xXNWCE1jsAP8DAjP+rKw2MbeqLczjI3TRx2VK+9OEYY=
github.com/dchest/blake256 v1.1.0 h1:4AuEhGPT/3TTKFhTfBpZ8hgZE7wJpawcYaEawwsbtqM=
github.com/dchest/blake256 v1.1.0/go.mod h1:xXNWCE1jsAP8DAjP+rKw2MbeqLczjI3TRx2VK+9OEYY=
github.com/dchest/siphash v1.2.1 h1:4cLinnzVJDKxTCl9B01807Yiy+W7ZzVHj/KIroQRvT4=
github.com/dchest/siphash v1.2.1/go.mod h1:q+IRvb2gOSrUnYoPqHiyHXS0FOBBOdl6tONBlVnOnt4=
github.com/decred/base58 v1.0.0 h1:BVi1FQCThIjZ0ehG+I99NJ51o0xcc9A/fDKhmJxY6+w=

2
blockchain/indexers/cfindex.go
View File

@ -174,7 +174,7 @@ func (idx *CFIndex) Create(dbTx database.Tx) error {
// storeFilter stores a given filter, and performs the steps needed to
// generate the filter's header.
func storeFilter(dbTx database.Tx, block *dcrutil.Block, f *gcs.Filter, filterType wire.FilterType) error {
func storeFilter(dbTx database.Tx, block *dcrutil.Block, f *gcs.FilterV1, filterType wire.FilterType) error {
if uint8(filterType) > maxFilterType {
return errors.New("unsupported filter type")
}

8
gcs/bench_test.go
View File

@ -46,7 +46,7 @@ func BenchmarkFilterBuild50000(b *testing.B) {
b.ResetTimer()
var key [KeySize]byte
for i := 0; i < b.N; i++ {
_, err := NewFilter(P, key, contents)
_, err := NewFilterV1(P, key, contents)
if err != nil {
b.Fatalf("unable to generate filter: %v", err)
}
@ -66,7 +66,7 @@ func BenchmarkFilterBuild100000(b *testing.B) {
b.ResetTimer()
var key [KeySize]byte
for i := 0; i < b.N; i++ {
_, err := NewFilter(P, key, contents)
_, err := NewFilterV1(P, key, contents)
if err != nil {
b.Fatalf("unable to generate filter: %v", err)
}
@ -83,7 +83,7 @@ func BenchmarkFilterMatch(b *testing.B) {
}
var key [KeySize]byte
filter, err := NewFilter(P, key, contents)
filter, err := NewFilterV1(P, key, contents)
if err != nil {
b.Fatalf("Failed to build filter")
}
@ -114,7 +114,7 @@ func BenchmarkFilterMatchAny(b *testing.B) {
}
var key [KeySize]byte
filter, err := NewFilter(P, key, contents)
filter, err := NewFilterV1(P, key, contents)
if err != nil {
b.Fatalf("Failed to build filter")
}

14
gcs/blockcf/blockcf.go
View File

@ -28,8 +28,10 @@ import (
"github.com/decred/dcrd/wire"
)
// P is the collision probability used for block committed filters (2^-20)
const P = 20
const (
// P is the collision probability used for block committed filters (2^-20)
P = 20
)
// Entries describes all of the filter entries used to create a GCS filter and
// provides methods for appending data structures found in blocks.
@ -95,7 +97,7 @@ func Key(hash *chainhash.Hash) [gcs.KeySize]byte {
// contain all the previous regular outpoints spent within a block, as well as
// the data pushes within all the outputs created within a block which can be
// spent by regular transactions.
func Regular(block *wire.MsgBlock) (*gcs.Filter, error) {
func Regular(block *wire.MsgBlock) (*gcs.FilterV1, error) {
var data Entries
// Add "regular" data from stake transactions. For each class of stake
@ -163,14 +165,14 @@ func Regular(block *wire.MsgBlock) (*gcs.Filter, error) {
blockHash := block.BlockHash()
key := Key(&blockHash)
return gcs.NewFilter(P, key, data)
return gcs.NewFilterV1(P, key, data)
}
// Extended builds an extended GCS filter from a block. An extended filter
// supplements a regular basic filter by including all transaction hashes of
// regular and stake transactions, and adding the witness data (a.k.a. the
// signature script) found within every non-coinbase regular transaction.
func Extended(block *wire.MsgBlock) (*gcs.Filter, error) {
func Extended(block *wire.MsgBlock) (*gcs.FilterV1, error) {
var data Entries
// For each stake transaction, commit the transaction hash. If the
@ -207,5 +209,5 @@ func Extended(block *wire.MsgBlock) (*gcs.Filter, error) {
blockHash := block.BlockHash()
key := Key(&blockHash)
return gcs.NewFilter(P, key, data)
return gcs.NewFilterV1(P, key, data)
}

102
gcs/gcs.go
View File

@ -31,6 +31,20 @@ func (s *uint64s) Len() int { return len(*s) }
func (s *uint64s) Less(i, j int) bool { return (*s)[i] < (*s)[j] }
func (s *uint64s) Swap(i, j int) { (*s)[i], (*s)[j] = (*s)[j], (*s)[i] }
// filter describes a versioned immutable filter that can be built from a set of
// data elements, serialized, deserialized, and queried in a thread-safe manner.
//
// It is used internally to implement the exported filter version types.
//
// See FilterV1 for more details.
type filter struct {
version uint16
n uint32
p uint8
modulusNP uint64
filterNData []byte // 4 bytes n big endian, remainder is filter data
}
// Filter describes an immutable filter that can be built from a set of data
// elements, serialized, deserialized, and queried in a thread-safe manner. The
// serialized form is compressed as a Golomb Coded Set (GCS), but does not
@ -38,20 +52,14 @@ func (s *uint64s) Swap(i, j int) { (*s)[i], (*s)[j] = (*s)[j], (*s)[i] }
// necessary. The hash function used is SipHash, a keyed function; the key used
// in building the filter is required in order to match filter values and is
// not included in the serialized form.
type Filter struct {
n uint32
p uint8
modulusNP uint64
filterNData []byte // 4 bytes n big endian, remainder is filter data
type FilterV1 struct {
filter
}
// NewFilter builds a new GCS filter with the collision probability of
// `1/(2**P)`, key `key`, and including every `[]byte` in `data` as a member of
// the set.
func NewFilter(P uint8, key [KeySize]byte, data [][]byte) (*Filter, error) {
// Some initial parameter checks: make sure we have data from which to
// build the filter, and make sure our parameters will fit the hash
// function we're using.
// newFilter builds a new GCS filter of the specified version with the collision
// probability of `1/(2**P)`, key `key`, and including every `[]byte` in `data`
// as a member of the set.
func newFilter(version uint16, P uint8, key [KeySize]byte, data [][]byte) (*filter, error) {
if len(data) > math.MaxInt32 {
str := fmt.Sprintf("unable to create filter with %d entries greater "+
"than max allowed %d", len(data), math.MaxInt32)
@ -65,7 +73,8 @@ func NewFilter(P uint8, key [KeySize]byte, data [][]byte) (*Filter, error) {
// Create the filter object and insert metadata.
modP := uint64(1 << P)
modPMask := modP - 1
f := Filter{
f := filter{
version: version,
n: uint32(len(data)),
p: P,
modulusNP: uint64(len(data)) * modP,
@ -126,9 +135,20 @@ func NewFilter(P uint8, key [KeySize]byte, data [][]byte) (*Filter, error) {
return &f, nil
}
// FromBytes deserializes a GCS filter from a known N, P, and serialized filter
// as returned by Bytes().
func FromBytes(N uint32, P uint8, d []byte) (*Filter, error) {
// NewFilter builds a new version 1 GCS filter with the collision probability of
// `1/(2**P)`, key `key`, and including every `[]byte` in `data` as a member of
// the set.
func NewFilterV1(P uint8, key [KeySize]byte, data [][]byte) (*FilterV1, error) {
filter, err := newFilter(1, P, key, data)
if err != nil {
return nil, err
}
return &FilterV1{filter: *filter}, nil
}
// FromBytesV1 deserializes a version 1 GCS filter from a known N, P, and
// serialized filter as returned by Bytes().
func FromBytesV1(N uint32, P uint8, d []byte) (*FilterV1, error) {
// Basic sanity check.
if P > 32 {
str := fmt.Sprintf("P value of %d is greater than max allowed 32", P)
@ -140,18 +160,21 @@ func FromBytes(N uint32, P uint8, d []byte) (*Filter, error) {
binary.BigEndian.PutUint32(ndata, N)
copy(ndata[4:], d)
f := &Filter{
n: N,
p: P,
modulusNP: uint64(N) * uint64(1<<P),
filterNData: ndata,
f := &FilterV1{
filter: filter{
version: 1,
n: N,
p: P,
modulusNP: uint64(N) * uint64(1<<P),
filterNData: ndata,
},
}
return f, nil
}
// FromNBytes deserializes a GCS filter from a known P, and serialized N and
// filter as returned by NBytes().
func FromNBytes(P uint8, d []byte) (*Filter, error) {
// FromNBytesV1 deserializes a version 1 GCS filter from a known P, and
// serialized N and filter as returned by NBytes().
func FromNBytesV1(P uint8, d []byte) (*FilterV1, error) {
var n uint32
if len(d) >= 4 {
n = binary.BigEndian.Uint32(d[:4])
@ -160,18 +183,21 @@ func FromNBytes(P uint8, d []byte) (*Filter, error) {
return nil, makeError(ErrMisserialized, str)
}
f := &Filter{
n: n,
p: P,
modulusNP: uint64(n) * uint64(1<<P),
filterNData: d,
f := &FilterV1{
filter: filter{
version: 1,
n: n,
p: P,
modulusNP: uint64(n) * uint64(1<<P),
filterNData: d,
},
}
return f, nil
}
// Bytes returns the serialized format of the GCS filter, which does not
// include N or P (returned by separate methods) or the key used by SipHash.
func (f *Filter) Bytes() []byte {
func (f *filter) Bytes() []byte {
if len(f.filterNData) == 0 {
return nil
}
@ -180,24 +206,24 @@ func (f *Filter) Bytes() []byte {
// NBytes returns the serialized format of the GCS filter with N, which does
// not include P (returned by a separate method) or the key used by SipHash.
func (f *Filter) NBytes() []byte {
func (f *filter) NBytes() []byte {
return f.filterNData
}
// P returns the filter's collision probability as a negative power of 2 (that
// is, a collision probability of `1/2**20` is represented as 20).
func (f *Filter) P() uint8 {
func (f *filter) P() uint8 {
return f.p
}
// N returns the size of the data set used to build the filter.
func (f *Filter) N() uint32 {
func (f *filter) N() uint32 {
return f.n
}
// Match checks whether a []byte value is likely (within collision probability)
// to be a member of the set represented by the filter.
func (f *Filter) Match(key [KeySize]byte, data []byte) bool {
func (f *filter) Match(key [KeySize]byte, data []byte) bool {
// An empty filter or empty data can't possibly match anything.
if len(f.filterNData) == 0 || len(data) == 0 {
return false
@ -239,7 +265,7 @@ var matchPool sync.Pool
// MatchAny checks whether any []byte value is likely (within collision
// probability) to be a member of the set represented by the filter faster than
// calling Match() for each value individually.
func (f *Filter) MatchAny(key [KeySize]byte, data [][]byte) bool {
func (f *filter) MatchAny(key [KeySize]byte, data [][]byte) bool {
// An empty filter or empty data can't possibly match anything.
if len(f.filterNData) == 0 || len(data) == 0 {
return false
@ -303,7 +329,7 @@ func (f *Filter) MatchAny(key [KeySize]byte, data [][]byte) bool {
// readFullUint64 reads a value represented by the sum of a unary multiple of
// the filter's P modulus (`2**P`) and a big-endian P-bit remainder.
func (f *Filter) readFullUint64(b *bitReader) (uint64, error) {
func (f *filter) readFullUint64(b *bitReader) (uint64, error) {
v, err := b.readUnary()
if err != nil {
return 0, err
@ -319,7 +345,7 @@ func (f *Filter) readFullUint64(b *bitReader) (uint64, error) {
}
// Hash returns the BLAKE256 hash of the filter.
func (f *Filter) Hash() chainhash.Hash {
func (f *filter) Hash() chainhash.Hash {
// Empty filters have a hash of all zeroes.
if len(f.filterNData) == 0 {
return chainhash.Hash{}
@ -335,7 +361,7 @@ func (f *Filter) Hash() chainhash.Hash {
// MakeHeaderForFilter makes a filter chain header for a filter, given the
// filter and the previous filter chain header.
func MakeHeaderForFilter(filter *Filter, prevHeader *chainhash.Hash) chainhash.Hash {
func MakeHeaderForFilter(filter *FilterV1, prevHeader *chainhash.Hash) chainhash.Hash {
filterTip := make([]byte, 2*chainhash.HashSize)
filterHash := filter.Hash()

54
gcs/gcs_test.go
View File

@ -50,6 +50,7 @@ func TestFilter(t *testing.T) {
tests := []struct {
name string // test description
version uint16 // filter version
p uint8 // collision probability
matchKey [KeySize]byte // random filter key for matches
contents [][]byte // data to include in the filter
@ -60,6 +61,7 @@ func TestFilter(t *testing.T) {
wantHash string // expected filter hash
}{{
name: "empty filter",
version: 1,
p: 20,
matchKey: randKey,
contents: nil,
@ -70,6 +72,7 @@ func TestFilter(t *testing.T) {
wantHash: "0000000000000000000000000000000000000000000000000000000000000000",
}, {
name: "contents1 with P=20",
version: 1,
p: 20,
matchKey: randKey,
contents: contents1,
@ -80,6 +83,7 @@ func TestFilter(t *testing.T) {
wantHash: "a802fbe6f06991877cde8f3d770d8da8cf195816f04874cab045ffccaddd880d",
}, {
name: "contents1 with P=19",
version: 1,
p: 19,
matchKey: randKey,
contents: contents1,
@ -90,6 +94,7 @@ func TestFilter(t *testing.T) {
wantHash: "be9ba34f03ced957e6f5c4d583ddfd34c136b486fbec2a42b4c7588a2d7813c1",
}, {
name: "contents2 with P=19",
version: 1,
p: 19,
matchKey: randKey,
contents: contents2,
@ -100,6 +105,7 @@ func TestFilter(t *testing.T) {
wantHash: "dcbaf452f6de4c82ea506fa551d75876c4979ef388f785509b130de62eeaec23",
}, {
name: "contents2 with P=10",
version: 1,
p: 10,
matchKey: randKey,
contents: contents2,
@ -113,7 +119,7 @@ func TestFilter(t *testing.T) {
for _, test := range tests {
// Create a filter with the match key for all tests not related to
// testing serialization.
f, err := NewFilter(test.p, test.matchKey, test.contents)
f, err := newFilter(test.version, test.p, test.matchKey, test.contents)
if err != nil {
t.Errorf("%q: unexpected err: %v", test.name, err)
continue
@ -198,7 +204,8 @@ func TestFilter(t *testing.T) {
}
// Recreate the filter with a fixed key for serialization testing.
fixedFilter, err := NewFilter(test.p, test.fixedKey, test.contents)
fixedFilter, err := newFilter(test.version, test.p, test.fixedKey,
test.contents)
if err != nil {
t.Errorf("%q: unexpected err: %v", test.name, err)
continue
@ -247,10 +254,26 @@ func TestFilter(t *testing.T) {
continue
}
// filterMatcher allows different versions of the filter types to be
// used for the match testing below.
type filterMatcher interface {
Match([KeySize]byte, []byte) bool
}
// Deserialize the filter from bytes.
f2, err := FromBytes(uint32(len(test.contents)), test.p, wantBytes)
if err != nil {
t.Errorf("%q: unexpected err: %v", test.name, err)
var f2 filterMatcher
switch test.version {
case 1:
tf2, err := FromBytesV1(uint32(len(test.contents)), test.p, wantBytes)
if err != nil {
t.Errorf("%q: unexpected err: %v", test.name, err)
continue
}
f2 = tf2
default:
t.Errorf("%q: unsupported filter version: %d", test.name,
test.version)
continue
}
@ -263,10 +286,15 @@ func TestFilter(t *testing.T) {
}
// Deserialize the filter from bytes with N parameter.
f3, err := FromNBytes(test.p, wantNBytes)
if err != nil {
t.Errorf("%q: unexpected err: %v", test.name, err)
continue
var f3 filterMatcher
switch test.version {
case 1:
tf3, err := FromNBytesV1(test.p, wantNBytes)
if err != nil {
t.Errorf("%q: unexpected err: %v", test.name, err)
continue
}
f3 = tf3
}
// Ensure all of the expected matches occur on the deserialized filter.
@ -285,7 +313,7 @@ func TestFilterMisses(t *testing.T) {
// Create a filter with the lowest supported false positive rate to reduce
// the chances of a false positive as much as possible.
var key [KeySize]byte
f, err := NewFilter(32, key, [][]byte{[]byte("entry")})
f, err := NewFilterV1(32, key, [][]byte{[]byte("entry")})
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
@ -326,19 +354,19 @@ func TestFilterCorners(t *testing.T) {
// Attempt to construct filter with parameters too large.
const largeP = 33
var key [KeySize]byte
_, err := NewFilter(largeP, key, nil)
_, err := NewFilterV1(largeP, key, nil)
if !IsErrorCode(err, ErrPTooBig) {
t.Fatalf("did not receive expected err for P too big -- got %v, want %v",
err, ErrPTooBig)
}
_, err = FromBytes(0, largeP, nil)
_, err = FromBytesV1(0, largeP, nil)
if !IsErrorCode(err, ErrPTooBig) {
t.Fatalf("did not receive expected err for P too big -- got %v, want %v",
err, ErrPTooBig)
}
// Attempt to decode a filter without the N value serialized properly.
_, err = FromNBytes(20, []byte{0x00})
_, err = FromNBytesV1(20, []byte{0x00})
if !IsErrorCode(err, ErrMisserialized) {
t.Fatalf("did not receive expected err -- got %v, want %v", err,
ErrMisserialized)

6
rpcclient/chain.go
View File

@ -787,7 +787,7 @@ type FutureGetCFilterResult chan *response
// Receive waits for the response promised by the future and returns the
// discovered rescan data.
func (r FutureGetCFilterResult) Receive() (*gcs.Filter, error) {
func (r FutureGetCFilterResult) Receive() (*gcs.FilterV1, error) {
res, err := receiveFuture(r)
if err != nil {
return nil, err
@ -803,7 +803,7 @@ func (r FutureGetCFilterResult) Receive() (*gcs.Filter, error) {
return nil, err
}
return gcs.FromNBytes(blockcf.P, filterNBytes)
return gcs.FromNBytesV1(blockcf.P, filterNBytes)
}
// GetCFilterAsync returns an instance of a type that can be used to get the
@ -827,7 +827,7 @@ func (c *Client) GetCFilterAsync(blockHash *chainhash.Hash, filterType wire.Filt
}
// GetCFilter returns the committed filter of type filterType for a block.
func (c *Client) GetCFilter(blockHash *chainhash.Hash, filterType wire.FilterType) (*gcs.Filter, error) {
func (c *Client) GetCFilter(blockHash *chainhash.Hash, filterType wire.FilterType) (*gcs.FilterV1, error) {
return c.GetCFilterAsync(blockHash, filterType).Receive()
}

2
server.go
View File

@ -957,7 +957,7 @@ func (sp *serverPeer) OnGetCFilter(p *peer.Peer, msg *wire.MsgGetCFilter) {
return
}
var f *gcs.Filter
var f *gcs.FilterV1
switch msg.FilterType {
case wire.GCSFilterRegular:
f, err = blockcf.Regular(block.MsgBlock())