From f07f320651fbaf4941ad60bf919c6dfed48a3457 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Mon, 30 Mar 2020 11:27:31 +0200 Subject: [PATCH] vendor: update go.etcd.io/bbolt v1.3.4 full diff: https://github.com/etcd-io/bbolt/compare/v1.3.3...v1.3.4 - Fix unsafe pointer conversions caught by Go 1.14 checkptr - Fix unexpected delete bucket error: "delete bucket: incompatible value" - Add support for aix - Add go.mod - db.Path() resolves to db.file.Name() Signed-off-by: Sebastiaan van Stijn --- vendor.conf | 2 +- vendor/go.etcd.io/bbolt/README.md | 4 +- vendor/go.etcd.io/bbolt/bolt_386.go | 3 - vendor/go.etcd.io/bbolt/bolt_amd64.go | 3 - vendor/go.etcd.io/bbolt/bolt_arm.go | 21 ------ vendor/go.etcd.io/bbolt/bolt_arm64.go | 3 - vendor/go.etcd.io/bbolt/bolt_mips64x.go | 3 - vendor/go.etcd.io/bbolt/bolt_mipsx.go | 3 - vendor/go.etcd.io/bbolt/bolt_ppc.go | 3 - vendor/go.etcd.io/bbolt/bolt_ppc64.go | 3 - vendor/go.etcd.io/bbolt/bolt_ppc64le.go | 3 - vendor/go.etcd.io/bbolt/bolt_riscv64.go | 3 - vendor/go.etcd.io/bbolt/bolt_s390x.go | 3 - vendor/go.etcd.io/bbolt/bolt_unix.go | 2 +- vendor/go.etcd.io/bbolt/bolt_unix_aix.go | 90 ++++++++++++++++++++++++ vendor/go.etcd.io/bbolt/bucket.go | 34 ++++----- vendor/go.etcd.io/bbolt/cursor.go | 2 +- vendor/go.etcd.io/bbolt/db.go | 4 +- vendor/go.etcd.io/bbolt/freelist.go | 37 +++++++--- vendor/go.etcd.io/bbolt/freelist_hmap.go | 2 +- vendor/go.etcd.io/bbolt/go.mod | 5 ++ vendor/go.etcd.io/bbolt/node.go | 61 ++++++++-------- vendor/go.etcd.io/bbolt/page.go | 56 ++++++++++----- vendor/go.etcd.io/bbolt/tx.go | 17 +++-- 24 files changed, 235 insertions(+), 132 deletions(-) create mode 100644 vendor/go.etcd.io/bbolt/bolt_unix_aix.go create mode 100644 vendor/go.etcd.io/bbolt/go.mod diff --git a/vendor.conf b/vendor.conf index 91ee6e918e..1517b567fb 100644 --- a/vendor.conf +++ b/vendor.conf @@ -64,7 +64,7 @@ github.com/ugorji/go b4c50a2b199d93b13dc15e78929c github.com/hashicorp/consul 9a9cc9341bb487651a0399e3fc5e1e8a42e62dd9 # v0.5.2 github.com/miekg/dns 6c0c4e6581f8e173cc562c8b3363ab984e4ae071 # v1.1.27 github.com/ishidawataru/sctp 6e2cb1366111dcf547c13531e3a263a067715847 -go.etcd.io/bbolt a0458a2b35708eef59eb5f620ceb3cd1c01a824d # v1.3.3 +go.etcd.io/bbolt 68cc10a767ea1c6b9e8dcb9847317ff192d6d974 # v1.3.4 # get graph and distribution packages github.com/docker/distribution 0d3efadf0154c2b8a4e7b6621fff9809655cc580 diff --git a/vendor/go.etcd.io/bbolt/README.md b/vendor/go.etcd.io/bbolt/README.md index e9989efc50..2dff3761da 100644 --- a/vendor/go.etcd.io/bbolt/README.md +++ b/vendor/go.etcd.io/bbolt/README.md @@ -275,7 +275,7 @@ should be writable. ### Using buckets Buckets are collections of key/value pairs within the database. All keys in a -bucket must be unique. You can create a bucket using the `DB.CreateBucket()` +bucket must be unique. You can create a bucket using the `Tx.CreateBucket()` function: ```go @@ -923,6 +923,7 @@ Below is a list of public, open source projects that use Bolt: * [GoWebApp](https://github.com/josephspurrier/gowebapp) - A basic MVC web application in Go using BoltDB. * [GoShort](https://github.com/pankajkhairnar/goShort) - GoShort is a URL shortener written in Golang and BoltDB for persistent key/value storage and for routing it's using high performent HTTPRouter. * [gopherpit](https://github.com/gopherpit/gopherpit) - A web service to manage Go remote import paths with custom domains +* [gokv](https://github.com/philippgille/gokv) - Simple key-value store abstraction and implementations for Go (Redis, Consul, etcd, bbolt, BadgerDB, LevelDB, Memcached, DynamoDB, S3, PostgreSQL, MongoDB, CockroachDB and many more) * [Gitchain](https://github.com/gitchain/gitchain) - Decentralized, peer-to-peer Git repositories aka "Git meets Bitcoin". * [InfluxDB](https://influxdata.com) - Scalable datastore for metrics, events, and real-time analytics. * [ipLocator](https://github.com/AndreasBriese/ipLocator) - A fast ip-geo-location-server using bolt with bloom filters. @@ -935,6 +936,7 @@ Below is a list of public, open source projects that use Bolt: * [mbuckets](https://github.com/abhigupta912/mbuckets) - A Bolt wrapper that allows easy operations on multi level (nested) buckets. * [MetricBase](https://github.com/msiebuhr/MetricBase) - Single-binary version of Graphite. * [MuLiFS](https://github.com/dankomiocevic/mulifs) - Music Library Filesystem creates a filesystem to organise your music files. +* [NATS](https://github.com/nats-io/nats-streaming-server) - NATS Streaming uses bbolt for message and metadata storage. * [Operation Go: A Routine Mission](http://gocode.io) - An online programming game for Golang using Bolt for user accounts and a leaderboard. * [photosite/session](https://godoc.org/bitbucket.org/kardianos/photosite/session) - Sessions for a photo viewing site. * [Prometheus Annotation Server](https://github.com/oliver006/prom_annotation_server) - Annotation server for PromDash & Prometheus service monitoring system. diff --git a/vendor/go.etcd.io/bbolt/bolt_386.go b/vendor/go.etcd.io/bbolt/bolt_386.go index 4d35ee7cf3..aee25960ff 100644 --- a/vendor/go.etcd.io/bbolt/bolt_386.go +++ b/vendor/go.etcd.io/bbolt/bolt_386.go @@ -5,6 +5,3 @@ const maxMapSize = 0x7FFFFFFF // 2GB // maxAllocSize is the size used when creating array pointers. const maxAllocSize = 0xFFFFFFF - -// Are unaligned load/stores broken on this arch? -var brokenUnaligned = false diff --git a/vendor/go.etcd.io/bbolt/bolt_amd64.go b/vendor/go.etcd.io/bbolt/bolt_amd64.go index 60a52dad56..5dd8f3f2ae 100644 --- a/vendor/go.etcd.io/bbolt/bolt_amd64.go +++ b/vendor/go.etcd.io/bbolt/bolt_amd64.go @@ -5,6 +5,3 @@ const maxMapSize = 0xFFFFFFFFFFFF // 256TB // maxAllocSize is the size used when creating array pointers. const maxAllocSize = 0x7FFFFFFF - -// Are unaligned load/stores broken on this arch? -var brokenUnaligned = false diff --git a/vendor/go.etcd.io/bbolt/bolt_arm.go b/vendor/go.etcd.io/bbolt/bolt_arm.go index 105d27ddb7..aee25960ff 100644 --- a/vendor/go.etcd.io/bbolt/bolt_arm.go +++ b/vendor/go.etcd.io/bbolt/bolt_arm.go @@ -1,28 +1,7 @@ package bbolt -import "unsafe" - // maxMapSize represents the largest mmap size supported by Bolt. const maxMapSize = 0x7FFFFFFF // 2GB // maxAllocSize is the size used when creating array pointers. const maxAllocSize = 0xFFFFFFF - -// Are unaligned load/stores broken on this arch? -var brokenUnaligned bool - -func init() { - // Simple check to see whether this arch handles unaligned load/stores - // correctly. - - // ARM9 and older devices require load/stores to be from/to aligned - // addresses. If not, the lower 2 bits are cleared and that address is - // read in a jumbled up order. - - // See http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.faqs/ka15414.html - - raw := [6]byte{0xfe, 0xef, 0x11, 0x22, 0x22, 0x11} - val := *(*uint32)(unsafe.Pointer(uintptr(unsafe.Pointer(&raw)) + 2)) - - brokenUnaligned = val != 0x11222211 -} diff --git a/vendor/go.etcd.io/bbolt/bolt_arm64.go b/vendor/go.etcd.io/bbolt/bolt_arm64.go index f5aa2a5ee2..810dfd55c5 100644 --- a/vendor/go.etcd.io/bbolt/bolt_arm64.go +++ b/vendor/go.etcd.io/bbolt/bolt_arm64.go @@ -7,6 +7,3 @@ const maxMapSize = 0xFFFFFFFFFFFF // 256TB // maxAllocSize is the size used when creating array pointers. const maxAllocSize = 0x7FFFFFFF - -// Are unaligned load/stores broken on this arch? -var brokenUnaligned = false diff --git a/vendor/go.etcd.io/bbolt/bolt_mips64x.go b/vendor/go.etcd.io/bbolt/bolt_mips64x.go index baeb289fd9..dd8ffe1239 100644 --- a/vendor/go.etcd.io/bbolt/bolt_mips64x.go +++ b/vendor/go.etcd.io/bbolt/bolt_mips64x.go @@ -7,6 +7,3 @@ const maxMapSize = 0x8000000000 // 512GB // maxAllocSize is the size used when creating array pointers. const maxAllocSize = 0x7FFFFFFF - -// Are unaligned load/stores broken on this arch? -var brokenUnaligned = false diff --git a/vendor/go.etcd.io/bbolt/bolt_mipsx.go b/vendor/go.etcd.io/bbolt/bolt_mipsx.go index 2d9b1a91f3..a669703a4e 100644 --- a/vendor/go.etcd.io/bbolt/bolt_mipsx.go +++ b/vendor/go.etcd.io/bbolt/bolt_mipsx.go @@ -7,6 +7,3 @@ const maxMapSize = 0x40000000 // 1GB // maxAllocSize is the size used when creating array pointers. const maxAllocSize = 0xFFFFFFF - -// Are unaligned load/stores broken on this arch? -var brokenUnaligned = false diff --git a/vendor/go.etcd.io/bbolt/bolt_ppc.go b/vendor/go.etcd.io/bbolt/bolt_ppc.go index 69804714aa..84e545ef3e 100644 --- a/vendor/go.etcd.io/bbolt/bolt_ppc.go +++ b/vendor/go.etcd.io/bbolt/bolt_ppc.go @@ -7,6 +7,3 @@ const maxMapSize = 0x7FFFFFFF // 2GB // maxAllocSize is the size used when creating array pointers. const maxAllocSize = 0xFFFFFFF - -// Are unaligned load/stores broken on this arch? -var brokenUnaligned = false diff --git a/vendor/go.etcd.io/bbolt/bolt_ppc64.go b/vendor/go.etcd.io/bbolt/bolt_ppc64.go index 3565908576..a76120908c 100644 --- a/vendor/go.etcd.io/bbolt/bolt_ppc64.go +++ b/vendor/go.etcd.io/bbolt/bolt_ppc64.go @@ -7,6 +7,3 @@ const maxMapSize = 0xFFFFFFFFFFFF // 256TB // maxAllocSize is the size used when creating array pointers. const maxAllocSize = 0x7FFFFFFF - -// Are unaligned load/stores broken on this arch? -var brokenUnaligned = false diff --git a/vendor/go.etcd.io/bbolt/bolt_ppc64le.go b/vendor/go.etcd.io/bbolt/bolt_ppc64le.go index 422c7c69d6..c830f2fc77 100644 --- a/vendor/go.etcd.io/bbolt/bolt_ppc64le.go +++ b/vendor/go.etcd.io/bbolt/bolt_ppc64le.go @@ -7,6 +7,3 @@ const maxMapSize = 0xFFFFFFFFFFFF // 256TB // maxAllocSize is the size used when creating array pointers. const maxAllocSize = 0x7FFFFFFF - -// Are unaligned load/stores broken on this arch? -var brokenUnaligned = false diff --git a/vendor/go.etcd.io/bbolt/bolt_riscv64.go b/vendor/go.etcd.io/bbolt/bolt_riscv64.go index 07b4b47cdb..c967613b00 100644 --- a/vendor/go.etcd.io/bbolt/bolt_riscv64.go +++ b/vendor/go.etcd.io/bbolt/bolt_riscv64.go @@ -7,6 +7,3 @@ const maxMapSize = 0xFFFFFFFFFFFF // 256TB // maxAllocSize is the size used when creating array pointers. const maxAllocSize = 0x7FFFFFFF - -// Are unaligned load/stores broken on this arch? -var brokenUnaligned = true diff --git a/vendor/go.etcd.io/bbolt/bolt_s390x.go b/vendor/go.etcd.io/bbolt/bolt_s390x.go index 6d3fcb825d..ff2a560970 100644 --- a/vendor/go.etcd.io/bbolt/bolt_s390x.go +++ b/vendor/go.etcd.io/bbolt/bolt_s390x.go @@ -7,6 +7,3 @@ const maxMapSize = 0xFFFFFFFFFFFF // 256TB // maxAllocSize is the size used when creating array pointers. const maxAllocSize = 0x7FFFFFFF - -// Are unaligned load/stores broken on this arch? -var brokenUnaligned = false diff --git a/vendor/go.etcd.io/bbolt/bolt_unix.go b/vendor/go.etcd.io/bbolt/bolt_unix.go index 5f2bb51451..2938fed584 100644 --- a/vendor/go.etcd.io/bbolt/bolt_unix.go +++ b/vendor/go.etcd.io/bbolt/bolt_unix.go @@ -1,4 +1,4 @@ -// +build !windows,!plan9,!solaris +// +build !windows,!plan9,!solaris,!aix package bbolt diff --git a/vendor/go.etcd.io/bbolt/bolt_unix_aix.go b/vendor/go.etcd.io/bbolt/bolt_unix_aix.go new file mode 100644 index 0000000000..a64c16f512 --- /dev/null +++ b/vendor/go.etcd.io/bbolt/bolt_unix_aix.go @@ -0,0 +1,90 @@ +// +build aix + +package bbolt + +import ( + "fmt" + "syscall" + "time" + "unsafe" + + "golang.org/x/sys/unix" +) + +// flock acquires an advisory lock on a file descriptor. +func flock(db *DB, exclusive bool, timeout time.Duration) error { + var t time.Time + if timeout != 0 { + t = time.Now() + } + fd := db.file.Fd() + var lockType int16 + if exclusive { + lockType = syscall.F_WRLCK + } else { + lockType = syscall.F_RDLCK + } + for { + // Attempt to obtain an exclusive lock. + lock := syscall.Flock_t{Type: lockType} + err := syscall.FcntlFlock(fd, syscall.F_SETLK, &lock) + if err == nil { + return nil + } else if err != syscall.EAGAIN { + return err + } + + // If we timed out then return an error. + if timeout != 0 && time.Since(t) > timeout-flockRetryTimeout { + return ErrTimeout + } + + // Wait for a bit and try again. + time.Sleep(flockRetryTimeout) + } +} + +// funlock releases an advisory lock on a file descriptor. +func funlock(db *DB) error { + var lock syscall.Flock_t + lock.Start = 0 + lock.Len = 0 + lock.Type = syscall.F_UNLCK + lock.Whence = 0 + return syscall.FcntlFlock(uintptr(db.file.Fd()), syscall.F_SETLK, &lock) +} + +// mmap memory maps a DB's data file. +func mmap(db *DB, sz int) error { + // Map the data file to memory. + b, err := unix.Mmap(int(db.file.Fd()), 0, sz, syscall.PROT_READ, syscall.MAP_SHARED|db.MmapFlags) + if err != nil { + return err + } + + // Advise the kernel that the mmap is accessed randomly. + if err := unix.Madvise(b, syscall.MADV_RANDOM); err != nil { + return fmt.Errorf("madvise: %s", err) + } + + // Save the original byte slice and convert to a byte array pointer. + db.dataref = b + db.data = (*[maxMapSize]byte)(unsafe.Pointer(&b[0])) + db.datasz = sz + return nil +} + +// munmap unmaps a DB's data file from memory. +func munmap(db *DB) error { + // Ignore the unmap if we have no mapped data. + if db.dataref == nil { + return nil + } + + // Unmap using the original byte slice. + err := unix.Munmap(db.dataref) + db.dataref = nil + db.data = nil + db.datasz = 0 + return err +} diff --git a/vendor/go.etcd.io/bbolt/bucket.go b/vendor/go.etcd.io/bbolt/bucket.go index 84bfd4d6a2..d8750b1487 100644 --- a/vendor/go.etcd.io/bbolt/bucket.go +++ b/vendor/go.etcd.io/bbolt/bucket.go @@ -123,10 +123,12 @@ func (b *Bucket) Bucket(name []byte) *Bucket { func (b *Bucket) openBucket(value []byte) *Bucket { var child = newBucket(b.tx) - // If unaligned load/stores are broken on this arch and value is - // unaligned simply clone to an aligned byte array. - unaligned := brokenUnaligned && uintptr(unsafe.Pointer(&value[0]))&3 != 0 - + // Unaligned access requires a copy to be made. + const unalignedMask = unsafe.Alignof(struct { + bucket + page + }{}) - 1 + unaligned := uintptr(unsafe.Pointer(&value[0]))&unalignedMask != 0 if unaligned { value = cloneBytes(value) } @@ -206,7 +208,7 @@ func (b *Bucket) CreateBucketIfNotExists(key []byte) (*Bucket, error) { } // DeleteBucket deletes a bucket at the given key. -// Returns an error if the bucket does not exists, or if the key represents a non-bucket value. +// Returns an error if the bucket does not exist, or if the key represents a non-bucket value. func (b *Bucket) DeleteBucket(key []byte) error { if b.tx.db == nil { return ErrTxClosed @@ -228,7 +230,7 @@ func (b *Bucket) DeleteBucket(key []byte) error { // Recursively delete all child buckets. child := b.Bucket(key) err := child.ForEach(func(k, v []byte) error { - if v == nil { + if _, _, childFlags := child.Cursor().seek(k); (childFlags & bucketLeafFlag) != 0 { if err := child.DeleteBucket(k); err != nil { return fmt.Errorf("delete bucket: %s", err) } @@ -409,7 +411,7 @@ func (b *Bucket) Stats() BucketStats { if p.count != 0 { // If page has any elements, add all element headers. - used += leafPageElementSize * int(p.count-1) + used += leafPageElementSize * uintptr(p.count-1) // Add all element key, value sizes. // The computation takes advantage of the fact that the position @@ -417,16 +419,16 @@ func (b *Bucket) Stats() BucketStats { // of all previous elements' keys and values. // It also includes the last element's header. lastElement := p.leafPageElement(p.count - 1) - used += int(lastElement.pos + lastElement.ksize + lastElement.vsize) + used += uintptr(lastElement.pos + lastElement.ksize + lastElement.vsize) } if b.root == 0 { // For inlined bucket just update the inline stats - s.InlineBucketInuse += used + s.InlineBucketInuse += int(used) } else { // For non-inlined bucket update all the leaf stats s.LeafPageN++ - s.LeafInuse += used + s.LeafInuse += int(used) s.LeafOverflowN += int(p.overflow) // Collect stats from sub-buckets. @@ -447,13 +449,13 @@ func (b *Bucket) Stats() BucketStats { // used totals the used bytes for the page // Add header and all element headers. - used := pageHeaderSize + (branchPageElementSize * int(p.count-1)) + used := pageHeaderSize + (branchPageElementSize * uintptr(p.count-1)) // Add size of all keys and values. // Again, use the fact that last element's position equals to // the total of key, value sizes of all previous elements. - used += int(lastElement.pos + lastElement.ksize) - s.BranchInuse += used + used += uintptr(lastElement.pos + lastElement.ksize) + s.BranchInuse += int(used) s.BranchOverflowN += int(p.overflow) } @@ -593,7 +595,7 @@ func (b *Bucket) inlineable() bool { // our threshold for inline bucket size. var size = pageHeaderSize for _, inode := range n.inodes { - size += leafPageElementSize + len(inode.key) + len(inode.value) + size += leafPageElementSize + uintptr(len(inode.key)) + uintptr(len(inode.value)) if inode.flags&bucketLeafFlag != 0 { return false @@ -606,8 +608,8 @@ func (b *Bucket) inlineable() bool { } // Returns the maximum total size of a bucket to make it a candidate for inlining. -func (b *Bucket) maxInlineBucketSize() int { - return b.tx.db.pageSize / 4 +func (b *Bucket) maxInlineBucketSize() uintptr { + return uintptr(b.tx.db.pageSize / 4) } // write allocates and writes a bucket to a byte slice. diff --git a/vendor/go.etcd.io/bbolt/cursor.go b/vendor/go.etcd.io/bbolt/cursor.go index 3000aced6c..98aeb449a4 100644 --- a/vendor/go.etcd.io/bbolt/cursor.go +++ b/vendor/go.etcd.io/bbolt/cursor.go @@ -366,7 +366,7 @@ func (c *Cursor) node() *node { } for _, ref := range c.stack[:len(c.stack)-1] { _assert(!n.isLeaf, "expected branch node") - n = n.childAt(int(ref.index)) + n = n.childAt(ref.index) } _assert(n.isLeaf, "expected leaf node") return n diff --git a/vendor/go.etcd.io/bbolt/db.go b/vendor/go.etcd.io/bbolt/db.go index 870c8b1cc9..80b0095cc3 100644 --- a/vendor/go.etcd.io/bbolt/db.go +++ b/vendor/go.etcd.io/bbolt/db.go @@ -206,12 +206,12 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) { } // Open data file and separate sync handler for metadata writes. - db.path = path var err error - if db.file, err = db.openFile(db.path, flag|os.O_CREATE, mode); err != nil { + if db.file, err = db.openFile(path, flag|os.O_CREATE, mode); err != nil { _ = db.close() return nil, err } + db.path = db.file.Name() // Lock file so that other processes using Bolt in read-write mode cannot // use the database at the same time. This would cause corruption since diff --git a/vendor/go.etcd.io/bbolt/freelist.go b/vendor/go.etcd.io/bbolt/freelist.go index 587b8cc02d..d441b69256 100644 --- a/vendor/go.etcd.io/bbolt/freelist.go +++ b/vendor/go.etcd.io/bbolt/freelist.go @@ -2,6 +2,7 @@ package bbolt import ( "fmt" + "reflect" "sort" "unsafe" ) @@ -71,7 +72,7 @@ func (f *freelist) size() int { // The first element will be used to store the count. See freelist.write. n++ } - return pageHeaderSize + (int(unsafe.Sizeof(pgid(0))) * n) + return int(pageHeaderSize) + (int(unsafe.Sizeof(pgid(0))) * n) } // count returns count of pages on the freelist @@ -93,8 +94,24 @@ func (f *freelist) pending_count() int { return count } -// copyall copies into dst a list of all free ids and all pending ids in one sorted list. +// copyallunsafe copies a list of all free ids and all pending ids in one sorted list. // f.count returns the minimum length required for dst. +func (f *freelist) copyallunsafe(dstptr unsafe.Pointer) { // dstptr is []pgid data pointer + m := make(pgids, 0, f.pending_count()) + for _, txp := range f.pending { + m = append(m, txp.ids...) + } + sort.Sort(m) + fpgids := f.getFreePageIDs() + sz := len(fpgids) + len(m) + dst := *(*[]pgid)(unsafe.Pointer(&reflect.SliceHeader{ + Data: uintptr(dstptr), + Len: sz, + Cap: sz, + })) + mergepgids(dst, fpgids, m) +} + func (f *freelist) copyall(dst []pgid) { m := make(pgids, 0, f.pending_count()) for _, txp := range f.pending { @@ -267,17 +284,21 @@ func (f *freelist) read(p *page) { } // If the page.count is at the max uint16 value (64k) then it's considered // an overflow and the size of the freelist is stored as the first element. - idx, count := 0, int(p.count) + var idx, count uintptr = 0, uintptr(p.count) if count == 0xFFFF { idx = 1 - count = int(((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[0]) + count = uintptr(*(*pgid)(unsafe.Pointer(uintptr(unsafe.Pointer(p)) + unsafe.Sizeof(*p)))) } // Copy the list of page ids from the freelist. if count == 0 { f.ids = nil } else { - ids := ((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[idx : idx+count] + ids := *(*[]pgid)(unsafe.Pointer(&reflect.SliceHeader{ + Data: uintptr(unsafe.Pointer(p)) + unsafe.Sizeof(*p) + idx*unsafe.Sizeof(pgid(0)), + Len: int(count), + Cap: int(count), + })) // copy the ids, so we don't modify on the freelist page directly idsCopy := make([]pgid, count) @@ -315,11 +336,11 @@ func (f *freelist) write(p *page) error { p.count = uint16(lenids) } else if lenids < 0xFFFF { p.count = uint16(lenids) - f.copyall(((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[:]) + f.copyallunsafe(unsafe.Pointer(uintptr(unsafe.Pointer(p)) + unsafe.Sizeof(*p))) } else { p.count = 0xFFFF - ((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[0] = pgid(lenids) - f.copyall(((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[1:]) + *(*pgid)(unsafe.Pointer(uintptr(unsafe.Pointer(p)) + unsafe.Sizeof(*p))) = pgid(lenids) + f.copyallunsafe(unsafe.Pointer(uintptr(unsafe.Pointer(p)) + unsafe.Sizeof(*p) + unsafe.Sizeof(pgid(0)))) } return nil diff --git a/vendor/go.etcd.io/bbolt/freelist_hmap.go b/vendor/go.etcd.io/bbolt/freelist_hmap.go index 6a03a6c3c8..02ef2be044 100644 --- a/vendor/go.etcd.io/bbolt/freelist_hmap.go +++ b/vendor/go.etcd.io/bbolt/freelist_hmap.go @@ -27,7 +27,7 @@ func (f *freelist) hashmapAllocate(txid txid, n int) pgid { f.allocs[pid] = txid for i := pgid(0); i < pgid(n); i++ { - delete(f.cache, pid+pgid(i)) + delete(f.cache, pid+i) } return pid } diff --git a/vendor/go.etcd.io/bbolt/go.mod b/vendor/go.etcd.io/bbolt/go.mod new file mode 100644 index 0000000000..c2366daef6 --- /dev/null +++ b/vendor/go.etcd.io/bbolt/go.mod @@ -0,0 +1,5 @@ +module go.etcd.io/bbolt + +go 1.12 + +require golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 diff --git a/vendor/go.etcd.io/bbolt/node.go b/vendor/go.etcd.io/bbolt/node.go index 6c3fa553ea..1690eef3f7 100644 --- a/vendor/go.etcd.io/bbolt/node.go +++ b/vendor/go.etcd.io/bbolt/node.go @@ -3,6 +3,7 @@ package bbolt import ( "bytes" "fmt" + "reflect" "sort" "unsafe" ) @@ -41,19 +42,19 @@ func (n *node) size() int { sz, elsz := pageHeaderSize, n.pageElementSize() for i := 0; i < len(n.inodes); i++ { item := &n.inodes[i] - sz += elsz + len(item.key) + len(item.value) + sz += elsz + uintptr(len(item.key)) + uintptr(len(item.value)) } - return sz + return int(sz) } // sizeLessThan returns true if the node is less than a given size. // This is an optimization to avoid calculating a large node when we only need // to know if it fits inside a certain page size. -func (n *node) sizeLessThan(v int) bool { +func (n *node) sizeLessThan(v uintptr) bool { sz, elsz := pageHeaderSize, n.pageElementSize() for i := 0; i < len(n.inodes); i++ { item := &n.inodes[i] - sz += elsz + len(item.key) + len(item.value) + sz += elsz + uintptr(len(item.key)) + uintptr(len(item.value)) if sz >= v { return false } @@ -62,7 +63,7 @@ func (n *node) sizeLessThan(v int) bool { } // pageElementSize returns the size of each page element based on the type of node. -func (n *node) pageElementSize() int { +func (n *node) pageElementSize() uintptr { if n.isLeaf { return leafPageElementSize } @@ -207,39 +208,39 @@ func (n *node) write(p *page) { } // Loop over each item and write it to the page. - b := (*[maxAllocSize]byte)(unsafe.Pointer(&p.ptr))[n.pageElementSize()*len(n.inodes):] + bp := uintptr(unsafe.Pointer(p)) + unsafe.Sizeof(*p) + n.pageElementSize()*uintptr(len(n.inodes)) for i, item := range n.inodes { _assert(len(item.key) > 0, "write: zero-length inode key") // Write the page element. if n.isLeaf { elem := p.leafPageElement(uint16(i)) - elem.pos = uint32(uintptr(unsafe.Pointer(&b[0])) - uintptr(unsafe.Pointer(elem))) + elem.pos = uint32(bp - uintptr(unsafe.Pointer(elem))) elem.flags = item.flags elem.ksize = uint32(len(item.key)) elem.vsize = uint32(len(item.value)) } else { elem := p.branchPageElement(uint16(i)) - elem.pos = uint32(uintptr(unsafe.Pointer(&b[0])) - uintptr(unsafe.Pointer(elem))) + elem.pos = uint32(bp - uintptr(unsafe.Pointer(elem))) elem.ksize = uint32(len(item.key)) elem.pgid = item.pgid _assert(elem.pgid != p.id, "write: circular dependency occurred") } - // If the length of key+value is larger than the max allocation size - // then we need to reallocate the byte array pointer. - // - // See: https://github.com/boltdb/bolt/pull/335 + // Create a slice to write into of needed size and advance + // byte pointer for next iteration. klen, vlen := len(item.key), len(item.value) - if len(b) < klen+vlen { - b = (*[maxAllocSize]byte)(unsafe.Pointer(&b[0]))[:] - } + sz := klen + vlen + b := *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{ + Data: bp, + Len: sz, + Cap: sz, + })) + bp += uintptr(sz) // Write data for the element to the end of the page. - copy(b[0:], item.key) - b = b[klen:] - copy(b[0:], item.value) - b = b[vlen:] + l := copy(b, item.key) + copy(b[l:], item.value) } // DEBUG ONLY: n.dump() @@ -247,7 +248,7 @@ func (n *node) write(p *page) { // split breaks up a node into multiple smaller nodes, if appropriate. // This should only be called from the spill() function. -func (n *node) split(pageSize int) []*node { +func (n *node) split(pageSize uintptr) []*node { var nodes []*node node := n @@ -270,7 +271,7 @@ func (n *node) split(pageSize int) []*node { // splitTwo breaks up a node into two smaller nodes, if appropriate. // This should only be called from the split() function. -func (n *node) splitTwo(pageSize int) (*node, *node) { +func (n *node) splitTwo(pageSize uintptr) (*node, *node) { // Ignore the split if the page doesn't have at least enough nodes for // two pages or if the nodes can fit in a single page. if len(n.inodes) <= (minKeysPerPage*2) || n.sizeLessThan(pageSize) { @@ -312,18 +313,18 @@ func (n *node) splitTwo(pageSize int) (*node, *node) { // splitIndex finds the position where a page will fill a given threshold. // It returns the index as well as the size of the first page. // This is only be called from split(). -func (n *node) splitIndex(threshold int) (index, sz int) { +func (n *node) splitIndex(threshold int) (index, sz uintptr) { sz = pageHeaderSize // Loop until we only have the minimum number of keys required for the second page. for i := 0; i < len(n.inodes)-minKeysPerPage; i++ { - index = i + index = uintptr(i) inode := n.inodes[i] - elsize := n.pageElementSize() + len(inode.key) + len(inode.value) + elsize := n.pageElementSize() + uintptr(len(inode.key)) + uintptr(len(inode.value)) // If we have at least the minimum number of keys and adding another // node would put us over the threshold then exit and return. - if i >= minKeysPerPage && sz+elsize > threshold { + if index >= minKeysPerPage && sz+elsize > uintptr(threshold) { break } @@ -356,7 +357,7 @@ func (n *node) spill() error { n.children = nil // Split nodes into appropriate sizes. The first node will always be n. - var nodes = n.split(tx.db.pageSize) + var nodes = n.split(uintptr(tx.db.pageSize)) for _, node := range nodes { // Add node's page to the freelist if it's not new. if node.pgid > 0 { @@ -587,9 +588,11 @@ func (n *node) dump() { type nodes []*node -func (s nodes) Len() int { return len(s) } -func (s nodes) Swap(i, j int) { s[i], s[j] = s[j], s[i] } -func (s nodes) Less(i, j int) bool { return bytes.Compare(s[i].inodes[0].key, s[j].inodes[0].key) == -1 } +func (s nodes) Len() int { return len(s) } +func (s nodes) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s nodes) Less(i, j int) bool { + return bytes.Compare(s[i].inodes[0].key, s[j].inodes[0].key) == -1 +} // inode represents an internal node inside of a node. // It can be used to point to elements in a page or point diff --git a/vendor/go.etcd.io/bbolt/page.go b/vendor/go.etcd.io/bbolt/page.go index bca9615f0f..b5c1699789 100644 --- a/vendor/go.etcd.io/bbolt/page.go +++ b/vendor/go.etcd.io/bbolt/page.go @@ -3,16 +3,17 @@ package bbolt import ( "fmt" "os" + "reflect" "sort" "unsafe" ) -const pageHeaderSize = int(unsafe.Offsetof(((*page)(nil)).ptr)) +const pageHeaderSize = unsafe.Sizeof(page{}) const minKeysPerPage = 2 -const branchPageElementSize = int(unsafe.Sizeof(branchPageElement{})) -const leafPageElementSize = int(unsafe.Sizeof(leafPageElement{})) +const branchPageElementSize = unsafe.Sizeof(branchPageElement{}) +const leafPageElementSize = unsafe.Sizeof(leafPageElement{}) const ( branchPageFlag = 0x01 @@ -32,7 +33,6 @@ type page struct { flags uint16 count uint16 overflow uint32 - ptr uintptr } // typ returns a human readable page type string used for debugging. @@ -51,13 +51,13 @@ func (p *page) typ() string { // meta returns a pointer to the metadata section of the page. func (p *page) meta() *meta { - return (*meta)(unsafe.Pointer(&p.ptr)) + return (*meta)(unsafe.Pointer(uintptr(unsafe.Pointer(p)) + unsafe.Sizeof(*p))) } // leafPageElement retrieves the leaf node by index func (p *page) leafPageElement(index uint16) *leafPageElement { - n := &((*[0x7FFFFFF]leafPageElement)(unsafe.Pointer(&p.ptr)))[index] - return n + off := uintptr(index) * unsafe.Sizeof(leafPageElement{}) + return (*leafPageElement)(unsafe.Pointer(uintptr(unsafe.Pointer(p)) + unsafe.Sizeof(*p) + off)) } // leafPageElements retrieves a list of leaf nodes. @@ -65,12 +65,17 @@ func (p *page) leafPageElements() []leafPageElement { if p.count == 0 { return nil } - return ((*[0x7FFFFFF]leafPageElement)(unsafe.Pointer(&p.ptr)))[:] + return *(*[]leafPageElement)(unsafe.Pointer(&reflect.SliceHeader{ + Data: uintptr(unsafe.Pointer(p)) + unsafe.Sizeof(*p), + Len: int(p.count), + Cap: int(p.count), + })) } // branchPageElement retrieves the branch node by index func (p *page) branchPageElement(index uint16) *branchPageElement { - return &((*[0x7FFFFFF]branchPageElement)(unsafe.Pointer(&p.ptr)))[index] + off := uintptr(index) * unsafe.Sizeof(branchPageElement{}) + return (*branchPageElement)(unsafe.Pointer(uintptr(unsafe.Pointer(p)) + unsafe.Sizeof(*p) + off)) } // branchPageElements retrieves a list of branch nodes. @@ -78,12 +83,20 @@ func (p *page) branchPageElements() []branchPageElement { if p.count == 0 { return nil } - return ((*[0x7FFFFFF]branchPageElement)(unsafe.Pointer(&p.ptr)))[:] + return *(*[]branchPageElement)(unsafe.Pointer(&reflect.SliceHeader{ + Data: uintptr(unsafe.Pointer(p)) + unsafe.Sizeof(*p), + Len: int(p.count), + Cap: int(p.count), + })) } // dump writes n bytes of the page to STDERR as hex output. func (p *page) hexdump(n int) { - buf := (*[maxAllocSize]byte)(unsafe.Pointer(p))[:n] + buf := *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{ + Data: uintptr(unsafe.Pointer(p)), + Len: n, + Cap: n, + })) fmt.Fprintf(os.Stderr, "%x\n", buf) } @@ -102,8 +115,11 @@ type branchPageElement struct { // key returns a byte slice of the node key. func (n *branchPageElement) key() []byte { - buf := (*[maxAllocSize]byte)(unsafe.Pointer(n)) - return (*[maxAllocSize]byte)(unsafe.Pointer(&buf[n.pos]))[:n.ksize] + return *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{ + Data: uintptr(unsafe.Pointer(n)) + uintptr(n.pos), + Len: int(n.ksize), + Cap: int(n.ksize), + })) } // leafPageElement represents a node on a leaf page. @@ -116,14 +132,20 @@ type leafPageElement struct { // key returns a byte slice of the node key. func (n *leafPageElement) key() []byte { - buf := (*[maxAllocSize]byte)(unsafe.Pointer(n)) - return (*[maxAllocSize]byte)(unsafe.Pointer(&buf[n.pos]))[:n.ksize:n.ksize] + return *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{ + Data: uintptr(unsafe.Pointer(n)) + uintptr(n.pos), + Len: int(n.ksize), + Cap: int(n.ksize), + })) } // value returns a byte slice of the node value. func (n *leafPageElement) value() []byte { - buf := (*[maxAllocSize]byte)(unsafe.Pointer(n)) - return (*[maxAllocSize]byte)(unsafe.Pointer(&buf[n.pos+n.ksize]))[:n.vsize:n.vsize] + return *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{ + Data: uintptr(unsafe.Pointer(n)) + uintptr(n.pos) + uintptr(n.ksize), + Len: int(n.vsize), + Cap: int(n.vsize), + })) } // PageInfo represents human readable information about a page. diff --git a/vendor/go.etcd.io/bbolt/tx.go b/vendor/go.etcd.io/bbolt/tx.go index 2df7688c2f..13937cdbf6 100644 --- a/vendor/go.etcd.io/bbolt/tx.go +++ b/vendor/go.etcd.io/bbolt/tx.go @@ -4,6 +4,7 @@ import ( "fmt" "io" "os" + "reflect" "sort" "strings" "time" @@ -527,7 +528,7 @@ func (tx *Tx) write() error { offset := int64(p.id) * int64(tx.db.pageSize) // Write out page in "max allocation" sized chunks. - ptr := (*[maxAllocSize]byte)(unsafe.Pointer(p)) + ptr := uintptr(unsafe.Pointer(p)) for { // Limit our write to our max allocation size. sz := size @@ -536,7 +537,11 @@ func (tx *Tx) write() error { } // Write chunk to disk. - buf := ptr[:sz] + buf := *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{ + Data: ptr, + Len: sz, + Cap: sz, + })) if _, err := tx.db.ops.writeAt(buf, offset); err != nil { return err } @@ -552,7 +557,7 @@ func (tx *Tx) write() error { // Otherwise move offset forward and move pointer to next chunk. offset += int64(sz) - ptr = (*[maxAllocSize]byte)(unsafe.Pointer(&ptr[sz])) + ptr += uintptr(sz) } } @@ -571,7 +576,11 @@ func (tx *Tx) write() error { continue } - buf := (*[maxAllocSize]byte)(unsafe.Pointer(p))[:tx.db.pageSize] + buf := *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{ + Data: uintptr(unsafe.Pointer(p)), + Len: tx.db.pageSize, + Cap: tx.db.pageSize, + })) // See https://go.googlesource.com/go/+/f03c9202c43e0abb130669852082117ca50aa9b1 for i := range buf {