Merge pull request #44864 from corhere/libnet/split-bitseq

libnetwork: split package bitseq

libnetwork/bitmap/sequence.go

@@ -0,0 +1,600 @@
+// Package bitmap provides a datatype for long vectors of bits.
+package bitmap
+
+import (
+	"encoding/binary"
+	"encoding/json"
+	"errors"
+	"fmt"
+)
+
+// block sequence constants
+// If needed we can think of making these configurable
+const (
+	blockLen      = uint32(32)
+	blockBytes    = uint64(blockLen / 8)
+	blockMAX      = uint32(1<<blockLen - 1)
+	blockFirstBit = uint32(1) << (blockLen - 1)
+	invalidPos    = uint64(0xFFFFFFFFFFFFFFFF)
+)
+
+var (
+	// ErrNoBitAvailable is returned when no more bits are available to set
+	ErrNoBitAvailable = errors.New("no bit available")
+	// ErrBitAllocated is returned when the specific bit requested is already set
+	ErrBitAllocated = errors.New("requested bit is already allocated")
+)
+
+// https://github.com/golang/go/issues/8005#issuecomment-190753527
+type noCopy struct{}
+
+func (noCopy) Lock() {}
+
+// Bitmap is a fixed-length bit vector. It is not safe for concurrent use.
+//
+// The data is stored as a list of run-length encoded blocks. It operates
+// directly on the encoded representation, without decompressing.
+type Bitmap struct {
+	bits       uint64
+	unselected uint64
+	head       *sequence
+	curr       uint64
+
+	// Shallow copies would share the same head pointer but a copy of the
+	// unselected count. Mutating the sequence through one would change the
+	// bits for all copies but only update that one copy's unselected count,
+	// which would result in subtle bugs.
+	noCopy noCopy
+}
+
+// NewHandle returns a new Bitmap n bits long.
+func New(n uint64) *Bitmap {
+	return &Bitmap{
+		bits:       n,
+		unselected: n,
+		head: &sequence{
+			block: 0x0,
+			count: getNumBlocks(n),
+		},
+	}
+}
+
+// Copy returns a deep copy of b.
+func Copy(b *Bitmap) *Bitmap {
+	return &Bitmap{
+		bits:       b.bits,
+		unselected: b.unselected,
+		head:       b.head.getCopy(),
+		curr:       b.curr,
+	}
+}
+
+// sequence represents a recurring sequence of 32 bits long bitmasks
+type sequence struct {
+	block uint32    // block is a symbol representing 4 byte long allocation bitmask
+	count uint64    // number of consecutive blocks (symbols)
+	next  *sequence // next sequence
+}
+
+// String returns a string representation of the block sequence starting from this block
+func (s *sequence) toString() string {
+	var nextBlock string
+	if s.next == nil {
+		nextBlock = "end"
+	} else {
+		nextBlock = s.next.toString()
+	}
+	return fmt.Sprintf("(0x%x, %d)->%s", s.block, s.count, nextBlock)
+}
+
+// GetAvailableBit returns the position of the first unset bit in the bitmask represented by this sequence
+func (s *sequence) getAvailableBit(from uint64) (uint64, uint64, error) {
+	if s.block == blockMAX || s.count == 0 {
+		return invalidPos, invalidPos, ErrNoBitAvailable
+	}
+	bits := from
+	bitSel := blockFirstBit >> from
+	for bitSel > 0 && s.block&bitSel != 0 {
+		bitSel >>= 1
+		bits++
+	}
+	// Check if the loop exited because it could not
+	// find any available bit int block  starting from
+	// "from". Return invalid pos in that case.
+	if bitSel == 0 {
+		return invalidPos, invalidPos, ErrNoBitAvailable
+	}
+	return bits / 8, bits % 8, nil
+}
+
+// GetCopy returns a copy of the linked list rooted at this node
+func (s *sequence) getCopy() *sequence {
+	n := &sequence{block: s.block, count: s.count}
+	pn := n
+	ps := s.next
+	for ps != nil {
+		pn.next = &sequence{block: ps.block, count: ps.count}
+		pn = pn.next
+		ps = ps.next
+	}
+	return n
+}
+
+// Equal checks if this sequence is equal to the passed one
+func (s *sequence) equal(o *sequence) bool {
+	this := s
+	other := o
+	for this != nil {
+		if other == nil {
+			return false
+		}
+		if this.block != other.block || this.count != other.count {
+			return false
+		}
+		this = this.next
+		other = other.next
+	}
+	return other == nil
+}
+
+// ToByteArray converts the sequence into a byte array
+func (s *sequence) toByteArray() ([]byte, error) {
+	var bb []byte
+
+	p := s
+	b := make([]byte, 12)
+	for p != nil {
+		binary.BigEndian.PutUint32(b[0:], p.block)
+		binary.BigEndian.PutUint64(b[4:], p.count)
+		bb = append(bb, b...)
+		p = p.next
+	}
+
+	return bb, nil
+}
+
+// fromByteArray construct the sequence from the byte array
+func (s *sequence) fromByteArray(data []byte) error {
+	l := len(data)
+	if l%12 != 0 {
+		return fmt.Errorf("cannot deserialize byte sequence of length %d (%v)", l, data)
+	}
+
+	p := s
+	i := 0
+	for {
+		p.block = binary.BigEndian.Uint32(data[i : i+4])
+		p.count = binary.BigEndian.Uint64(data[i+4 : i+12])
+		i += 12
+		if i == l {
+			break
+		}
+		p.next = &sequence{}
+		p = p.next
+	}
+
+	return nil
+}
+
+// SetAnyInRange sets the first unset bit in the range [start, end) and returns
+// the ordinal of the set bit.
+//
+// When serial=true, the bitmap is scanned starting from the ordinal following
+// the bit most recently set by [Bitmap.SetAny] or [Bitmap.SetAnyInRange].
+func (h *Bitmap) SetAnyInRange(start, end uint64, serial bool) (uint64, error) {
+	if end < start || end >= h.bits {
+		return invalidPos, fmt.Errorf("invalid bit range [%d, %d)", start, end)
+	}
+	if h.Unselected() == 0 {
+		return invalidPos, ErrNoBitAvailable
+	}
+	return h.set(0, start, end, true, false, serial)
+}
+
+// SetAny sets the first unset bit in the sequence and returns the ordinal of
+// the set bit.
+//
+// When serial=true, the bitmap is scanned starting from the ordinal following
+// the bit most recently set by [Bitmap.SetAny] or [Bitmap.SetAnyInRange].
+func (h *Bitmap) SetAny(serial bool) (uint64, error) {
+	if h.Unselected() == 0 {
+		return invalidPos, ErrNoBitAvailable
+	}
+	return h.set(0, 0, h.bits-1, true, false, serial)
+}
+
+// Set atomically sets the corresponding bit in the sequence
+func (h *Bitmap) Set(ordinal uint64) error {
+	if err := h.validateOrdinal(ordinal); err != nil {
+		return err
+	}
+	_, err := h.set(ordinal, 0, 0, false, false, false)
+	return err
+}
+
+// Unset atomically unsets the corresponding bit in the sequence
+func (h *Bitmap) Unset(ordinal uint64) error {
+	if err := h.validateOrdinal(ordinal); err != nil {
+		return err
+	}
+	_, err := h.set(ordinal, 0, 0, false, true, false)
+	return err
+}
+
+// IsSet atomically checks if the ordinal bit is set. In case ordinal
+// is outside of the bit sequence limits, false is returned.
+func (h *Bitmap) IsSet(ordinal uint64) bool {
+	if err := h.validateOrdinal(ordinal); err != nil {
+		return false
+	}
+	_, _, err := checkIfAvailable(h.head, ordinal)
+	return err != nil
+}
+
+// CheckConsistency checks if the bit sequence is in an inconsistent state and attempts to fix it.
+// It looks for a corruption signature that may happen in docker 1.9.0 and 1.9.1.
+func (h *Bitmap) CheckConsistency() bool {
+	corrupted := false
+	for p, c := h.head, h.head.next; c != nil; c = c.next {
+		if c.count == 0 {
+			corrupted = true
+			p.next = c.next
+			continue // keep same p
+		}
+		p = c
+	}
+	return corrupted
+}
+
+// set/reset the bit
+func (h *Bitmap) set(ordinal, start, end uint64, any bool, release bool, serial bool) (uint64, error) {
+	var (
+		bitPos  uint64
+		bytePos uint64
+		ret     uint64
+		err     error
+	)
+
+	curr := uint64(0)
+	if serial {
+		curr = h.curr
+	}
+	// Get position if available
+	if release {
+		bytePos, bitPos = ordinalToPos(ordinal)
+	} else {
+		if any {
+			bytePos, bitPos, err = getAvailableFromCurrent(h.head, start, curr, end)
+			ret = posToOrdinal(bytePos, bitPos)
+			if err == nil {
+				h.curr = ret + 1
+			}
+		} else {
+			bytePos, bitPos, err = checkIfAvailable(h.head, ordinal)
+			ret = ordinal
+		}
+	}
+	if err != nil {
+		return ret, err
+	}
+
+	h.head = pushReservation(bytePos, bitPos, h.head, release)
+	if release {
+		h.unselected++
+	} else {
+		h.unselected--
+	}
+
+	return ret, nil
+}
+
+// checks is needed because to cover the case where the number of bits is not a multiple of blockLen
+func (h *Bitmap) validateOrdinal(ordinal uint64) error {
+	if ordinal >= h.bits {
+		return errors.New("bit does not belong to the sequence")
+	}
+	return nil
+}
+
+// MarshalBinary encodes h into a binary representation.
+func (h *Bitmap) MarshalBinary() ([]byte, error) {
+	ba := make([]byte, 16)
+	binary.BigEndian.PutUint64(ba[0:], h.bits)
+	binary.BigEndian.PutUint64(ba[8:], h.unselected)
+	bm, err := h.head.toByteArray()
+	if err != nil {
+		return nil, fmt.Errorf("failed to serialize head: %v", err)
+	}
+	ba = append(ba, bm...)
+
+	return ba, nil
+}
+
+// UnmarshalBinary decodes a binary representation of a Bitmap value which was
+// generated using [Bitmap.MarshalBinary].
+//
+// The scan position for serial [Bitmap.SetAny] and [Bitmap.SetAnyInRange]
+// operations is neither unmarshaled nor reset.
+func (h *Bitmap) UnmarshalBinary(ba []byte) error {
+	if ba == nil {
+		return errors.New("nil byte array")
+	}
+
+	nh := &sequence{}
+	err := nh.fromByteArray(ba[16:])
+	if err != nil {
+		return fmt.Errorf("failed to deserialize head: %v", err)
+	}
+
+	h.head = nh
+	h.bits = binary.BigEndian.Uint64(ba[0:8])
+	h.unselected = binary.BigEndian.Uint64(ba[8:16])
+	return nil
+}
+
+// Bits returns the length of the bit sequence
+func (h *Bitmap) Bits() uint64 {
+	return h.bits
+}
+
+// Unselected returns the number of bits which are not selected
+func (h *Bitmap) Unselected() uint64 {
+	return h.unselected
+}
+
+func (h *Bitmap) String() string {
+	return fmt.Sprintf("Bits: %d, Unselected: %d, Sequence: %s Curr:%d",
+		h.bits, h.unselected, h.head.toString(), h.curr)
+}
+
+// MarshalJSON encodes h into a JSON message
+func (h *Bitmap) MarshalJSON() ([]byte, error) {
+	b, err := h.MarshalBinary()
+	if err != nil {
+		return nil, err
+	}
+	return json.Marshal(b)
+}
+
+// UnmarshalJSON decodes JSON message into h
+func (h *Bitmap) UnmarshalJSON(data []byte) error {
+	var b []byte
+	if err := json.Unmarshal(data, &b); err != nil {
+		return err
+	}
+	return h.UnmarshalBinary(b)
+}
+
+// getFirstAvailable looks for the first unset bit in passed mask starting from start
+func getFirstAvailable(head *sequence, start uint64) (uint64, uint64, error) {
+	// Find sequence which contains the start bit
+	byteStart, bitStart := ordinalToPos(start)
+	current, _, precBlocks, inBlockBytePos := findSequence(head, byteStart)
+	// Derive the this sequence offsets
+	byteOffset := byteStart - inBlockBytePos
+	bitOffset := inBlockBytePos*8 + bitStart
+	for current != nil {
+		if current.block != blockMAX {
+			// If the current block is not full, check if there is any bit
+			// from the current bit in the current block. If not, before proceeding to the
+			// next block node, make sure we check for available bit in the next
+			// instance of the same block. Due to RLE same block signature will be
+			// compressed.
+		retry:
+			bytePos, bitPos, err := current.getAvailableBit(bitOffset)
+			if err != nil && precBlocks == current.count-1 {
+				// This is the last instance in the same block node,
+				// so move to the next block.
+				goto next
+			}
+			if err != nil {
+				// There are some more instances of the same block, so add the offset
+				// and be optimistic that you will find the available bit in the next
+				// instance of the same block.
+				bitOffset = 0
+				byteOffset += blockBytes
+				precBlocks++
+				goto retry
+			}
+			return byteOffset + bytePos, bitPos, err
+		}
+		// Moving to next block: Reset bit offset.
+	next:
+		bitOffset = 0
+		byteOffset += (current.count * blockBytes) - (precBlocks * blockBytes)
+		precBlocks = 0
+		current = current.next
+	}
+	return invalidPos, invalidPos, ErrNoBitAvailable
+}
+
+// getAvailableFromCurrent will look for available ordinal from the current ordinal.
+// If none found then it will loop back to the start to check of the available bit.
+// This can be further optimized to check from start till curr in case of a rollover
+func getAvailableFromCurrent(head *sequence, start, curr, end uint64) (uint64, uint64, error) {
+	var bytePos, bitPos uint64
+	var err error
+	if curr != 0 && curr > start {
+		bytePos, bitPos, err = getFirstAvailable(head, curr)
+		ret := posToOrdinal(bytePos, bitPos)
+		if end < ret || err != nil {
+			goto begin
+		}
+		return bytePos, bitPos, nil
+	}
+
+begin:
+	bytePos, bitPos, err = getFirstAvailable(head, start)
+	ret := posToOrdinal(bytePos, bitPos)
+	if end < ret || err != nil {
+		return invalidPos, invalidPos, ErrNoBitAvailable
+	}
+	return bytePos, bitPos, nil
+}
+
+// checkIfAvailable checks if the bit correspondent to the specified ordinal is unset
+// If the ordinal is beyond the sequence limits, a negative response is returned
+func checkIfAvailable(head *sequence, ordinal uint64) (uint64, uint64, error) {
+	bytePos, bitPos := ordinalToPos(ordinal)
+
+	// Find the sequence containing this byte
+	current, _, _, inBlockBytePos := findSequence(head, bytePos)
+	if current != nil {
+		// Check whether the bit corresponding to the ordinal address is unset
+		bitSel := blockFirstBit >> (inBlockBytePos*8 + bitPos)
+		if current.block&bitSel == 0 {
+			return bytePos, bitPos, nil
+		}
+	}
+
+	return invalidPos, invalidPos, ErrBitAllocated
+}
+
+// Given the byte position and the sequences list head, return the pointer to the
+// sequence containing the byte (current), the pointer to the previous sequence,
+// the number of blocks preceding the block containing the byte inside the current sequence.
+// If bytePos is outside of the list, function will return (nil, nil, 0, invalidPos)
+func findSequence(head *sequence, bytePos uint64) (*sequence, *sequence, uint64, uint64) {
+	// Find the sequence containing this byte
+	previous := head
+	current := head
+	n := bytePos
+	for current.next != nil && n >= (current.count*blockBytes) { // Nil check for less than 32 addresses masks
+		n -= (current.count * blockBytes)
+		previous = current
+		current = current.next
+	}
+
+	// If byte is outside of the list, let caller know
+	if n >= (current.count * blockBytes) {
+		return nil, nil, 0, invalidPos
+	}
+
+	// Find the byte position inside the block and the number of blocks
+	// preceding the block containing the byte inside this sequence
+	precBlocks := n / blockBytes
+	inBlockBytePos := bytePos % blockBytes
+
+	return current, previous, precBlocks, inBlockBytePos
+}
+
+// PushReservation pushes the bit reservation inside the bitmask.
+// Given byte and bit positions, identify the sequence (current) which holds the block containing the affected bit.
+// Create a new block with the modified bit according to the operation (allocate/release).
+// Create a new sequence containing the new block and insert it in the proper position.
+// Remove current sequence if empty.
+// Check if new sequence can be merged with neighbour (previous/next) sequences.
+//
+// Identify "current" sequence containing block:
+//
+//	[prev seq] [current seq] [next seq]
+//
+// Based on block position, resulting list of sequences can be any of three forms:
+//
+// block position                        Resulting list of sequences
+//
+// A) block is first in current:         [prev seq] [new] [modified current seq] [next seq]
+// B) block is last in current:          [prev seq] [modified current seq] [new] [next seq]
+// C) block is in the middle of current: [prev seq] [curr pre] [new] [curr post] [next seq]
+func pushReservation(bytePos, bitPos uint64, head *sequence, release bool) *sequence {
+	// Store list's head
+	newHead := head
+
+	// Find the sequence containing this byte
+	current, previous, precBlocks, inBlockBytePos := findSequence(head, bytePos)
+	if current == nil {
+		return newHead
+	}
+
+	// Construct updated block
+	bitSel := blockFirstBit >> (inBlockBytePos*8 + bitPos)
+	newBlock := current.block
+	if release {
+		newBlock &^= bitSel
+	} else {
+		newBlock |= bitSel
+	}
+
+	// Quit if it was a redundant request
+	if current.block == newBlock {
+		return newHead
+	}
+
+	// Current sequence inevitably looses one block, upadate count
+	current.count--
+
+	// Create new sequence
+	newSequence := &sequence{block: newBlock, count: 1}
+
+	// Insert the new sequence in the list based on block position
+	if precBlocks == 0 { // First in sequence (A)
+		newSequence.next = current
+		if current == head {
+			newHead = newSequence
+			previous = newHead
+		} else {
+			previous.next = newSequence
+		}
+		removeCurrentIfEmpty(&newHead, newSequence, current)
+		mergeSequences(previous)
+	} else if precBlocks == current.count { // Last in sequence (B)
+		newSequence.next = current.next
+		current.next = newSequence
+		mergeSequences(current)
+	} else { // In between the sequence (C)
+		currPre := &sequence{block: current.block, count: precBlocks, next: newSequence}
+		currPost := current
+		currPost.count -= precBlocks
+		newSequence.next = currPost
+		if currPost == head {
+			newHead = currPre
+		} else {
+			previous.next = currPre
+		}
+		// No merging or empty current possible here
+	}
+
+	return newHead
+}
+
+// Removes the current sequence from the list if empty, adjusting the head pointer if needed
+func removeCurrentIfEmpty(head **sequence, previous, current *sequence) {
+	if current.count == 0 {
+		if current == *head {
+			*head = current.next
+		} else {
+			previous.next = current.next
+		}
+	}
+}
+
+// Given a pointer to a sequence, it checks if it can be merged with any following sequences
+// It stops when no more merging is possible.
+// TODO: Optimization: only attempt merge from start to end sequence, no need to scan till the end of the list
+func mergeSequences(seq *sequence) {
+	if seq != nil {
+		// Merge all what possible from seq
+		for seq.next != nil && seq.block == seq.next.block {
+			seq.count += seq.next.count
+			seq.next = seq.next.next
+		}
+		// Move to next
+		mergeSequences(seq.next)
+	}
+}
+
+func getNumBlocks(numBits uint64) uint64 {
+	numBlocks := numBits / uint64(blockLen)
+	if numBits%uint64(blockLen) != 0 {
+		numBlocks++
+	}
+	return numBlocks
+}
+
+func ordinalToPos(ordinal uint64) (uint64, uint64) {
+	return ordinal / 8, ordinal % 8
+}
+
+func posToOrdinal(bytePos, bitPos uint64) uint64 {
+	return bytePos*8 + bitPos
+}

libnetwork/bitmap/sequence_test.go

@@ -0,0 +1,1255 @@
+package bitmap
+
+import (
+	"math/rand"
+	"testing"
+	"time"
+)
+
+func TestSequenceGetAvailableBit(t *testing.T) {
+	input := []struct {
+		head    *sequence
+		from    uint64
+		bytePos uint64
+		bitPos  uint64
+	}{
+		{&sequence{block: 0x0, count: 0}, 0, invalidPos, invalidPos},
+		{&sequence{block: 0x0, count: 1}, 0, 0, 0},
+		{&sequence{block: 0x0, count: 100}, 0, 0, 0},
+
+		{&sequence{block: 0x80000000, count: 0}, 0, invalidPos, invalidPos},
+		{&sequence{block: 0x80000000, count: 1}, 0, 0, 1},
+		{&sequence{block: 0x80000000, count: 100}, 0, 0, 1},
+
+		{&sequence{block: 0xFF000000, count: 0}, 0, invalidPos, invalidPos},
+		{&sequence{block: 0xFF000000, count: 1}, 0, 1, 0},
+		{&sequence{block: 0xFF000000, count: 100}, 0, 1, 0},
+
+		{&sequence{block: 0xFF800000, count: 0}, 0, invalidPos, invalidPos},
+		{&sequence{block: 0xFF800000, count: 1}, 0, 1, 1},
+		{&sequence{block: 0xFF800000, count: 100}, 0, 1, 1},
+
+		{&sequence{block: 0xFFC0FF00, count: 0}, 0, invalidPos, invalidPos},
+		{&sequence{block: 0xFFC0FF00, count: 1}, 0, 1, 2},
+		{&sequence{block: 0xFFC0FF00, count: 100}, 0, 1, 2},
+
+		{&sequence{block: 0xFFE0FF00, count: 0}, 0, invalidPos, invalidPos},
+		{&sequence{block: 0xFFE0FF00, count: 1}, 0, 1, 3},
+		{&sequence{block: 0xFFE0FF00, count: 100}, 0, 1, 3},
+
+		{&sequence{block: 0xFFFEFF00, count: 0}, 0, invalidPos, invalidPos},
+		{&sequence{block: 0xFFFEFF00, count: 1}, 0, 1, 7},
+		{&sequence{block: 0xFFFEFF00, count: 100}, 0, 1, 7},
+
+		{&sequence{block: 0xFFFFC0FF, count: 0}, 0, invalidPos, invalidPos},
+		{&sequence{block: 0xFFFFC0FF, count: 1}, 0, 2, 2},
+		{&sequence{block: 0xFFFFC0FF, count: 100}, 0, 2, 2},
+
+		{&sequence{block: 0xFFFFFF00, count: 0}, 0, invalidPos, invalidPos},
+		{&sequence{block: 0xFFFFFF00, count: 1}, 0, 3, 0},
+		{&sequence{block: 0xFFFFFF00, count: 100}, 0, 3, 0},
+
+		{&sequence{block: 0xFFFFFFFE, count: 0}, 0, invalidPos, invalidPos},
+		{&sequence{block: 0xFFFFFFFE, count: 1}, 0, 3, 7},
+		{&sequence{block: 0xFFFFFFFE, count: 100}, 0, 3, 7},
+
+		{&sequence{block: 0xFFFFFFFF, count: 0}, 0, invalidPos, invalidPos},
+		{&sequence{block: 0xFFFFFFFF, count: 1}, 0, invalidPos, invalidPos},
+		{&sequence{block: 0xFFFFFFFF, count: 100}, 0, invalidPos, invalidPos},
+
+		// now test with offset
+		{&sequence{block: 0x0, count: 0}, 0, invalidPos, invalidPos},
+		{&sequence{block: 0x0, count: 0}, 31, invalidPos, invalidPos},
+		{&sequence{block: 0x0, count: 0}, 32, invalidPos, invalidPos},
+		{&sequence{block: 0x0, count: 1}, 0, 0, 0},
+		{&sequence{block: 0x0, count: 1}, 1, 0, 1},
+		{&sequence{block: 0x0, count: 1}, 31, 3, 7},
+		{&sequence{block: 0xF0FF0000, count: 1}, 0, 0, 4},
+		{&sequence{block: 0xF0FF0000, count: 1}, 8, 2, 0},
+		{&sequence{block: 0xFFFFFFFF, count: 1}, 0, invalidPos, invalidPos},
+		{&sequence{block: 0xFFFFFFFF, count: 1}, 16, invalidPos, invalidPos},
+		{&sequence{block: 0xFFFFFFFF, count: 1}, 31, invalidPos, invalidPos},
+		{&sequence{block: 0xFFFFFFFE, count: 1}, 0, 3, 7},
+		{&sequence{block: 0xFFFFFFFF, count: 2}, 0, invalidPos, invalidPos},
+		{&sequence{block: 0xFFFFFFFF, count: 2}, 32, invalidPos, invalidPos},
+	}
+
+	for n, i := range input {
+		b, bb, err := i.head.getAvailableBit(i.from)
+		if b != i.bytePos || bb != i.bitPos {
+			t.Fatalf("Error in sequence.getAvailableBit(%d) (%d).\nExp: (%d, %d)\nGot: (%d, %d), err: %v", i.from, n, i.bytePos, i.bitPos, b, bb, err)
+		}
+	}
+}
+
+func TestSequenceEqual(t *testing.T) {
+	input := []struct {
+		first    *sequence
+		second   *sequence
+		areEqual bool
+	}{
+		{&sequence{block: 0x0, count: 8, next: nil}, &sequence{block: 0x0, count: 8}, true},
+		{&sequence{block: 0x0, count: 0, next: nil}, &sequence{block: 0x0, count: 0}, true},
+		{&sequence{block: 0x0, count: 2, next: nil}, &sequence{block: 0x0, count: 1, next: &sequence{block: 0x0, count: 1}}, false},
+		{&sequence{block: 0x0, count: 2, next: &sequence{block: 0x1, count: 1}}, &sequence{block: 0x0, count: 2}, false},
+
+		{&sequence{block: 0x12345678, count: 8, next: nil}, &sequence{block: 0x12345678, count: 8}, true},
+		{&sequence{block: 0x12345678, count: 8, next: nil}, &sequence{block: 0x12345678, count: 9}, false},
+		{&sequence{block: 0x12345678, count: 1, next: &sequence{block: 0xFFFFFFFF, count: 1}}, &sequence{block: 0x12345678, count: 1}, false},
+		{&sequence{block: 0x12345678, count: 1}, &sequence{block: 0x12345678, count: 1, next: &sequence{block: 0xFFFFFFFF, count: 1}}, false},
+	}
+
+	for n, i := range input {
+		if i.areEqual != i.first.equal(i.second) {
+			t.Fatalf("Error in sequence.equal() (%d).\nExp: %t\nGot: %t,", n, i.areEqual, !i.areEqual)
+		}
+	}
+}
+
+func TestSequenceCopy(t *testing.T) {
+	s := getTestSequence()
+	n := s.getCopy()
+	if !s.equal(n) {
+		t.Fatal("copy of s failed")
+	}
+	if n == s {
+		t.Fatal("not true copy of s")
+	}
+}
+
+func TestGetFirstAvailable(t *testing.T) {
+	input := []struct {
+		mask    *sequence
+		bytePos uint64
+		bitPos  uint64
+		start   uint64
+	}{
+		{&sequence{block: 0xffffffff, count: 2048}, invalidPos, invalidPos, 0},
+		{&sequence{block: 0x0, count: 8}, 0, 0, 0},
+		{&sequence{block: 0x80000000, count: 8}, 0, 1, 0},
+		{&sequence{block: 0xC0000000, count: 8}, 0, 2, 0},
+		{&sequence{block: 0xE0000000, count: 8}, 0, 3, 0},
+		{&sequence{block: 0xF0000000, count: 8}, 0, 4, 0},
+		{&sequence{block: 0xF8000000, count: 8}, 0, 5, 0},
+		{&sequence{block: 0xFC000000, count: 8}, 0, 6, 0},
+		{&sequence{block: 0xFE000000, count: 8}, 0, 7, 0},
+		{&sequence{block: 0xFE000000, count: 8}, 3, 0, 24},
+
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0x00000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 4, 0, 0},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0x80000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 4, 1, 0},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xC0000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 4, 2, 0},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xE0000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 4, 3, 0},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xF0000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 4, 4, 0},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xF8000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 4, 5, 0},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFC000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 4, 6, 0},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFE000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 4, 7, 0},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0x0E000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 4, 0, 16},
+
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFF000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 5, 0, 0},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFF800000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 5, 1, 0},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFC00000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 5, 2, 0},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFE00000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 5, 3, 0},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFF00000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 5, 4, 0},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFF80000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 5, 5, 0},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFFC0000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 5, 6, 0},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFFE0000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 5, 7, 0},
+
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xfffffffe, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 7, 7, 0},
+
+		{&sequence{block: 0xffffffff, count: 2, next: &sequence{block: 0x0, count: 6}}, 8, 0, 0},
+		{&sequence{block: 0xfffcffff, count: 1, next: &sequence{block: 0x0, count: 6}}, 4, 0, 16},
+		{&sequence{block: 0xfffcffff, count: 1, next: &sequence{block: 0x0, count: 6}}, 1, 7, 15},
+		{&sequence{block: 0xfffcffff, count: 1, next: &sequence{block: 0x0, count: 6}}, 1, 6, 10},
+		{&sequence{block: 0xfffcfffe, count: 1, next: &sequence{block: 0x0, count: 6}}, 3, 7, 31},
+		{&sequence{block: 0xfffcffff, count: 1, next: &sequence{block: 0xffffffff, count: 6}}, invalidPos, invalidPos, 31},
+	}
+
+	for n, i := range input {
+		bytePos, bitPos, _ := getFirstAvailable(i.mask, i.start)
+		if bytePos != i.bytePos || bitPos != i.bitPos {
+			t.Fatalf("Error in (%d) getFirstAvailable(). Expected (%d, %d). Got (%d, %d)", n, i.bytePos, i.bitPos, bytePos, bitPos)
+		}
+	}
+}
+
+func TestFindSequence(t *testing.T) {
+	input := []struct {
+		head           *sequence
+		bytePos        uint64
+		precBlocks     uint64
+		inBlockBytePos uint64
+	}{
+		{&sequence{block: 0xffffffff, count: 0}, 0, 0, invalidPos},
+		{&sequence{block: 0xffffffff, count: 0}, 31, 0, invalidPos},
+		{&sequence{block: 0xffffffff, count: 0}, 100, 0, invalidPos},
+
+		{&sequence{block: 0x0, count: 1}, 0, 0, 0},
+		{&sequence{block: 0x0, count: 1}, 1, 0, 1},
+		{&sequence{block: 0x0, count: 1}, 31, 0, invalidPos},
+		{&sequence{block: 0x0, count: 1}, 60, 0, invalidPos},
+
+		{&sequence{block: 0xffffffff, count: 10}, 0, 0, 0},
+		{&sequence{block: 0xffffffff, count: 10}, 3, 0, 3},
+		{&sequence{block: 0xffffffff, count: 10}, 4, 1, 0},
+		{&sequence{block: 0xffffffff, count: 10}, 7, 1, 3},
+		{&sequence{block: 0xffffffff, count: 10}, 8, 2, 0},
+		{&sequence{block: 0xffffffff, count: 10}, 39, 9, 3},
+
+		{&sequence{block: 0xffffffff, count: 10, next: &sequence{block: 0xcc000000, count: 10}}, 79, 9, 3},
+		{&sequence{block: 0xffffffff, count: 10, next: &sequence{block: 0xcc000000, count: 10}}, 80, 0, invalidPos},
+	}
+
+	for n, i := range input {
+		_, _, precBlocks, inBlockBytePos := findSequence(i.head, i.bytePos)
+		if precBlocks != i.precBlocks || inBlockBytePos != i.inBlockBytePos {
+			t.Fatalf("Error in (%d) findSequence(). Expected (%d, %d). Got (%d, %d)", n, i.precBlocks, i.inBlockBytePos, precBlocks, inBlockBytePos)
+		}
+	}
+}
+
+func TestCheckIfAvailable(t *testing.T) {
+	input := []struct {
+		head    *sequence
+		ordinal uint64
+		bytePos uint64
+		bitPos  uint64
+	}{
+		{&sequence{block: 0xffffffff, count: 0}, 0, invalidPos, invalidPos},
+		{&sequence{block: 0xffffffff, count: 0}, 31, invalidPos, invalidPos},
+		{&sequence{block: 0xffffffff, count: 0}, 100, invalidPos, invalidPos},
+
+		{&sequence{block: 0x0, count: 1}, 0, 0, 0},
+		{&sequence{block: 0x0, count: 1}, 1, 0, 1},
+		{&sequence{block: 0x0, count: 1}, 31, 3, 7},
+		{&sequence{block: 0x0, count: 1}, 60, invalidPos, invalidPos},
+
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0x800000ff, count: 1}}, 31, invalidPos, invalidPos},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0x800000ff, count: 1}}, 32, invalidPos, invalidPos},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0x800000ff, count: 1}}, 33, 4, 1},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xC00000ff, count: 1}}, 33, invalidPos, invalidPos},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xC00000ff, count: 1}}, 34, 4, 2},
+
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xC00000ff, count: 1, next: &sequence{block: 0x0, count: 1}}}, 55, 6, 7},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xC00000ff, count: 1, next: &sequence{block: 0x0, count: 1}}}, 56, invalidPos, invalidPos},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xC00000ff, count: 1, next: &sequence{block: 0x0, count: 1}}}, 63, invalidPos, invalidPos},
+
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xC00000ff, count: 1, next: &sequence{block: 0x0, count: 1}}}, 64, 8, 0},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xC00000ff, count: 1, next: &sequence{block: 0x0, count: 1}}}, 95, 11, 7},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xC00000ff, count: 1, next: &sequence{block: 0x0, count: 1}}}, 96, invalidPos, invalidPos},
+	}
+
+	for n, i := range input {
+		bytePos, bitPos, err := checkIfAvailable(i.head, i.ordinal)
+		if bytePos != i.bytePos || bitPos != i.bitPos {
+			t.Fatalf("Error in (%d) checkIfAvailable(ord:%d). Expected (%d, %d). Got (%d, %d). err: %v", n, i.ordinal, i.bytePos, i.bitPos, bytePos, bitPos, err)
+		}
+	}
+}
+
+func TestMergeSequences(t *testing.T) {
+	input := []struct {
+		original *sequence
+		merged   *sequence
+	}{
+		{&sequence{block: 0xFE000000, count: 8, next: &sequence{block: 0xFE000000, count: 2}}, &sequence{block: 0xFE000000, count: 10}},
+		{&sequence{block: 0xFFFFFFFF, count: 8, next: &sequence{block: 0xFFFFFFFF, count: 1}}, &sequence{block: 0xFFFFFFFF, count: 9}},
+		{&sequence{block: 0xFFFFFFFF, count: 1, next: &sequence{block: 0xFFFFFFFF, count: 8}}, &sequence{block: 0xFFFFFFFF, count: 9}},
+
+		{&sequence{block: 0xFFFFFFF0, count: 8, next: &sequence{block: 0xFFFFFFF0, count: 1}}, &sequence{block: 0xFFFFFFF0, count: 9}},
+		{&sequence{block: 0xFFFFFFF0, count: 1, next: &sequence{block: 0xFFFFFFF0, count: 8}}, &sequence{block: 0xFFFFFFF0, count: 9}},
+
+		{&sequence{block: 0xFE, count: 8, next: &sequence{block: 0xFE, count: 1, next: &sequence{block: 0xFE, count: 5}}}, &sequence{block: 0xFE, count: 14}},
+		{&sequence{block: 0xFE, count: 8, next: &sequence{block: 0xFE, count: 1, next: &sequence{block: 0xFE, count: 5, next: &sequence{block: 0xFF, count: 1}}}},
+			&sequence{block: 0xFE, count: 14, next: &sequence{block: 0xFF, count: 1}}},
+
+		// No merge
+		{&sequence{block: 0xFE, count: 8, next: &sequence{block: 0xF8, count: 1, next: &sequence{block: 0xFE, count: 5}}},
+			&sequence{block: 0xFE, count: 8, next: &sequence{block: 0xF8, count: 1, next: &sequence{block: 0xFE, count: 5}}}},
+
+		// No merge from head: // Merge function tries to merge from passed head. If it can't merge with next, it does not reattempt with next as head
+		{&sequence{block: 0xFE, count: 8, next: &sequence{block: 0xFF, count: 1, next: &sequence{block: 0xFF, count: 5}}},
+			&sequence{block: 0xFE, count: 8, next: &sequence{block: 0xFF, count: 6}}},
+	}
+
+	for n, i := range input {
+		mergeSequences(i.original)
+		for !i.merged.equal(i.original) {
+			t.Fatalf("Error in (%d) mergeSequences().\nExp: %s\nGot: %s,", n, i.merged.toString(), i.original.toString())
+		}
+	}
+}
+
+func TestPushReservation(t *testing.T) {
+	input := []struct {
+		mask    *sequence
+		bytePos uint64
+		bitPos  uint64
+		newMask *sequence
+	}{
+		// Create first sequence and fill in 8 addresses starting from address 0
+		{&sequence{block: 0x0, count: 8, next: nil}, 0, 0, &sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x0, count: 7, next: nil}}},
+		{&sequence{block: 0x80000000, count: 8}, 0, 1, &sequence{block: 0xC0000000, count: 1, next: &sequence{block: 0x80000000, count: 7, next: nil}}},
+		{&sequence{block: 0xC0000000, count: 8}, 0, 2, &sequence{block: 0xE0000000, count: 1, next: &sequence{block: 0xC0000000, count: 7, next: nil}}},
+		{&sequence{block: 0xE0000000, count: 8}, 0, 3, &sequence{block: 0xF0000000, count: 1, next: &sequence{block: 0xE0000000, count: 7, next: nil}}},
+		{&sequence{block: 0xF0000000, count: 8}, 0, 4, &sequence{block: 0xF8000000, count: 1, next: &sequence{block: 0xF0000000, count: 7, next: nil}}},
+		{&sequence{block: 0xF8000000, count: 8}, 0, 5, &sequence{block: 0xFC000000, count: 1, next: &sequence{block: 0xF8000000, count: 7, next: nil}}},
+		{&sequence{block: 0xFC000000, count: 8}, 0, 6, &sequence{block: 0xFE000000, count: 1, next: &sequence{block: 0xFC000000, count: 7, next: nil}}},
+		{&sequence{block: 0xFE000000, count: 8}, 0, 7, &sequence{block: 0xFF000000, count: 1, next: &sequence{block: 0xFE000000, count: 7, next: nil}}},
+
+		{&sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x0, count: 7}}, 0, 1, &sequence{block: 0xC0000000, count: 1, next: &sequence{block: 0x0, count: 7, next: nil}}},
+
+		// Create second sequence and fill in 8 addresses starting from address 32
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0x00000000, count: 1, next: &sequence{block: 0xffffffff, count: 6, next: nil}}}, 4, 0,
+			&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0x80000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0x80000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 4, 1,
+			&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xC0000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xC0000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 4, 2,
+			&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xE0000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xE0000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 4, 3,
+			&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xF0000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xF0000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 4, 4,
+			&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xF8000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xF8000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 4, 5,
+			&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFC000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFC000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 4, 6,
+			&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFE000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFE000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 4, 7,
+			&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFF000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}},
+		// fill in 8 addresses starting from address 40
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFF000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 5, 0,
+			&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFF800000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFF800000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 5, 1,
+			&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFC00000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFC00000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 5, 2,
+			&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFE00000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFE00000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 5, 3,
+			&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFF00000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFF00000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 5, 4,
+			&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFF80000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFF80000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 5, 5,
+			&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFFC0000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFFC0000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 5, 6,
+			&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFFE0000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFFE0000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 5, 7,
+			&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFFF0000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}},
+
+		// Insert new sequence
+		{&sequence{block: 0xffffffff, count: 2, next: &sequence{block: 0x0, count: 6}}, 8, 0,
+			&sequence{block: 0xffffffff, count: 2, next: &sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x0, count: 5}}}},
+		{&sequence{block: 0xffffffff, count: 2, next: &sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x0, count: 5}}}, 8, 1,
+			&sequence{block: 0xffffffff, count: 2, next: &sequence{block: 0xC0000000, count: 1, next: &sequence{block: 0x0, count: 5}}}},
+
+		// Merge affected with next
+		{&sequence{block: 0xffffffff, count: 7, next: &sequence{block: 0xfffffffe, count: 2, next: &sequence{block: 0xffffffff, count: 1}}}, 31, 7,
+			&sequence{block: 0xffffffff, count: 8, next: &sequence{block: 0xfffffffe, count: 1, next: &sequence{block: 0xffffffff, count: 1}}}},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xfffffffc, count: 1, next: &sequence{block: 0xfffffffe, count: 6}}}, 7, 6,
+			&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xfffffffe, count: 7}}},
+
+		// Merge affected with next and next.next
+		{&sequence{block: 0xffffffff, count: 7, next: &sequence{block: 0xfffffffe, count: 1, next: &sequence{block: 0xffffffff, count: 1}}}, 31, 7,
+			&sequence{block: 0xffffffff, count: 9}},
+		{&sequence{block: 0xffffffff, count: 7, next: &sequence{block: 0xfffffffe, count: 1}}, 31, 7,
+			&sequence{block: 0xffffffff, count: 8}},
+
+		// Merge affected with previous and next
+		{&sequence{block: 0xffffffff, count: 7, next: &sequence{block: 0xfffffffe, count: 1, next: &sequence{block: 0xffffffff, count: 1}}}, 31, 7,
+			&sequence{block: 0xffffffff, count: 9}},
+
+		// Redundant push: No change
+		{&sequence{block: 0xffff0000, count: 1}, 0, 0, &sequence{block: 0xffff0000, count: 1}},
+		{&sequence{block: 0xffff0000, count: 7}, 25, 7, &sequence{block: 0xffff0000, count: 7}},
+		{&sequence{block: 0xffffffff, count: 7, next: &sequence{block: 0xfffffffe, count: 1, next: &sequence{block: 0xffffffff, count: 1}}}, 7, 7,
+			&sequence{block: 0xffffffff, count: 7, next: &sequence{block: 0xfffffffe, count: 1, next: &sequence{block: 0xffffffff, count: 1}}}},
+
+		// Set last bit
+		{&sequence{block: 0x0, count: 8}, 31, 7, &sequence{block: 0x0, count: 7, next: &sequence{block: 0x1, count: 1}}},
+
+		// Set bit in a middle sequence in the first block, first bit
+		{&sequence{block: 0x40000000, count: 1, next: &sequence{block: 0x0, count: 6, next: &sequence{block: 0x1, count: 1}}}, 4, 0,
+			&sequence{block: 0x40000000, count: 1, next: &sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x0, count: 5,
+				next: &sequence{block: 0x1, count: 1}}}}},
+
+		// Set bit in a middle sequence in the first block, first bit (merge involved)
+		{&sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x0, count: 6, next: &sequence{block: 0x1, count: 1}}}, 4, 0,
+			&sequence{block: 0x80000000, count: 2, next: &sequence{block: 0x0, count: 5, next: &sequence{block: 0x1, count: 1}}}},
+
+		// Set bit in a middle sequence in the first block, last bit
+		{&sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x0, count: 6, next: &sequence{block: 0x1, count: 1}}}, 4, 31,
+			&sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x1, count: 1, next: &sequence{block: 0x0, count: 5,
+				next: &sequence{block: 0x1, count: 1}}}}},
+
+		// Set bit in a middle sequence in the first block, middle bit
+		{&sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x0, count: 6, next: &sequence{block: 0x1, count: 1}}}, 4, 16,
+			&sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x8000, count: 1, next: &sequence{block: 0x0, count: 5,
+				next: &sequence{block: 0x1, count: 1}}}}},
+
+		// Set bit in a middle sequence in a middle block, first bit
+		{&sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x0, count: 6, next: &sequence{block: 0x1, count: 1}}}, 16, 0,
+			&sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x0, count: 3, next: &sequence{block: 0x80000000, count: 1,
+				next: &sequence{block: 0x0, count: 2, next: &sequence{block: 0x1, count: 1}}}}}},
+
+		// Set bit in a middle sequence in a middle block, last bit
+		{&sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x0, count: 6, next: &sequence{block: 0x1, count: 1}}}, 16, 31,
+			&sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x0, count: 3, next: &sequence{block: 0x1, count: 1,
+				next: &sequence{block: 0x0, count: 2, next: &sequence{block: 0x1, count: 1}}}}}},
+
+		// Set bit in a middle sequence in a middle block, middle bit
+		{&sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x0, count: 6, next: &sequence{block: 0x1, count: 1}}}, 16, 15,
+			&sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x0, count: 3, next: &sequence{block: 0x10000, count: 1,
+				next: &sequence{block: 0x0, count: 2, next: &sequence{block: 0x1, count: 1}}}}}},
+
+		// Set bit in a middle sequence in the last block, first bit
+		{&sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x0, count: 6, next: &sequence{block: 0x1, count: 1}}}, 24, 0,
+			&sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x0, count: 5, next: &sequence{block: 0x80000000, count: 1,
+				next: &sequence{block: 0x1, count: 1}}}}},
+
+		// Set bit in a middle sequence in the last block, last bit
+		{&sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x0, count: 6, next: &sequence{block: 0x4, count: 1}}}, 24, 31,
+			&sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x0, count: 5, next: &sequence{block: 0x1, count: 1,
+				next: &sequence{block: 0x4, count: 1}}}}},
+
+		// Set bit in a middle sequence in the last block, last bit (merge involved)
+		{&sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x0, count: 6, next: &sequence{block: 0x1, count: 1}}}, 24, 31,
+			&sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x0, count: 5, next: &sequence{block: 0x1, count: 2}}}},
+
+		// Set bit in a middle sequence in the last block, middle bit
+		{&sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x0, count: 6, next: &sequence{block: 0x1, count: 1}}}, 24, 16,
+			&sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x0, count: 5, next: &sequence{block: 0x8000, count: 1,
+				next: &sequence{block: 0x1, count: 1}}}}},
+	}
+
+	for n, i := range input {
+		mask := pushReservation(i.bytePos, i.bitPos, i.mask, false)
+		if !mask.equal(i.newMask) {
+			t.Fatalf("Error in (%d) pushReservation():\n%s + (%d,%d):\nExp: %s\nGot: %s,",
+				n, i.mask.toString(), i.bytePos, i.bitPos, i.newMask.toString(), mask.toString())
+		}
+	}
+}
+
+func TestSerializeDeserialize(t *testing.T) {
+	s := getTestSequence()
+
+	data, err := s.toByteArray()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	r := &sequence{}
+	err = r.fromByteArray(data)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !s.equal(r) {
+		t.Fatalf("Sequences are different: \n%v\n%v", s, r)
+	}
+}
+
+func getTestSequence() *sequence {
+	// Returns a custom sequence of 1024 * 32 bits
+	return &sequence{
+		block: 0xFFFFFFFF,
+		count: 100,
+		next: &sequence{
+			block: 0xFFFFFFFE,
+			count: 1,
+			next: &sequence{
+				block: 0xFF000000,
+				count: 10,
+				next: &sequence{
+					block: 0xFFFFFFFF,
+					count: 50,
+					next: &sequence{
+						block: 0xFFFFFFFC,
+						count: 1,
+						next: &sequence{
+							block: 0xFF800000,
+							count: 1,
+							next: &sequence{
+								block: 0xFFFFFFFF,
+								count: 87,
+								next: &sequence{
+									block: 0x0,
+									count: 150,
+									next: &sequence{
+										block: 0xFFFFFFFF,
+										count: 200,
+										next: &sequence{
+											block: 0x0000FFFF,
+											count: 1,
+											next: &sequence{
+												block: 0x0,
+												count: 399,
+												next: &sequence{
+													block: 0xFFFFFFFF,
+													count: 23,
+													next: &sequence{
+														block: 0x1,
+														count: 1,
+													},
+												},
+											},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
+func TestSet(t *testing.T) {
+	hnd := New(1024 * 32)
+	hnd.head = getTestSequence()
+
+	firstAv := uint64(32*100 + 31)
+	last := uint64(1024*32 - 1)
+
+	if hnd.IsSet(100000) {
+		t.Fatal("IsSet() returned wrong result")
+	}
+
+	if !hnd.IsSet(0) {
+		t.Fatal("IsSet() returned wrong result")
+	}
+
+	if hnd.IsSet(firstAv) {
+		t.Fatal("IsSet() returned wrong result")
+	}
+
+	if !hnd.IsSet(last) {
+		t.Fatal("IsSet() returned wrong result")
+	}
+
+	if err := hnd.Set(0); err == nil {
+		t.Fatal("Expected failure, but succeeded")
+	}
+
+	os, err := hnd.SetAny(false)
+	if err != nil {
+		t.Fatalf("Unexpected failure: %v", err)
+	}
+	if os != firstAv {
+		t.Fatalf("SetAny returned unexpected ordinal. Expected %d. Got %d.", firstAv, os)
+	}
+	if !hnd.IsSet(firstAv) {
+		t.Fatal("IsSet() returned wrong result")
+	}
+
+	if err := hnd.Unset(firstAv); err != nil {
+		t.Fatalf("Unexpected failure: %v", err)
+	}
+
+	if hnd.IsSet(firstAv) {
+		t.Fatal("IsSet() returned wrong result")
+	}
+
+	if err := hnd.Set(firstAv); err != nil {
+		t.Fatalf("Unexpected failure: %v", err)
+	}
+
+	if err := hnd.Set(last); err == nil {
+		t.Fatal("Expected failure, but succeeded")
+	}
+}
+
+func TestSetUnset(t *testing.T) {
+	numBits := uint64(32 * blockLen)
+	hnd := New(numBits)
+
+	if err := hnd.Set(uint64(32 * blockLen)); err == nil {
+		t.Fatal("Expected failure, but succeeded")
+	}
+	if err := hnd.Unset(uint64(32 * blockLen)); err == nil {
+		t.Fatal("Expected failure, but succeeded")
+	}
+
+	// set and unset all one by one
+	for hnd.Unselected() > 0 {
+		if _, err := hnd.SetAny(false); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if _, err := hnd.SetAny(false); err != ErrNoBitAvailable {
+		t.Fatal("Expected error. Got success")
+	}
+	if _, err := hnd.SetAnyInRange(10, 20, false); err != ErrNoBitAvailable {
+		t.Fatal("Expected error. Got success")
+	}
+	if err := hnd.Set(50); err != ErrBitAllocated {
+		t.Fatalf("Expected error. Got %v: %s", err, hnd)
+	}
+	i := uint64(0)
+	for hnd.Unselected() < numBits {
+		if err := hnd.Unset(i); err != nil {
+			t.Fatal(err)
+		}
+		i++
+	}
+}
+
+func TestOffsetSetUnset(t *testing.T) {
+	numBits := uint64(32 * blockLen)
+	hnd := New(numBits)
+
+	// set and unset all one by one
+	for hnd.Unselected() > 0 {
+		if _, err := hnd.SetAny(false); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	if _, err := hnd.SetAny(false); err != ErrNoBitAvailable {
+		t.Fatal("Expected error. Got success")
+	}
+
+	if _, err := hnd.SetAnyInRange(10, 20, false); err != ErrNoBitAvailable {
+		t.Fatal("Expected error. Got success")
+	}
+
+	if err := hnd.Unset(288); err != nil {
+		t.Fatal(err)
+	}
+
+	//At this point sequence is (0xffffffff, 9)->(0x7fffffff, 1)->(0xffffffff, 22)->end
+	o, err := hnd.SetAnyInRange(32, 500, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if o != 288 {
+		t.Fatalf("Expected ordinal not received, Received:%d", o)
+	}
+}
+
+func TestSetInRange(t *testing.T) {
+	numBits := uint64(1024 * blockLen)
+	hnd := New(numBits)
+	hnd.head = getTestSequence()
+
+	firstAv := uint64(100*blockLen + blockLen - 1)
+
+	if o, err := hnd.SetAnyInRange(4, 3, false); err == nil {
+		t.Fatalf("Expected failure. Got success with ordinal:%d", o)
+	}
+
+	if o, err := hnd.SetAnyInRange(0, numBits, false); err == nil {
+		t.Fatalf("Expected failure. Got success with ordinal:%d", o)
+	}
+
+	o, err := hnd.SetAnyInRange(100*uint64(blockLen), 101*uint64(blockLen), false)
+	if err != nil {
+		t.Fatalf("Unexpected failure: (%d, %v)", o, err)
+	}
+	if o != firstAv {
+		t.Fatalf("Unexpected ordinal: %d", o)
+	}
+
+	if o, err := hnd.SetAnyInRange(0, uint64(blockLen), false); err == nil {
+		t.Fatalf("Expected failure. Got success with ordinal:%d", o)
+	}
+
+	if o, err := hnd.SetAnyInRange(0, firstAv-1, false); err == nil {
+		t.Fatalf("Expected failure. Got success with ordinal:%d", o)
+	}
+
+	if o, err := hnd.SetAnyInRange(111*uint64(blockLen), 161*uint64(blockLen), false); err == nil {
+		t.Fatalf("Expected failure. Got success with ordinal:%d", o)
+	}
+
+	o, err = hnd.SetAnyInRange(161*uint64(blockLen), 162*uint64(blockLen), false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if o != 161*uint64(blockLen)+30 {
+		t.Fatalf("Unexpected ordinal: %d", o)
+	}
+
+	o, err = hnd.SetAnyInRange(161*uint64(blockLen), 162*uint64(blockLen), false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if o != 161*uint64(blockLen)+31 {
+		t.Fatalf("Unexpected ordinal: %d", o)
+	}
+
+	o, err = hnd.SetAnyInRange(161*uint64(blockLen), 162*uint64(blockLen), false)
+	if err == nil {
+		t.Fatalf("Expected failure. Got success with ordinal:%d", o)
+	}
+
+	if _, err := hnd.SetAnyInRange(0, numBits-1, false); err != nil {
+		t.Fatalf("Unexpected failure: %v", err)
+	}
+
+	// set one bit using the set range with 1 bit size range
+	if _, err := hnd.SetAnyInRange(uint64(163*blockLen-1), uint64(163*blockLen-1), false); err != nil {
+		t.Fatal(err)
+	}
+
+	// create a non multiple of 32 mask
+	hnd = New(30)
+
+	// set all bit in the first range
+	for hnd.Unselected() > 22 {
+		if o, err := hnd.SetAnyInRange(0, 7, false); err != nil {
+			t.Fatalf("Unexpected failure: (%d, %v)", o, err)
+		}
+	}
+	// try one more set, which should fail
+	o, err = hnd.SetAnyInRange(0, 7, false)
+	if err == nil {
+		t.Fatalf("Expected failure. Got success with ordinal:%d", o)
+	}
+	if err != ErrNoBitAvailable {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+
+	// set all bit in a second range
+	for hnd.Unselected() > 14 {
+		if o, err := hnd.SetAnyInRange(8, 15, false); err != nil {
+			t.Fatalf("Unexpected failure: (%d, %v)", o, err)
+		}
+	}
+
+	// try one more set, which should fail
+	o, err = hnd.SetAnyInRange(0, 15, false)
+	if err == nil {
+		t.Fatalf("Expected failure. Got success with ordinal:%d", o)
+	}
+	if err != ErrNoBitAvailable {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+
+	// set all bit in a range which includes the last bit
+	for hnd.Unselected() > 12 {
+		if o, err := hnd.SetAnyInRange(28, 29, false); err != nil {
+			t.Fatalf("Unexpected failure: (%d, %v)", o, err)
+		}
+	}
+	o, err = hnd.SetAnyInRange(28, 29, false)
+	if err == nil {
+		t.Fatalf("Expected failure. Got success with ordinal:%d", o)
+	}
+	if err != ErrNoBitAvailable {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+}
+
+// This one tests an allocation pattern which unveiled an issue in pushReservation
+// Specifically a failure in detecting when we are in the (B) case (the bit to set
+// belongs to the last block of the current sequence). Because of a bug, code
+// was assuming the bit belonged to a block in the middle of the current sequence.
+// Which in turn caused an incorrect allocation when requesting a bit which is not
+// in the first or last sequence block.
+func TestSetAnyInRange(t *testing.T) {
+	numBits := uint64(8 * blockLen)
+	hnd := New(numBits)
+
+	if err := hnd.Set(0); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := hnd.Set(255); err != nil {
+		t.Fatal(err)
+	}
+
+	o, err := hnd.SetAnyInRange(128, 255, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if o != 128 {
+		t.Fatalf("Unexpected ordinal: %d", o)
+	}
+
+	o, err = hnd.SetAnyInRange(128, 255, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if o != 129 {
+		t.Fatalf("Unexpected ordinal: %d", o)
+	}
+
+	o, err = hnd.SetAnyInRange(246, 255, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if o != 246 {
+		t.Fatalf("Unexpected ordinal: %d", o)
+	}
+
+	o, err = hnd.SetAnyInRange(246, 255, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if o != 247 {
+		t.Fatalf("Unexpected ordinal: %d", o)
+	}
+}
+
+func TestMethods(t *testing.T) {
+	numBits := uint64(256 * blockLen)
+	hnd := New(numBits)
+
+	if hnd.Bits() != numBits {
+		t.Fatalf("Unexpected bit number: %d", hnd.Bits())
+	}
+
+	if hnd.Unselected() != numBits {
+		t.Fatalf("Unexpected bit number: %d", hnd.Unselected())
+	}
+
+	exp := "(0x0, 256)->end"
+	if hnd.head.toString() != exp {
+		t.Fatalf("Unexpected sequence string: %s", hnd.head.toString())
+	}
+
+	for i := 0; i < 192; i++ {
+		_, err := hnd.SetAny(false)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	exp = "(0xffffffff, 6)->(0x0, 250)->end"
+	if hnd.head.toString() != exp {
+		t.Fatalf("Unexpected sequence string: %s", hnd.head.toString())
+	}
+}
+
+func TestRandomAllocateDeallocate(t *testing.T) {
+	numBits := int(16 * blockLen)
+	hnd := New(uint64(numBits))
+
+	seed := time.Now().Unix()
+	rand.Seed(seed)
+
+	// Allocate all bits using a random pattern
+	pattern := rand.Perm(numBits)
+	for _, bit := range pattern {
+		err := hnd.Set(uint64(bit))
+		if err != nil {
+			t.Fatalf("Unexpected failure on allocation of %d: %v.\nSeed: %d.\n%s", bit, err, seed, hnd)
+		}
+	}
+	if hnd.Unselected() != 0 {
+		t.Fatalf("Expected full sequence. Instead found %d free bits. Seed: %d.\n%s", hnd.unselected, seed, hnd)
+	}
+	if hnd.head.toString() != "(0xffffffff, 16)->end" {
+		t.Fatalf("Unexpected db: %s", hnd.head.toString())
+	}
+
+	// Deallocate all bits using a random pattern
+	pattern = rand.Perm(numBits)
+	for _, bit := range pattern {
+		err := hnd.Unset(uint64(bit))
+		if err != nil {
+			t.Fatalf("Unexpected failure on deallocation of %d: %v.\nSeed: %d.\n%s", bit, err, seed, hnd)
+		}
+	}
+	if hnd.Unselected() != uint64(numBits) {
+		t.Fatalf("Expected full sequence. Instead found %d free bits. Seed: %d.\n%s", hnd.unselected, seed, hnd)
+	}
+	if hnd.head.toString() != "(0x0, 16)->end" {
+		t.Fatalf("Unexpected db: %s", hnd.head.toString())
+	}
+}
+
+func TestAllocateRandomDeallocate(t *testing.T) {
+	numBlocks := uint32(8)
+	numBits := int(numBlocks * blockLen)
+	hnd := New(uint64(numBits))
+
+	expected := &sequence{block: 0xffffffff, count: uint64(numBlocks / 2), next: &sequence{block: 0x0, count: uint64(numBlocks / 2)}}
+
+	// Allocate first half of the bits
+	for i := 0; i < numBits/2; i++ {
+		_, err := hnd.SetAny(false)
+		if err != nil {
+			t.Fatalf("Unexpected failure on allocation %d: %v\n%s", i, err, hnd)
+		}
+	}
+	if hnd.Unselected() != uint64(numBits/2) {
+		t.Fatalf("Expected full sequence. Instead found %d free bits. %s", hnd.unselected, hnd)
+	}
+	if !hnd.head.equal(expected) {
+		t.Fatalf("Unexpected sequence. Got:\n%s", hnd)
+	}
+
+	seed := time.Now().Unix()
+	rand.Seed(seed)
+
+	// Deallocate half of the allocated bits following a random pattern
+	pattern := rand.Perm(numBits / 2)
+	for i := 0; i < numBits/4; i++ {
+		bit := pattern[i]
+		err := hnd.Unset(uint64(bit))
+		if err != nil {
+			t.Fatalf("Unexpected failure on deallocation of %d: %v.\nSeed: %d.\n%s", bit, err, seed, hnd)
+		}
+	}
+	if hnd.Unselected() != uint64(3*numBits/4) {
+		t.Fatalf("Expected full sequence. Instead found %d free bits.\nSeed: %d.\n%s", hnd.unselected, seed, hnd)
+	}
+
+	// Request a quarter of bits
+	for i := 0; i < numBits/4; i++ {
+		_, err := hnd.SetAny(false)
+		if err != nil {
+			t.Fatalf("Unexpected failure on allocation %d: %v\nSeed: %d\n%s", i, err, seed, hnd)
+		}
+	}
+	if hnd.Unselected() != uint64(numBits/2) {
+		t.Fatalf("Expected half sequence. Instead found %d free bits.\nSeed: %d\n%s", hnd.unselected, seed, hnd)
+	}
+	if !hnd.head.equal(expected) {
+		t.Fatalf("Unexpected sequence. Got:\n%s", hnd)
+	}
+}
+
+func TestAllocateRandomDeallocateSerialize(t *testing.T) {
+
+	numBlocks := uint32(8)
+	numBits := int(numBlocks * blockLen)
+	hnd := New(uint64(numBits))
+
+	expected := &sequence{block: 0xffffffff, count: uint64(numBlocks / 2), next: &sequence{block: 0x0, count: uint64(numBlocks / 2)}}
+
+	// Allocate first half of the bits
+	for i := 0; i < numBits/2; i++ {
+		_, err := hnd.SetAny(true)
+		if err != nil {
+			t.Fatalf("Unexpected failure on allocation %d: %v\n%s", i, err, hnd)
+		}
+	}
+
+	if hnd.Unselected() != uint64(numBits/2) {
+		t.Fatalf("Expected full sequence. Instead found %d free bits. %s", hnd.unselected, hnd)
+	}
+	if !hnd.head.equal(expected) {
+		t.Fatalf("Unexpected sequence. Got:\n%s", hnd)
+	}
+
+	seed := time.Now().Unix()
+	rand.Seed(seed)
+
+	// Deallocate half of the allocated bits following a random pattern
+	pattern := rand.Perm(numBits / 2)
+	for i := 0; i < numBits/4; i++ {
+		bit := pattern[i]
+		err := hnd.Unset(uint64(bit))
+		if err != nil {
+			t.Fatalf("Unexpected failure on deallocation of %d: %v.\nSeed: %d.\n%s", bit, err, seed, hnd)
+		}
+	}
+	if hnd.Unselected() != uint64(3*numBits/4) {
+		t.Fatalf("Expected full sequence. Instead found %d free bits.\nSeed: %d.\n%s", hnd.unselected, seed, hnd)
+	}
+
+	// Request a quarter of bits
+	for i := 0; i < numBits/4; i++ {
+		_, err := hnd.SetAny(true)
+		if err != nil {
+			t.Fatalf("Unexpected failure on allocation %d: %v\nSeed: %d\n%s", i, err, seed, hnd)
+		}
+	}
+	if hnd.Unselected() != uint64(numBits/2) {
+		t.Fatalf("Expected half sequence. Instead found %d free bits.\nSeed: %d\n%s", hnd.unselected, seed, hnd)
+	}
+}
+
+func TestIsCorrupted(t *testing.T) {
+	// Negative test
+	hnd := New(1024)
+
+	if hnd.CheckConsistency() {
+		t.Fatalf("Unexpected corrupted for %s", hnd)
+	}
+
+	hnd.Set(0)
+	if hnd.CheckConsistency() {
+		t.Fatalf("Unexpected corrupted for %s", hnd)
+	}
+
+	hnd.Set(1023)
+	if hnd.CheckConsistency() {
+		t.Fatalf("Unexpected corrupted for %s", hnd)
+	}
+
+	// Try real corrupted ipam handles found in the local store files reported by three docker users,
+	// plus a generic ipam handle from docker 1.9.1. This last will fail as well, because of how the
+	// last node in the sequence is expressed (This is true for IPAM handle only, because of the broadcast
+	// address reservation: last bit). This will allow an application using bitseq that runs a consistency
+	// check to detect and replace the 1.9.0/1 old vulnerable handle with the new one.
+	input := []*Bitmap{
+		{
+			bits:       65536,
+			unselected: 65412,
+			head: &sequence{
+				block: 0xffffffff,
+				count: 3,
+				next: &sequence{
+					block: 0xffffffbf,
+					count: 0,
+					next: &sequence{
+						block: 0xfe98816e,
+						count: 1,
+						next: &sequence{
+							block: 0xffffffff,
+							count: 0,
+							next: &sequence{
+								block: 0xe3bc0000,
+								count: 1,
+								next: &sequence{
+									block: 0x0,
+									count: 2042,
+									next: &sequence{
+										block: 0x1, count: 1,
+										next: &sequence{
+											block: 0x0, count: 0,
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			bits:       65536,
+			unselected: 65319,
+			head: &sequence{
+				block: 0xffffffff,
+				count: 7,
+				next: &sequence{
+					block: 0xffffff7f,
+					count: 0,
+					next: &sequence{
+						block: 0xffffffff,
+						count: 0,
+						next: &sequence{
+							block: 0x2000000,
+							count: 1,
+							next: &sequence{
+								block: 0x0,
+								count: 2039,
+								next: &sequence{
+									block: 0x1,
+									count: 1,
+									next: &sequence{
+										block: 0x0,
+										count: 0,
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			bits:       65536,
+			unselected: 65456,
+			head: &sequence{
+				block: 0xffffffff, count: 2,
+				next: &sequence{
+					block: 0xfffbffff, count: 0,
+					next: &sequence{
+						block: 0xffd07000, count: 1,
+						next: &sequence{
+							block: 0x0, count: 333,
+							next: &sequence{
+								block: 0x40000000, count: 1,
+								next: &sequence{
+									block: 0x0, count: 1710,
+									next: &sequence{
+										block: 0x1, count: 1,
+										next: &sequence{
+											block: 0x0, count: 0,
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for idx, hnd := range input {
+		if !hnd.CheckConsistency() {
+			t.Fatalf("Expected corrupted for (%d): %s", idx, hnd)
+		}
+		if hnd.CheckConsistency() {
+			t.Fatalf("Sequence still marked corrupted (%d): %s", idx, hnd)
+		}
+	}
+}
+
+func testSetRollover(t *testing.T, serial bool) {
+	numBlocks := uint32(8)
+	numBits := int(numBlocks * blockLen)
+	hnd := New(uint64(numBits))
+
+	// Allocate first half of the bits
+	for i := 0; i < numBits/2; i++ {
+		_, err := hnd.SetAny(serial)
+		if err != nil {
+			t.Fatalf("Unexpected failure on allocation %d: %v\n%s", i, err, hnd)
+		}
+	}
+
+	if hnd.Unselected() != uint64(numBits/2) {
+		t.Fatalf("Expected full sequence. Instead found %d free bits. %s", hnd.unselected, hnd)
+	}
+
+	seed := time.Now().Unix()
+	rand.Seed(seed)
+
+	// Deallocate half of the allocated bits following a random pattern
+	pattern := rand.Perm(numBits / 2)
+	for i := 0; i < numBits/4; i++ {
+		bit := pattern[i]
+		err := hnd.Unset(uint64(bit))
+		if err != nil {
+			t.Fatalf("Unexpected failure on deallocation of %d: %v.\nSeed: %d.\n%s", bit, err, seed, hnd)
+		}
+	}
+	if hnd.Unselected() != uint64(3*numBits/4) {
+		t.Fatalf("Unexpected free bits: found %d free bits.\nSeed: %d.\n%s", hnd.unselected, seed, hnd)
+	}
+
+	//request to allocate for remaining half of the bits
+	for i := 0; i < numBits/2; i++ {
+		_, err := hnd.SetAny(serial)
+		if err != nil {
+			t.Fatalf("Unexpected failure on allocation %d: %v\nSeed: %d\n%s", i, err, seed, hnd)
+		}
+	}
+
+	//At this point all the bits must be allocated except the randomly unallocated bits
+	//which were unallocated in the first half of the bit sequence
+	if hnd.Unselected() != uint64(numBits/4) {
+		t.Fatalf("Unexpected number of unselected bits %d, Expected %d", hnd.Unselected(), numBits/4)
+	}
+
+	for i := 0; i < numBits/4; i++ {
+		_, err := hnd.SetAny(serial)
+		if err != nil {
+			t.Fatalf("Unexpected failure on allocation %d: %v\nSeed: %d\n%s", i, err, seed, hnd)
+		}
+	}
+	//Now requesting to allocate the unallocated random bits (qurter of the number of bits) should
+	//leave no more bits that can be allocated.
+	if hnd.Unselected() != 0 {
+		t.Fatalf("Unexpected number of unselected bits %d, Expected %d", hnd.Unselected(), 0)
+	}
+}
+
+func TestSetRollover(t *testing.T) {
+	testSetRollover(t, false)
+}
+
+func TestSetRolloverSerial(t *testing.T) {
+	testSetRollover(t, true)
+}
+
+func TestGetFirstAvailableFromCurrent(t *testing.T) {
+	input := []struct {
+		mask    *sequence
+		bytePos uint64
+		bitPos  uint64
+		start   uint64
+		curr    uint64
+		end     uint64
+	}{
+		{&sequence{block: 0xffffffff, count: 2048}, invalidPos, invalidPos, 0, 0, 65536},
+		{&sequence{block: 0x0, count: 8}, 0, 0, 0, 0, 256},
+		{&sequence{block: 0x80000000, count: 8}, 1, 0, 0, 8, 256},
+		{&sequence{block: 0xC0000000, count: 8}, 0, 2, 0, 2, 256},
+		{&sequence{block: 0xE0000000, count: 8}, 0, 3, 0, 0, 256},
+		{&sequence{block: 0xFFFB1FFF, count: 8}, 2, 0, 14, 0, 256},
+		{&sequence{block: 0xFFFFFFFE, count: 8}, 3, 7, 0, 0, 256},
+
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0x00000000, count: 1, next: &sequence{block: 0xffffffff, count: 14}}}, 4, 0, 0, 32, 512},
+		{&sequence{block: 0xfffeffff, count: 1, next: &sequence{block: 0xffffffff, count: 15}}, 1, 7, 0, 16, 512},
+		{&sequence{block: 0xfffeffff, count: 15, next: &sequence{block: 0xffffffff, count: 1}}, 5, 7, 0, 16, 512},
+		{&sequence{block: 0xfffeffff, count: 15, next: &sequence{block: 0xffffffff, count: 1}}, 9, 7, 0, 48, 512},
+		{&sequence{block: 0xffffffff, count: 2, next: &sequence{block: 0xffffffef, count: 14}}, 19, 3, 0, 124, 512},
+		{&sequence{block: 0xfffeffff, count: 15, next: &sequence{block: 0x0fffffff, count: 1}}, 60, 0, 0, 480, 512},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xfffeffff, count: 14, next: &sequence{block: 0xffffffff, count: 1}}}, 17, 7, 0, 124, 512},
+		{&sequence{block: 0xfffffffb, count: 1, next: &sequence{block: 0xffffffff, count: 14, next: &sequence{block: 0xffffffff, count: 1}}}, 3, 5, 0, 124, 512},
+		{&sequence{block: 0xfffffffb, count: 1, next: &sequence{block: 0xfffeffff, count: 14, next: &sequence{block: 0xffffffff, count: 1}}}, 13, 7, 0, 80, 512},
+	}
+
+	for n, i := range input {
+		bytePos, bitPos, _ := getAvailableFromCurrent(i.mask, i.start, i.curr, i.end)
+		if bytePos != i.bytePos || bitPos != i.bitPos {
+			t.Fatalf("Error in (%d) getFirstAvailable(). Expected (%d, %d). Got (%d, %d)", n, i.bytePos, i.bitPos, bytePos, bitPos)
+		}
+	}
+}
+
+func TestMarshalJSON(t *testing.T) {
+	expected := []byte("hello libnetwork")
+	hnd := New(uint64(len(expected) * 8))
+
+	for i, c := range expected {
+		for j := 0; j < 8; j++ {
+			if c&(1<<j) == 0 {
+				continue
+			}
+			if err := hnd.Set(uint64(i*8 + j)); err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
+
+	hstr := hnd.String()
+	t.Log(hstr)
+	marshaled, err := hnd.MarshalJSON()
+	if err != nil {
+		t.Fatalf("MarshalJSON() err = %v", err)
+	}
+	t.Logf("%s", marshaled)
+
+	// Serializations of hnd as would be marshaled by versions of the code
+	// found in the wild. We need to support unmarshaling old versions to
+	// maintain backwards compatibility with sequences persisted on disk.
+	const (
+		goldenV0 = `"AAAAAAAAAIAAAAAAAAAAPRamNjYAAAAAAAAAAfYENpYAAAAAAAAAAUZ2pi4AAAAAAAAAAe72TtYAAAAAAAAAAQ=="`
+	)
+
+	if string(marshaled) != goldenV0 {
+		t.Errorf("MarshalJSON() output differs from golden. Please add a new golden case to this test.")
+	}
+
+	for _, tt := range []struct {
+		name string
+		data []byte
+	}{
+		{name: "Live", data: marshaled},
+		{name: "Golden-v0", data: []byte(goldenV0)},
+	} {
+		tt := tt
+		t.Run("UnmarshalJSON="+tt.name, func(t *testing.T) {
+			hnd2 := New(0)
+			if err := hnd2.UnmarshalJSON(tt.data); err != nil {
+				t.Errorf("UnmarshalJSON() err = %v", err)
+			}
+
+			h2str := hnd2.String()
+			t.Log(h2str)
+			if hstr != h2str {
+				t.Errorf("Unmarshaled a different bitmap: want %q, got %q", hstr, h2str)
+			}
+		})
+	}
+}

libnetwork/bitseq/sequence.go

@@ -1,63 +1,44 @@
-// Package bitseq provides a structure and utilities for representing long bitmask
-// as sequence of run-length encoded blocks. It operates directly on the encoded
-// representation, it does not decode/encode.
+// Package bitseq provides a structure and utilities for representing a long
+// bitmask which is persisted in a datastore. It is backed by [bitmap.Bitmap]
+// which operates directly on the encoded representation, without uncompressing.
 package bitseq
 
 import (
-	"encoding/binary"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"sync"
 
+	"github.com/docker/docker/libnetwork/bitmap"
 	"github.com/docker/docker/libnetwork/datastore"
 	"github.com/docker/docker/libnetwork/types"
 	"github.com/sirupsen/logrus"
 )
 
-// block sequence constants
-// If needed we can think of making these configurable
-const (
-	blockLen      = uint32(32)
-	blockBytes    = uint64(blockLen / 8)
-	blockMAX      = uint32(1<<blockLen - 1)
-	blockFirstBit = uint32(1) << (blockLen - 1)
-	invalidPos    = uint64(0xFFFFFFFFFFFFFFFF)
-)
-
 var (
 	// ErrNoBitAvailable is returned when no more bits are available to set
-	ErrNoBitAvailable = errors.New("no bit available")
+	ErrNoBitAvailable = bitmap.ErrNoBitAvailable
 	// ErrBitAllocated is returned when the specific bit requested is already set
-	ErrBitAllocated = errors.New("requested bit is already allocated")
+	ErrBitAllocated = bitmap.ErrBitAllocated
 )
 
 // Handle contains the sequence representing the bitmask and its identifier
 type Handle struct {
-	bits       uint64
-	unselected uint64
-	head       *sequence
-	app        string
-	id         string
-	dbIndex    uint64
-	dbExists   bool
-	curr       uint64
-	store      datastore.DataStore
-	sync.Mutex
+	app      string
+	id       string
+	dbIndex  uint64
+	dbExists bool
+	store    datastore.DataStore
+	bm       *bitmap.Bitmap
+	mu       sync.Mutex
 }
 
 // NewHandle returns a thread-safe instance of the bitmask handler
 func NewHandle(app string, ds datastore.DataStore, id string, numElements uint64) (*Handle, error) {
 	h := &Handle{
-		app:        app,
-		id:         id,
-		store:      ds,
-		bits:       numElements,
-		unselected: numElements,
-		head: &sequence{
-			block: 0x0,
-			count: getNumBlocks(numElements),
-		},
+		bm:    bitmap.New(numElements),
+		app:   app,
+		id:    id,
+		store: ds,
 	}
 
 	if h.store == nil {
@@ -79,196 +60,54 @@ func NewHandle(app string, ds datastore.DataStore, id string, numElements uint64
 	return h, nil
 }
 
-// sequence represents a recurring sequence of 32 bits long bitmasks
-type sequence struct {
-	block uint32    // block is a symbol representing 4 byte long allocation bitmask
-	count uint64    // number of consecutive blocks (symbols)
-	next  *sequence // next sequence
-}
-
-// String returns a string representation of the block sequence starting from this block
-func (s *sequence) toString() string {
-	var nextBlock string
-	if s.next == nil {
-		nextBlock = "end"
-	} else {
-		nextBlock = s.next.toString()
-	}
-	return fmt.Sprintf("(0x%x, %d)->%s", s.block, s.count, nextBlock)
-}
-
-// GetAvailableBit returns the position of the first unset bit in the bitmask represented by this sequence
-func (s *sequence) getAvailableBit(from uint64) (uint64, uint64, error) {
-	if s.block == blockMAX || s.count == 0 {
-		return invalidPos, invalidPos, ErrNoBitAvailable
-	}
-	bits := from
-	bitSel := blockFirstBit >> from
-	for bitSel > 0 && s.block&bitSel != 0 {
-		bitSel >>= 1
-		bits++
-	}
-	// Check if the loop exited because it could not
-	// find any available bit int block  starting from
-	// "from". Return invalid pos in that case.
-	if bitSel == 0 {
-		return invalidPos, invalidPos, ErrNoBitAvailable
-	}
-	return bits / 8, bits % 8, nil
-}
-
-// GetCopy returns a copy of the linked list rooted at this node
-func (s *sequence) getCopy() *sequence {
-	n := &sequence{block: s.block, count: s.count}
-	pn := n
-	ps := s.next
-	for ps != nil {
-		pn.next = &sequence{block: ps.block, count: ps.count}
-		pn = pn.next
-		ps = ps.next
-	}
-	return n
-}
-
-// Equal checks if this sequence is equal to the passed one
-func (s *sequence) equal(o *sequence) bool {
-	this := s
-	other := o
-	for this != nil {
-		if other == nil {
-			return false
-		}
-		if this.block != other.block || this.count != other.count {
-			return false
-		}
-		this = this.next
-		other = other.next
-	}
-	return other == nil
-}
-
-// ToByteArray converts the sequence into a byte array
-func (s *sequence) toByteArray() ([]byte, error) {
-	var bb []byte
-
-	p := s
-	for p != nil {
-		b := make([]byte, 12)
-		binary.BigEndian.PutUint32(b[0:], p.block)
-		binary.BigEndian.PutUint64(b[4:], p.count)
-		bb = append(bb, b...)
-		p = p.next
-	}
-
-	return bb, nil
-}
-
-// fromByteArray construct the sequence from the byte array
-func (s *sequence) fromByteArray(data []byte) error {
-	l := len(data)
-	if l%12 != 0 {
-		return fmt.Errorf("cannot deserialize byte sequence of length %d (%v)", l, data)
-	}
-
-	p := s
-	i := 0
-	for {
-		p.block = binary.BigEndian.Uint32(data[i : i+4])
-		p.count = binary.BigEndian.Uint64(data[i+4 : i+12])
-		i += 12
-		if i == l {
-			break
-		}
-		p.next = &sequence{}
-		p = p.next
-	}
-
-	return nil
-}
-
 func (h *Handle) getCopy() *Handle {
 	return &Handle{
-		bits:       h.bits,
-		unselected: h.unselected,
-		head:       h.head.getCopy(),
-		app:        h.app,
-		id:         h.id,
-		dbIndex:    h.dbIndex,
-		dbExists:   h.dbExists,
-		store:      h.store,
-		curr:       h.curr,
+		bm:       bitmap.Copy(h.bm),
+		app:      h.app,
+		id:       h.id,
+		dbIndex:  h.dbIndex,
+		dbExists: h.dbExists,
+		store:    h.store,
 	}
 }
 
 // SetAnyInRange atomically sets the first unset bit in the specified range in the sequence and returns the corresponding ordinal
 func (h *Handle) SetAnyInRange(start, end uint64, serial bool) (uint64, error) {
-	if end < start || end >= h.bits {
-		return invalidPos, fmt.Errorf("invalid bit range [%d, %d]", start, end)
-	}
-	if h.Unselected() == 0 {
-		return invalidPos, ErrNoBitAvailable
-	}
-	return h.set(0, start, end, true, false, serial)
+	return h.apply(func(b *bitmap.Bitmap) (uint64, error) { return b.SetAnyInRange(start, end, serial) })
 }
 
 // SetAny atomically sets the first unset bit in the sequence and returns the corresponding ordinal
 func (h *Handle) SetAny(serial bool) (uint64, error) {
-	if h.Unselected() == 0 {
-		return invalidPos, ErrNoBitAvailable
-	}
-	return h.set(0, 0, h.bits-1, true, false, serial)
+	return h.apply(func(b *bitmap.Bitmap) (uint64, error) { return b.SetAny(serial) })
 }
 
 // Set atomically sets the corresponding bit in the sequence
 func (h *Handle) Set(ordinal uint64) error {
-	if err := h.validateOrdinal(ordinal); err != nil {
-		return err
-	}
-	_, err := h.set(ordinal, 0, 0, false, false, false)
+	_, err := h.apply(func(b *bitmap.Bitmap) (uint64, error) { return 0, b.Set(ordinal) })
 	return err
 }
 
 // Unset atomically unsets the corresponding bit in the sequence
 func (h *Handle) Unset(ordinal uint64) error {
-	if err := h.validateOrdinal(ordinal); err != nil {
-		return err
-	}
-	_, err := h.set(ordinal, 0, 0, false, true, false)
+	_, err := h.apply(func(b *bitmap.Bitmap) (uint64, error) { return 0, b.Unset(ordinal) })
 	return err
 }
 
 // IsSet atomically checks if the ordinal bit is set. In case ordinal
 // is outside of the bit sequence limits, false is returned.
 func (h *Handle) IsSet(ordinal uint64) bool {
-	if err := h.validateOrdinal(ordinal); err != nil {
-		return false
-	}
-	h.Lock()
-	_, _, err := checkIfAvailable(h.head, ordinal)
-	h.Unlock()
-	return err != nil
-}
-
-func (h *Handle) runConsistencyCheck() bool {
-	corrupted := false
-	for p, c := h.head, h.head.next; c != nil; c = c.next {
-		if c.count == 0 {
-			corrupted = true
-			p.next = c.next
-			continue // keep same p
-		}
-		p = c
-	}
-	return corrupted
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	return h.bm.IsSet(ordinal)
 }
 
 // CheckConsistency checks if the bit sequence is in an inconsistent state and attempts to fix it.
 // It looks for a corruption signature that may happen in docker 1.9.0 and 1.9.1.
 func (h *Handle) CheckConsistency() error {
 	for {
-		h.Lock()
+		h.mu.Lock()
 		store := h.store
-		h.Unlock()
+		h.mu.Unlock()
 
 		if store != nil {
 			if err := store.GetObject(datastore.Key(h.Key()...), h); err != nil && err != datastore.ErrKeyNotFound {
@@ -276,11 +115,11 @@ func (h *Handle) CheckConsistency() error {
 			}
 		}
 
-		h.Lock()
+		h.mu.Lock()
 		nh := h.getCopy()
-		h.Unlock()
+		h.mu.Unlock()
 
-		if !nh.runConsistencyCheck() {
+		if !nh.bm.CheckConsistency() {
 			return nil
 		}
 
@@ -293,70 +132,39 @@ func (h *Handle) CheckConsistency() error {
 
 		logrus.Infof("Fixed inconsistent bit sequence in datastore:\n%s\n%s", h, nh)
 
-		h.Lock()
-		h.head = nh.head
-		h.Unlock()
+		h.mu.Lock()
+		h.bm = nh.bm
+		h.mu.Unlock()
 
 		return nil
 	}
 }
 
 // set/reset the bit
-func (h *Handle) set(ordinal, start, end uint64, any bool, release bool, serial bool) (uint64, error) {
-	var (
-		bitPos  uint64
-		bytePos uint64
-		ret     uint64
-		err     error
-	)
-
+func (h *Handle) apply(op func(*bitmap.Bitmap) (uint64, error)) (uint64, error) {
 	for {
 		var store datastore.DataStore
-		curr := uint64(0)
-		h.Lock()
+		h.mu.Lock()
 		store = h.store
 		if store != nil {
-			h.Unlock() // The lock is acquired in the GetObject
+			h.mu.Unlock() // The lock is acquired in the GetObject
 			if err := store.GetObject(datastore.Key(h.Key()...), h); err != nil && err != datastore.ErrKeyNotFound {
-				return ret, err
-			}
-			h.Lock() // Acquire the lock back
-		}
-		if serial {
-			curr = h.curr
-		}
-		// Get position if available
-		if release {
-			bytePos, bitPos = ordinalToPos(ordinal)
-		} else {
-			if any {
-				bytePos, bitPos, err = getAvailableFromCurrent(h.head, start, curr, end)
-				ret = posToOrdinal(bytePos, bitPos)
-				if err == nil {
-					h.curr = ret + 1
-				}
-			} else {
-				bytePos, bitPos, err = checkIfAvailable(h.head, ordinal)
-				ret = ordinal
+				return 0, err
 			}
-		}
-		if err != nil {
-			h.Unlock()
-			return ret, err
+			h.mu.Lock() // Acquire the lock back
 		}
 
 		// Create a private copy of h and work on it
 		nh := h.getCopy()
 
-		nh.head = pushReservation(bytePos, bitPos, nh.head, release)
-		if release {
-			nh.unselected++
-		} else {
-			nh.unselected--
+		ret, err := op(nh.bm)
+		if err != nil {
+			h.mu.Unlock()
+			return ret, err
 		}
 
 		if h.store != nil {
-			h.Unlock()
+			h.mu.Unlock()
 			// Attempt to write private copy to store
 			if err := nh.writeToStore(); err != nil {
 				if _, ok := err.(types.RetryError); !ok {
@@ -365,29 +173,18 @@ func (h *Handle) set(ordinal, start, end uint64, any bool, release bool, serial
 				// Retry
 				continue
 			}
-			h.Lock()
+			h.mu.Lock()
 		}
 
 		// Previous atomic push was successful. Save private copy to local copy
-		h.unselected = nh.unselected
-		h.head = nh.head
+		h.bm = nh.bm
 		h.dbExists = nh.dbExists
 		h.dbIndex = nh.dbIndex
-		h.Unlock()
+		h.mu.Unlock()
 		return ret, nil
 	}
 }
 
-// checks is needed because to cover the case where the number of bits is not a multiple of blockLen
-func (h *Handle) validateOrdinal(ordinal uint64) error {
-	h.Lock()
-	defer h.Unlock()
-	if ordinal >= h.bits {
-		return errors.New("bit does not belong to the sequence")
-	}
-	return nil
-}
-
 // Destroy removes from the datastore the data belonging to this handle
 func (h *Handle) Destroy() error {
 	for {
@@ -408,324 +205,49 @@ func (h *Handle) Destroy() error {
 	}
 }
 
-// ToByteArray converts this handle's data into a byte array
-func (h *Handle) ToByteArray() ([]byte, error) {
-	h.Lock()
-	defer h.Unlock()
-	ba := make([]byte, 16)
-	binary.BigEndian.PutUint64(ba[0:], h.bits)
-	binary.BigEndian.PutUint64(ba[8:], h.unselected)
-	bm, err := h.head.toByteArray()
-	if err != nil {
-		return nil, fmt.Errorf("failed to serialize head: %s", err.Error())
-	}
-	ba = append(ba, bm...)
-
-	return ba, nil
-}
-
-// FromByteArray reads his handle's data from a byte array
-func (h *Handle) FromByteArray(ba []byte) error {
-	if ba == nil {
-		return errors.New("nil byte array")
-	}
-
-	nh := &sequence{}
-	err := nh.fromByteArray(ba[16:])
-	if err != nil {
-		return fmt.Errorf("failed to deserialize head: %s", err.Error())
-	}
-
-	h.Lock()
-	h.head = nh
-	h.bits = binary.BigEndian.Uint64(ba[0:8])
-	h.unselected = binary.BigEndian.Uint64(ba[8:16])
-	h.Unlock()
-
-	return nil
-}
-
 // Bits returns the length of the bit sequence
 func (h *Handle) Bits() uint64 {
-	return h.bits
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	return h.bm.Bits()
 }
 
 // Unselected returns the number of bits which are not selected
 func (h *Handle) Unselected() uint64 {
-	h.Lock()
-	defer h.Unlock()
-	return h.unselected
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	return h.bm.Unselected()
 }
 
 func (h *Handle) String() string {
-	h.Lock()
-	defer h.Unlock()
-	return fmt.Sprintf("App: %s, ID: %s, DBIndex: 0x%x, Bits: %d, Unselected: %d, Sequence: %s Curr:%d",
-		h.app, h.id, h.dbIndex, h.bits, h.unselected, h.head.toString(), h.curr)
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	return fmt.Sprintf("App: %s, ID: %s, DBIndex: 0x%x, %s",
+		h.app, h.id, h.dbIndex, h.bm)
 }
 
-// MarshalJSON encodes Handle into json message
-func (h *Handle) MarshalJSON() ([]byte, error) {
-	m := map[string]interface{}{
-		"id": h.id,
-	}
+type jsonMessage struct {
+	ID       string         `json:"id"`
+	Sequence *bitmap.Bitmap `json:"sequence"`
+}
 
-	b, err := h.ToByteArray()
-	if err != nil {
-		return nil, err
-	}
-	m["sequence"] = b
+// MarshalJSON encodes h into a JSON message.
+func (h *Handle) MarshalJSON() ([]byte, error) {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	m := jsonMessage{ID: h.id, Sequence: h.bm}
 	return json.Marshal(m)
 }
 
-// UnmarshalJSON decodes json message into Handle
+// UnmarshalJSON decodes a JSON message into h.
 func (h *Handle) UnmarshalJSON(data []byte) error {
-	var (
-		m   map[string]interface{}
-		b   []byte
-		err error
-	)
-	if err = json.Unmarshal(data, &m); err != nil {
-		return err
-	}
-	h.id = m["id"].(string)
-	bi, _ := json.Marshal(m["sequence"])
-	if err := json.Unmarshal(bi, &b); err != nil {
+	var m jsonMessage
+	if err := json.Unmarshal(data, &m); err != nil {
 		return err
 	}
-	return h.FromByteArray(b)
-}
-
-// getFirstAvailable looks for the first unset bit in passed mask starting from start
-func getFirstAvailable(head *sequence, start uint64) (uint64, uint64, error) {
-	// Find sequence which contains the start bit
-	byteStart, bitStart := ordinalToPos(start)
-	current, _, precBlocks, inBlockBytePos := findSequence(head, byteStart)
-	// Derive the this sequence offsets
-	byteOffset := byteStart - inBlockBytePos
-	bitOffset := inBlockBytePos*8 + bitStart
-	for current != nil {
-		if current.block != blockMAX {
-			// If the current block is not full, check if there is any bit
-			// from the current bit in the current block. If not, before proceeding to the
-			// next block node, make sure we check for available bit in the next
-			// instance of the same block. Due to RLE same block signature will be
-			// compressed.
-		retry:
-			bytePos, bitPos, err := current.getAvailableBit(bitOffset)
-			if err != nil && precBlocks == current.count-1 {
-				// This is the last instance in the same block node,
-				// so move to the next block.
-				goto next
-			}
-			if err != nil {
-				// There are some more instances of the same block, so add the offset
-				// and be optimistic that you will find the available bit in the next
-				// instance of the same block.
-				bitOffset = 0
-				byteOffset += blockBytes
-				precBlocks++
-				goto retry
-			}
-			return byteOffset + bytePos, bitPos, err
-		}
-		// Moving to next block: Reset bit offset.
-	next:
-		bitOffset = 0
-		byteOffset += (current.count * blockBytes) - (precBlocks * blockBytes)
-		precBlocks = 0
-		current = current.next
-	}
-	return invalidPos, invalidPos, ErrNoBitAvailable
-}
-
-// getAvailableFromCurrent will look for available ordinal from the current ordinal.
-// If none found then it will loop back to the start to check of the available bit.
-// This can be further optimized to check from start till curr in case of a rollover
-func getAvailableFromCurrent(head *sequence, start, curr, end uint64) (uint64, uint64, error) {
-	var bytePos, bitPos uint64
-	var err error
-	if curr != 0 && curr > start {
-		bytePos, bitPos, err = getFirstAvailable(head, curr)
-		ret := posToOrdinal(bytePos, bitPos)
-		if end < ret || err != nil {
-			goto begin
-		}
-		return bytePos, bitPos, nil
-	}
-
-begin:
-	bytePos, bitPos, err = getFirstAvailable(head, start)
-	ret := posToOrdinal(bytePos, bitPos)
-	if end < ret || err != nil {
-		return invalidPos, invalidPos, ErrNoBitAvailable
-	}
-	return bytePos, bitPos, nil
-}
-
-// checkIfAvailable checks if the bit correspondent to the specified ordinal is unset
-// If the ordinal is beyond the sequence limits, a negative response is returned
-func checkIfAvailable(head *sequence, ordinal uint64) (uint64, uint64, error) {
-	bytePos, bitPos := ordinalToPos(ordinal)
-
-	// Find the sequence containing this byte
-	current, _, _, inBlockBytePos := findSequence(head, bytePos)
-	if current != nil {
-		// Check whether the bit corresponding to the ordinal address is unset
-		bitSel := blockFirstBit >> (inBlockBytePos*8 + bitPos)
-		if current.block&bitSel == 0 {
-			return bytePos, bitPos, nil
-		}
-	}
-
-	return invalidPos, invalidPos, ErrBitAllocated
-}
-
-// Given the byte position and the sequences list head, return the pointer to the
-// sequence containing the byte (current), the pointer to the previous sequence,
-// the number of blocks preceding the block containing the byte inside the current sequence.
-// If bytePos is outside of the list, function will return (nil, nil, 0, invalidPos)
-func findSequence(head *sequence, bytePos uint64) (*sequence, *sequence, uint64, uint64) {
-	// Find the sequence containing this byte
-	previous := head
-	current := head
-	n := bytePos
-	for current.next != nil && n >= (current.count*blockBytes) { // Nil check for less than 32 addresses masks
-		n -= (current.count * blockBytes)
-		previous = current
-		current = current.next
-	}
 
-	// If byte is outside of the list, let caller know
-	if n >= (current.count * blockBytes) {
-		return nil, nil, 0, invalidPos
-	}
-
-	// Find the byte position inside the block and the number of blocks
-	// preceding the block containing the byte inside this sequence
-	precBlocks := n / blockBytes
-	inBlockBytePos := bytePos % blockBytes
-
-	return current, previous, precBlocks, inBlockBytePos
-}
-
-// PushReservation pushes the bit reservation inside the bitmask.
-// Given byte and bit positions, identify the sequence (current) which holds the block containing the affected bit.
-// Create a new block with the modified bit according to the operation (allocate/release).
-// Create a new sequence containing the new block and insert it in the proper position.
-// Remove current sequence if empty.
-// Check if new sequence can be merged with neighbour (previous/next) sequences.
-//
-// Identify "current" sequence containing block:
-//
-//	[prev seq] [current seq] [next seq]
-//
-// Based on block position, resulting list of sequences can be any of three forms:
-//
-// block position                        Resulting list of sequences
-//
-// A) block is first in current:         [prev seq] [new] [modified current seq] [next seq]
-// B) block is last in current:          [prev seq] [modified current seq] [new] [next seq]
-// C) block is in the middle of current: [prev seq] [curr pre] [new] [curr post] [next seq]
-func pushReservation(bytePos, bitPos uint64, head *sequence, release bool) *sequence {
-	// Store list's head
-	newHead := head
-
-	// Find the sequence containing this byte
-	current, previous, precBlocks, inBlockBytePos := findSequence(head, bytePos)
-	if current == nil {
-		return newHead
-	}
-
-	// Construct updated block
-	bitSel := blockFirstBit >> (inBlockBytePos*8 + bitPos)
-	newBlock := current.block
-	if release {
-		newBlock &^= bitSel
-	} else {
-		newBlock |= bitSel
-	}
-
-	// Quit if it was a redundant request
-	if current.block == newBlock {
-		return newHead
-	}
-
-	// Current sequence inevitably looses one block, upadate count
-	current.count--
-
-	// Create new sequence
-	newSequence := &sequence{block: newBlock, count: 1}
-
-	// Insert the new sequence in the list based on block position
-	if precBlocks == 0 { // First in sequence (A)
-		newSequence.next = current
-		if current == head {
-			newHead = newSequence
-			previous = newHead
-		} else {
-			previous.next = newSequence
-		}
-		removeCurrentIfEmpty(&newHead, newSequence, current)
-		mergeSequences(previous)
-	} else if precBlocks == current.count { // Last in sequence (B)
-		newSequence.next = current.next
-		current.next = newSequence
-		mergeSequences(current)
-	} else { // In between the sequence (C)
-		currPre := &sequence{block: current.block, count: precBlocks, next: newSequence}
-		currPost := current
-		currPost.count -= precBlocks
-		newSequence.next = currPost
-		if currPost == head {
-			newHead = currPre
-		} else {
-			previous.next = currPre
-		}
-		// No merging or empty current possible here
-	}
-
-	return newHead
-}
-
-// Removes the current sequence from the list if empty, adjusting the head pointer if needed
-func removeCurrentIfEmpty(head **sequence, previous, current *sequence) {
-	if current.count == 0 {
-		if current == *head {
-			*head = current.next
-		} else {
-			previous.next = current.next
-		}
-	}
-}
-
-// Given a pointer to a sequence, it checks if it can be merged with any following sequences
-// It stops when no more merging is possible.
-// TODO: Optimization: only attempt merge from start to end sequence, no need to scan till the end of the list
-func mergeSequences(seq *sequence) {
-	if seq != nil {
-		// Merge all what possible from seq
-		for seq.next != nil && seq.block == seq.next.block {
-			seq.count += seq.next.count
-			seq.next = seq.next.next
-		}
-		// Move to next
-		mergeSequences(seq.next)
-	}
-}
-
-func getNumBlocks(numBits uint64) uint64 {
-	numBlocks := numBits / uint64(blockLen)
-	if numBits%uint64(blockLen) != 0 {
-		numBlocks++
-	}
-	return numBlocks
-}
-
-func ordinalToPos(ordinal uint64) (uint64, uint64) {
-	return ordinal / 8, ordinal % 8
-}
-
-func posToOrdinal(bytePos, bitPos uint64) uint64 {
-	return bytePos*8 + bitPos
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	h.id, h.bm = m.ID, m.Sequence
+	return nil
 }

libnetwork/bitseq/store.go

@@ -3,21 +3,22 @@ package bitseq
 import (
 	"encoding/json"
 
+	"github.com/docker/docker/libnetwork/bitmap"
 	"github.com/docker/docker/libnetwork/datastore"
 	"github.com/docker/docker/libnetwork/types"
 )
 
 // Key provides the Key to be used in KV Store
 func (h *Handle) Key() []string {
-	h.Lock()
-	defer h.Unlock()
+	h.mu.Lock()
+	defer h.mu.Unlock()
 	return []string{h.app, h.id}
 }
 
 // KeyPrefix returns the immediate parent key that can be used for tree walk
 func (h *Handle) KeyPrefix() []string {
-	h.Lock()
-	defer h.Unlock()
+	h.mu.Lock()
+	defer h.mu.Unlock()
 	return []string{h.app}
 }
 
@@ -37,30 +38,30 @@ func (h *Handle) SetValue(value []byte) error {
 
 // Index returns the latest DB Index as seen by this object
 func (h *Handle) Index() uint64 {
-	h.Lock()
-	defer h.Unlock()
+	h.mu.Lock()
+	defer h.mu.Unlock()
 	return h.dbIndex
 }
 
 // SetIndex method allows the datastore to store the latest DB Index into this object
 func (h *Handle) SetIndex(index uint64) {
-	h.Lock()
+	h.mu.Lock()
 	h.dbIndex = index
 	h.dbExists = true
-	h.Unlock()
+	h.mu.Unlock()
 }
 
 // Exists method is true if this object has been stored in the DB.
 func (h *Handle) Exists() bool {
-	h.Lock()
-	defer h.Unlock()
+	h.mu.Lock()
+	defer h.mu.Unlock()
 	return h.dbExists
 }
 
 // New method returns a handle based on the receiver handle
 func (h *Handle) New() datastore.KVObject {
-	h.Lock()
-	defer h.Unlock()
+	h.mu.Lock()
+	defer h.mu.Unlock()
 
 	return &Handle{
 		app:   h.app,
@@ -70,24 +71,21 @@ func (h *Handle) New() datastore.KVObject {
 
 // CopyTo deep copies the handle into the passed destination object
 func (h *Handle) CopyTo(o datastore.KVObject) error {
-	h.Lock()
-	defer h.Unlock()
+	h.mu.Lock()
+	defer h.mu.Unlock()
 
 	dstH := o.(*Handle)
 	if h == dstH {
 		return nil
 	}
-	dstH.Lock()
-	dstH.bits = h.bits
-	dstH.unselected = h.unselected
-	dstH.head = h.head.getCopy()
+	dstH.mu.Lock()
+	defer dstH.mu.Unlock()
+	dstH.bm = bitmap.Copy(h.bm)
 	dstH.app = h.app
 	dstH.id = h.id
 	dstH.dbIndex = h.dbIndex
 	dstH.dbExists = h.dbExists
 	dstH.store = h.store
-	dstH.curr = h.curr
-	dstH.Unlock()
 
 	return nil
 }
@@ -99,16 +97,16 @@ func (h *Handle) Skip() bool {
 
 // DataScope method returns the storage scope of the datastore
 func (h *Handle) DataScope() string {
-	h.Lock()
-	defer h.Unlock()
+	h.mu.Lock()
+	defer h.mu.Unlock()
 
 	return h.store.Scope()
 }
 
 func (h *Handle) writeToStore() error {
-	h.Lock()
+	h.mu.Lock()
 	store := h.store
-	h.Unlock()
+	h.mu.Unlock()
 	if store == nil {
 		return nil
 	}
@@ -120,9 +118,9 @@ func (h *Handle) writeToStore() error {
 }
 
 func (h *Handle) deleteFromStore() error {
-	h.Lock()
+	h.mu.Lock()
 	store := h.store
-	h.Unlock()
+	h.mu.Unlock()
 	if store == nil {
 		return nil
 	}