libnetwork/bitmap: dup from package bitseq

...in preparation for separating the bit-sequence datatype from the
datastore persistence (KVObject) concerns. The new package's contents
are identical to those of package libnetwork/bitseq to assist in
reviewing the changes made on each side of the split.

Signed-off-by: Cory Snider <csnider@mirantis.com>

libnetwork/bitmap/sequence.go

@@ -0,0 +1,731 @@
+// Package bitseq provides a structure and utilities for representing long bitmask
+// as sequence of run-length encoded blocks. It operates directly on the encoded
+// representation, it does not decode/encode.
+package bitseq
+
+import (
+	"encoding/binary"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"sync"
+
+	"github.com/docker/docker/libnetwork/datastore"
+	"github.com/docker/docker/libnetwork/types"
+	"github.com/sirupsen/logrus"
+)
+
+// block sequence constants
+// If needed we can think of making these configurable
+const (
+	blockLen      = uint32(32)
+	blockBytes    = uint64(blockLen / 8)
+	blockMAX      = uint32(1<<blockLen - 1)
+	blockFirstBit = uint32(1) << (blockLen - 1)
+	invalidPos    = uint64(0xFFFFFFFFFFFFFFFF)
+)
+
+var (
+	// ErrNoBitAvailable is returned when no more bits are available to set
+	ErrNoBitAvailable = errors.New("no bit available")
+	// ErrBitAllocated is returned when the specific bit requested is already set
+	ErrBitAllocated = errors.New("requested bit is already allocated")
+)
+
+// Handle contains the sequence representing the bitmask and its identifier
+type Handle struct {
+	bits       uint64
+	unselected uint64
+	head       *sequence
+	app        string
+	id         string
+	dbIndex    uint64
+	dbExists   bool
+	curr       uint64
+	store      datastore.DataStore
+	sync.Mutex
+}
+
+// NewHandle returns a thread-safe instance of the bitmask handler
+func NewHandle(app string, ds datastore.DataStore, id string, numElements uint64) (*Handle, error) {
+	h := &Handle{
+		app:        app,
+		id:         id,
+		store:      ds,
+		bits:       numElements,
+		unselected: numElements,
+		head: &sequence{
+			block: 0x0,
+			count: getNumBlocks(numElements),
+		},
+	}
+
+	if h.store == nil {
+		return h, nil
+	}
+
+	// Get the initial status from the ds if present.
+	if err := h.store.GetObject(datastore.Key(h.Key()...), h); err != nil && err != datastore.ErrKeyNotFound {
+		return nil, err
+	}
+
+	// If the handle is not in store, write it.
+	if !h.Exists() {
+		if err := h.writeToStore(); err != nil {
+			return nil, fmt.Errorf("failed to write bitsequence to store: %v", err)
+		}
+	}
+
+	return h, nil
+}
+
+// sequence represents a recurring sequence of 32 bits long bitmasks
+type sequence struct {
+	block uint32    // block is a symbol representing 4 byte long allocation bitmask
+	count uint64    // number of consecutive blocks (symbols)
+	next  *sequence // next sequence
+}
+
+// String returns a string representation of the block sequence starting from this block
+func (s *sequence) toString() string {
+	var nextBlock string
+	if s.next == nil {
+		nextBlock = "end"
+	} else {
+		nextBlock = s.next.toString()
+	}
+	return fmt.Sprintf("(0x%x, %d)->%s", s.block, s.count, nextBlock)
+}
+
+// GetAvailableBit returns the position of the first unset bit in the bitmask represented by this sequence
+func (s *sequence) getAvailableBit(from uint64) (uint64, uint64, error) {
+	if s.block == blockMAX || s.count == 0 {
+		return invalidPos, invalidPos, ErrNoBitAvailable
+	}
+	bits := from
+	bitSel := blockFirstBit >> from
+	for bitSel > 0 && s.block&bitSel != 0 {
+		bitSel >>= 1
+		bits++
+	}
+	// Check if the loop exited because it could not
+	// find any available bit int block  starting from
+	// "from". Return invalid pos in that case.
+	if bitSel == 0 {
+		return invalidPos, invalidPos, ErrNoBitAvailable
+	}
+	return bits / 8, bits % 8, nil
+}
+
+// GetCopy returns a copy of the linked list rooted at this node
+func (s *sequence) getCopy() *sequence {
+	n := &sequence{block: s.block, count: s.count}
+	pn := n
+	ps := s.next
+	for ps != nil {
+		pn.next = &sequence{block: ps.block, count: ps.count}
+		pn = pn.next
+		ps = ps.next
+	}
+	return n
+}
+
+// Equal checks if this sequence is equal to the passed one
+func (s *sequence) equal(o *sequence) bool {
+	this := s
+	other := o
+	for this != nil {
+		if other == nil {
+			return false
+		}
+		if this.block != other.block || this.count != other.count {
+			return false
+		}
+		this = this.next
+		other = other.next
+	}
+	return other == nil
+}
+
+// ToByteArray converts the sequence into a byte array
+func (s *sequence) toByteArray() ([]byte, error) {
+	var bb []byte
+
+	p := s
+	for p != nil {
+		b := make([]byte, 12)
+		binary.BigEndian.PutUint32(b[0:], p.block)
+		binary.BigEndian.PutUint64(b[4:], p.count)
+		bb = append(bb, b...)
+		p = p.next
+	}
+
+	return bb, nil
+}
+
+// fromByteArray construct the sequence from the byte array
+func (s *sequence) fromByteArray(data []byte) error {
+	l := len(data)
+	if l%12 != 0 {
+		return fmt.Errorf("cannot deserialize byte sequence of length %d (%v)", l, data)
+	}
+
+	p := s
+	i := 0
+	for {
+		p.block = binary.BigEndian.Uint32(data[i : i+4])
+		p.count = binary.BigEndian.Uint64(data[i+4 : i+12])
+		i += 12
+		if i == l {
+			break
+		}
+		p.next = &sequence{}
+		p = p.next
+	}
+
+	return nil
+}
+
+func (h *Handle) getCopy() *Handle {
+	return &Handle{
+		bits:       h.bits,
+		unselected: h.unselected,
+		head:       h.head.getCopy(),
+		app:        h.app,
+		id:         h.id,
+		dbIndex:    h.dbIndex,
+		dbExists:   h.dbExists,
+		store:      h.store,
+		curr:       h.curr,
+	}
+}
+
+// SetAnyInRange atomically sets the first unset bit in the specified range in the sequence and returns the corresponding ordinal
+func (h *Handle) SetAnyInRange(start, end uint64, serial bool) (uint64, error) {
+	if end < start || end >= h.bits {
+		return invalidPos, fmt.Errorf("invalid bit range [%d, %d]", start, end)
+	}
+	if h.Unselected() == 0 {
+		return invalidPos, ErrNoBitAvailable
+	}
+	return h.set(0, start, end, true, false, serial)
+}
+
+// SetAny atomically sets the first unset bit in the sequence and returns the corresponding ordinal
+func (h *Handle) SetAny(serial bool) (uint64, error) {
+	if h.Unselected() == 0 {
+		return invalidPos, ErrNoBitAvailable
+	}
+	return h.set(0, 0, h.bits-1, true, false, serial)
+}
+
+// Set atomically sets the corresponding bit in the sequence
+func (h *Handle) Set(ordinal uint64) error {
+	if err := h.validateOrdinal(ordinal); err != nil {
+		return err
+	}
+	_, err := h.set(ordinal, 0, 0, false, false, false)
+	return err
+}
+
+// Unset atomically unsets the corresponding bit in the sequence
+func (h *Handle) Unset(ordinal uint64) error {
+	if err := h.validateOrdinal(ordinal); err != nil {
+		return err
+	}
+	_, err := h.set(ordinal, 0, 0, false, true, false)
+	return err
+}
+
+// IsSet atomically checks if the ordinal bit is set. In case ordinal
+// is outside of the bit sequence limits, false is returned.
+func (h *Handle) IsSet(ordinal uint64) bool {
+	if err := h.validateOrdinal(ordinal); err != nil {
+		return false
+	}
+	h.Lock()
+	_, _, err := checkIfAvailable(h.head, ordinal)
+	h.Unlock()
+	return err != nil
+}
+
+func (h *Handle) runConsistencyCheck() bool {
+	corrupted := false
+	for p, c := h.head, h.head.next; c != nil; c = c.next {
+		if c.count == 0 {
+			corrupted = true
+			p.next = c.next
+			continue // keep same p
+		}
+		p = c
+	}
+	return corrupted
+}
+
+// CheckConsistency checks if the bit sequence is in an inconsistent state and attempts to fix it.
+// It looks for a corruption signature that may happen in docker 1.9.0 and 1.9.1.
+func (h *Handle) CheckConsistency() error {
+	for {
+		h.Lock()
+		store := h.store
+		h.Unlock()
+
+		if store != nil {
+			if err := store.GetObject(datastore.Key(h.Key()...), h); err != nil && err != datastore.ErrKeyNotFound {
+				return err
+			}
+		}
+
+		h.Lock()
+		nh := h.getCopy()
+		h.Unlock()
+
+		if !nh.runConsistencyCheck() {
+			return nil
+		}
+
+		if err := nh.writeToStore(); err != nil {
+			if _, ok := err.(types.RetryError); !ok {
+				return fmt.Errorf("internal failure while fixing inconsistent bitsequence: %v", err)
+			}
+			continue
+		}
+
+		logrus.Infof("Fixed inconsistent bit sequence in datastore:\n%s\n%s", h, nh)
+
+		h.Lock()
+		h.head = nh.head
+		h.Unlock()
+
+		return nil
+	}
+}
+
+// set/reset the bit
+func (h *Handle) set(ordinal, start, end uint64, any bool, release bool, serial bool) (uint64, error) {
+	var (
+		bitPos  uint64
+		bytePos uint64
+		ret     uint64
+		err     error
+	)
+
+	for {
+		var store datastore.DataStore
+		curr := uint64(0)
+		h.Lock()
+		store = h.store
+		if store != nil {
+			h.Unlock() // The lock is acquired in the GetObject
+			if err := store.GetObject(datastore.Key(h.Key()...), h); err != nil && err != datastore.ErrKeyNotFound {
+				return ret, err
+			}
+			h.Lock() // Acquire the lock back
+		}
+		if serial {
+			curr = h.curr
+		}
+		// Get position if available
+		if release {
+			bytePos, bitPos = ordinalToPos(ordinal)
+		} else {
+			if any {
+				bytePos, bitPos, err = getAvailableFromCurrent(h.head, start, curr, end)
+				ret = posToOrdinal(bytePos, bitPos)
+				if err == nil {
+					h.curr = ret + 1
+				}
+			} else {
+				bytePos, bitPos, err = checkIfAvailable(h.head, ordinal)
+				ret = ordinal
+			}
+		}
+		if err != nil {
+			h.Unlock()
+			return ret, err
+		}
+
+		// Create a private copy of h and work on it
+		nh := h.getCopy()
+
+		nh.head = pushReservation(bytePos, bitPos, nh.head, release)
+		if release {
+			nh.unselected++
+		} else {
+			nh.unselected--
+		}
+
+		if h.store != nil {
+			h.Unlock()
+			// Attempt to write private copy to store
+			if err := nh.writeToStore(); err != nil {
+				if _, ok := err.(types.RetryError); !ok {
+					return ret, fmt.Errorf("internal failure while setting the bit: %v", err)
+				}
+				// Retry
+				continue
+			}
+			h.Lock()
+		}
+
+		// Previous atomic push was successful. Save private copy to local copy
+		h.unselected = nh.unselected
+		h.head = nh.head
+		h.dbExists = nh.dbExists
+		h.dbIndex = nh.dbIndex
+		h.Unlock()
+		return ret, nil
+	}
+}
+
+// checks is needed because to cover the case where the number of bits is not a multiple of blockLen
+func (h *Handle) validateOrdinal(ordinal uint64) error {
+	h.Lock()
+	defer h.Unlock()
+	if ordinal >= h.bits {
+		return errors.New("bit does not belong to the sequence")
+	}
+	return nil
+}
+
+// Destroy removes from the datastore the data belonging to this handle
+func (h *Handle) Destroy() error {
+	for {
+		if err := h.deleteFromStore(); err != nil {
+			if _, ok := err.(types.RetryError); !ok {
+				return fmt.Errorf("internal failure while destroying the sequence: %v", err)
+			}
+			// Fetch latest
+			if err := h.store.GetObject(datastore.Key(h.Key()...), h); err != nil {
+				if err == datastore.ErrKeyNotFound { // already removed
+					return nil
+				}
+				return fmt.Errorf("failed to fetch from store when destroying the sequence: %v", err)
+			}
+			continue
+		}
+		return nil
+	}
+}
+
+// ToByteArray converts this handle's data into a byte array
+func (h *Handle) ToByteArray() ([]byte, error) {
+	h.Lock()
+	defer h.Unlock()
+	ba := make([]byte, 16)
+	binary.BigEndian.PutUint64(ba[0:], h.bits)
+	binary.BigEndian.PutUint64(ba[8:], h.unselected)
+	bm, err := h.head.toByteArray()
+	if err != nil {
+		return nil, fmt.Errorf("failed to serialize head: %s", err.Error())
+	}
+	ba = append(ba, bm...)
+
+	return ba, nil
+}
+
+// FromByteArray reads his handle's data from a byte array
+func (h *Handle) FromByteArray(ba []byte) error {
+	if ba == nil {
+		return errors.New("nil byte array")
+	}
+
+	nh := &sequence{}
+	err := nh.fromByteArray(ba[16:])
+	if err != nil {
+		return fmt.Errorf("failed to deserialize head: %s", err.Error())
+	}
+
+	h.Lock()
+	h.head = nh
+	h.bits = binary.BigEndian.Uint64(ba[0:8])
+	h.unselected = binary.BigEndian.Uint64(ba[8:16])
+	h.Unlock()
+
+	return nil
+}
+
+// Bits returns the length of the bit sequence
+func (h *Handle) Bits() uint64 {
+	return h.bits
+}
+
+// Unselected returns the number of bits which are not selected
+func (h *Handle) Unselected() uint64 {
+	h.Lock()
+	defer h.Unlock()
+	return h.unselected
+}
+
+func (h *Handle) String() string {
+	h.Lock()
+	defer h.Unlock()
+	return fmt.Sprintf("App: %s, ID: %s, DBIndex: 0x%x, Bits: %d, Unselected: %d, Sequence: %s Curr:%d",
+		h.app, h.id, h.dbIndex, h.bits, h.unselected, h.head.toString(), h.curr)
+}
+
+// MarshalJSON encodes Handle into json message
+func (h *Handle) MarshalJSON() ([]byte, error) {
+	m := map[string]interface{}{
+		"id": h.id,
+	}
+
+	b, err := h.ToByteArray()
+	if err != nil {
+		return nil, err
+	}
+	m["sequence"] = b
+	return json.Marshal(m)
+}
+
+// UnmarshalJSON decodes json message into Handle
+func (h *Handle) UnmarshalJSON(data []byte) error {
+	var (
+		m   map[string]interface{}
+		b   []byte
+		err error
+	)
+	if err = json.Unmarshal(data, &m); err != nil {
+		return err
+	}
+	h.id = m["id"].(string)
+	bi, _ := json.Marshal(m["sequence"])
+	if err := json.Unmarshal(bi, &b); err != nil {
+		return err
+	}
+	return h.FromByteArray(b)
+}
+
+// getFirstAvailable looks for the first unset bit in passed mask starting from start
+func getFirstAvailable(head *sequence, start uint64) (uint64, uint64, error) {
+	// Find sequence which contains the start bit
+	byteStart, bitStart := ordinalToPos(start)
+	current, _, precBlocks, inBlockBytePos := findSequence(head, byteStart)
+	// Derive the this sequence offsets
+	byteOffset := byteStart - inBlockBytePos
+	bitOffset := inBlockBytePos*8 + bitStart
+	for current != nil {
+		if current.block != blockMAX {
+			// If the current block is not full, check if there is any bit
+			// from the current bit in the current block. If not, before proceeding to the
+			// next block node, make sure we check for available bit in the next
+			// instance of the same block. Due to RLE same block signature will be
+			// compressed.
+		retry:
+			bytePos, bitPos, err := current.getAvailableBit(bitOffset)
+			if err != nil && precBlocks == current.count-1 {
+				// This is the last instance in the same block node,
+				// so move to the next block.
+				goto next
+			}
+			if err != nil {
+				// There are some more instances of the same block, so add the offset
+				// and be optimistic that you will find the available bit in the next
+				// instance of the same block.
+				bitOffset = 0
+				byteOffset += blockBytes
+				precBlocks++
+				goto retry
+			}
+			return byteOffset + bytePos, bitPos, err
+		}
+		// Moving to next block: Reset bit offset.
+	next:
+		bitOffset = 0
+		byteOffset += (current.count * blockBytes) - (precBlocks * blockBytes)
+		precBlocks = 0
+		current = current.next
+	}
+	return invalidPos, invalidPos, ErrNoBitAvailable
+}
+
+// getAvailableFromCurrent will look for available ordinal from the current ordinal.
+// If none found then it will loop back to the start to check of the available bit.
+// This can be further optimized to check from start till curr in case of a rollover
+func getAvailableFromCurrent(head *sequence, start, curr, end uint64) (uint64, uint64, error) {
+	var bytePos, bitPos uint64
+	var err error
+	if curr != 0 && curr > start {
+		bytePos, bitPos, err = getFirstAvailable(head, curr)
+		ret := posToOrdinal(bytePos, bitPos)
+		if end < ret || err != nil {
+			goto begin
+		}
+		return bytePos, bitPos, nil
+	}
+
+begin:
+	bytePos, bitPos, err = getFirstAvailable(head, start)
+	ret := posToOrdinal(bytePos, bitPos)
+	if end < ret || err != nil {
+		return invalidPos, invalidPos, ErrNoBitAvailable
+	}
+	return bytePos, bitPos, nil
+}
+
+// checkIfAvailable checks if the bit correspondent to the specified ordinal is unset
+// If the ordinal is beyond the sequence limits, a negative response is returned
+func checkIfAvailable(head *sequence, ordinal uint64) (uint64, uint64, error) {
+	bytePos, bitPos := ordinalToPos(ordinal)
+
+	// Find the sequence containing this byte
+	current, _, _, inBlockBytePos := findSequence(head, bytePos)
+	if current != nil {
+		// Check whether the bit corresponding to the ordinal address is unset
+		bitSel := blockFirstBit >> (inBlockBytePos*8 + bitPos)
+		if current.block&bitSel == 0 {
+			return bytePos, bitPos, nil
+		}
+	}
+
+	return invalidPos, invalidPos, ErrBitAllocated
+}
+
+// Given the byte position and the sequences list head, return the pointer to the
+// sequence containing the byte (current), the pointer to the previous sequence,
+// the number of blocks preceding the block containing the byte inside the current sequence.
+// If bytePos is outside of the list, function will return (nil, nil, 0, invalidPos)
+func findSequence(head *sequence, bytePos uint64) (*sequence, *sequence, uint64, uint64) {
+	// Find the sequence containing this byte
+	previous := head
+	current := head
+	n := bytePos
+	for current.next != nil && n >= (current.count*blockBytes) { // Nil check for less than 32 addresses masks
+		n -= (current.count * blockBytes)
+		previous = current
+		current = current.next
+	}
+
+	// If byte is outside of the list, let caller know
+	if n >= (current.count * blockBytes) {
+		return nil, nil, 0, invalidPos
+	}
+
+	// Find the byte position inside the block and the number of blocks
+	// preceding the block containing the byte inside this sequence
+	precBlocks := n / blockBytes
+	inBlockBytePos := bytePos % blockBytes
+
+	return current, previous, precBlocks, inBlockBytePos
+}
+
+// PushReservation pushes the bit reservation inside the bitmask.
+// Given byte and bit positions, identify the sequence (current) which holds the block containing the affected bit.
+// Create a new block with the modified bit according to the operation (allocate/release).
+// Create a new sequence containing the new block and insert it in the proper position.
+// Remove current sequence if empty.
+// Check if new sequence can be merged with neighbour (previous/next) sequences.
+//
+// Identify "current" sequence containing block:
+//
+//	[prev seq] [current seq] [next seq]
+//
+// Based on block position, resulting list of sequences can be any of three forms:
+//
+// block position                        Resulting list of sequences
+//
+// A) block is first in current:         [prev seq] [new] [modified current seq] [next seq]
+// B) block is last in current:          [prev seq] [modified current seq] [new] [next seq]
+// C) block is in the middle of current: [prev seq] [curr pre] [new] [curr post] [next seq]
+func pushReservation(bytePos, bitPos uint64, head *sequence, release bool) *sequence {
+	// Store list's head
+	newHead := head
+
+	// Find the sequence containing this byte
+	current, previous, precBlocks, inBlockBytePos := findSequence(head, bytePos)
+	if current == nil {
+		return newHead
+	}
+
+	// Construct updated block
+	bitSel := blockFirstBit >> (inBlockBytePos*8 + bitPos)
+	newBlock := current.block
+	if release {
+		newBlock &^= bitSel
+	} else {
+		newBlock |= bitSel
+	}
+
+	// Quit if it was a redundant request
+	if current.block == newBlock {
+		return newHead
+	}
+
+	// Current sequence inevitably looses one block, upadate count
+	current.count--
+
+	// Create new sequence
+	newSequence := &sequence{block: newBlock, count: 1}
+
+	// Insert the new sequence in the list based on block position
+	if precBlocks == 0 { // First in sequence (A)
+		newSequence.next = current
+		if current == head {
+			newHead = newSequence
+			previous = newHead
+		} else {
+			previous.next = newSequence
+		}
+		removeCurrentIfEmpty(&newHead, newSequence, current)
+		mergeSequences(previous)
+	} else if precBlocks == current.count { // Last in sequence (B)
+		newSequence.next = current.next
+		current.next = newSequence
+		mergeSequences(current)
+	} else { // In between the sequence (C)
+		currPre := &sequence{block: current.block, count: precBlocks, next: newSequence}
+		currPost := current
+		currPost.count -= precBlocks
+		newSequence.next = currPost
+		if currPost == head {
+			newHead = currPre
+		} else {
+			previous.next = currPre
+		}
+		// No merging or empty current possible here
+	}
+
+	return newHead
+}
+
+// Removes the current sequence from the list if empty, adjusting the head pointer if needed
+func removeCurrentIfEmpty(head **sequence, previous, current *sequence) {
+	if current.count == 0 {
+		if current == *head {
+			*head = current.next
+		} else {
+			previous.next = current.next
+		}
+	}
+}
+
+// Given a pointer to a sequence, it checks if it can be merged with any following sequences
+// It stops when no more merging is possible.
+// TODO: Optimization: only attempt merge from start to end sequence, no need to scan till the end of the list
+func mergeSequences(seq *sequence) {
+	if seq != nil {
+		// Merge all what possible from seq
+		for seq.next != nil && seq.block == seq.next.block {
+			seq.count += seq.next.count
+			seq.next = seq.next.next
+		}
+		// Move to next
+		mergeSequences(seq.next)
+	}
+}
+
+func getNumBlocks(numBits uint64) uint64 {
+	numBlocks := numBits / uint64(blockLen)
+	if numBits%uint64(blockLen) != 0 {
+		numBlocks++
+	}
+	return numBlocks
+}
+
+func ordinalToPos(ordinal uint64) (uint64, uint64) {
+	return ordinal / 8, ordinal % 8
+}
+
+func posToOrdinal(bytePos, bitPos uint64) uint64 {
+	return bytePos*8 + bitPos
+}

libnetwork/bitmap/sequence_test.go

@@ -0,0 +1,1425 @@
+package bitseq
+
+import (
+	"fmt"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/docker/docker/libnetwork/datastore"
+	"github.com/docker/libkv/store"
+	"github.com/docker/libkv/store/boltdb"
+)
+
+var (
+	defaultPrefix = filepath.Join(os.TempDir(), "libnetwork", "test", "bitseq")
+)
+
+func init() {
+	boltdb.Register()
+}
+
+func randomLocalStore() (datastore.DataStore, error) {
+	tmp, err := os.CreateTemp("", "libnetwork-")
+	if err != nil {
+		return nil, fmt.Errorf("Error creating temp file: %v", err)
+	}
+	if err := tmp.Close(); err != nil {
+		return nil, fmt.Errorf("Error closing temp file: %v", err)
+	}
+	return datastore.NewDataStore(datastore.LocalScope, &datastore.ScopeCfg{
+		Client: datastore.ScopeClientCfg{
+			Provider: "boltdb",
+			Address:  filepath.Join(defaultPrefix, filepath.Base(tmp.Name())),
+			Config: &store.Config{
+				Bucket:            "libnetwork",
+				ConnectionTimeout: 3 * time.Second,
+			},
+		},
+	})
+}
+
+func TestSequenceGetAvailableBit(t *testing.T) {
+	input := []struct {
+		head    *sequence
+		from    uint64
+		bytePos uint64
+		bitPos  uint64
+	}{
+		{&sequence{block: 0x0, count: 0}, 0, invalidPos, invalidPos},
+		{&sequence{block: 0x0, count: 1}, 0, 0, 0},
+		{&sequence{block: 0x0, count: 100}, 0, 0, 0},
+
+		{&sequence{block: 0x80000000, count: 0}, 0, invalidPos, invalidPos},
+		{&sequence{block: 0x80000000, count: 1}, 0, 0, 1},
+		{&sequence{block: 0x80000000, count: 100}, 0, 0, 1},
+
+		{&sequence{block: 0xFF000000, count: 0}, 0, invalidPos, invalidPos},
+		{&sequence{block: 0xFF000000, count: 1}, 0, 1, 0},
+		{&sequence{block: 0xFF000000, count: 100}, 0, 1, 0},
+
+		{&sequence{block: 0xFF800000, count: 0}, 0, invalidPos, invalidPos},
+		{&sequence{block: 0xFF800000, count: 1}, 0, 1, 1},
+		{&sequence{block: 0xFF800000, count: 100}, 0, 1, 1},
+
+		{&sequence{block: 0xFFC0FF00, count: 0}, 0, invalidPos, invalidPos},
+		{&sequence{block: 0xFFC0FF00, count: 1}, 0, 1, 2},
+		{&sequence{block: 0xFFC0FF00, count: 100}, 0, 1, 2},
+
+		{&sequence{block: 0xFFE0FF00, count: 0}, 0, invalidPos, invalidPos},
+		{&sequence{block: 0xFFE0FF00, count: 1}, 0, 1, 3},
+		{&sequence{block: 0xFFE0FF00, count: 100}, 0, 1, 3},
+
+		{&sequence{block: 0xFFFEFF00, count: 0}, 0, invalidPos, invalidPos},
+		{&sequence{block: 0xFFFEFF00, count: 1}, 0, 1, 7},
+		{&sequence{block: 0xFFFEFF00, count: 100}, 0, 1, 7},
+
+		{&sequence{block: 0xFFFFC0FF, count: 0}, 0, invalidPos, invalidPos},
+		{&sequence{block: 0xFFFFC0FF, count: 1}, 0, 2, 2},
+		{&sequence{block: 0xFFFFC0FF, count: 100}, 0, 2, 2},
+
+		{&sequence{block: 0xFFFFFF00, count: 0}, 0, invalidPos, invalidPos},
+		{&sequence{block: 0xFFFFFF00, count: 1}, 0, 3, 0},
+		{&sequence{block: 0xFFFFFF00, count: 100}, 0, 3, 0},
+
+		{&sequence{block: 0xFFFFFFFE, count: 0}, 0, invalidPos, invalidPos},
+		{&sequence{block: 0xFFFFFFFE, count: 1}, 0, 3, 7},
+		{&sequence{block: 0xFFFFFFFE, count: 100}, 0, 3, 7},
+
+		{&sequence{block: 0xFFFFFFFF, count: 0}, 0, invalidPos, invalidPos},
+		{&sequence{block: 0xFFFFFFFF, count: 1}, 0, invalidPos, invalidPos},
+		{&sequence{block: 0xFFFFFFFF, count: 100}, 0, invalidPos, invalidPos},
+
+		// now test with offset
+		{&sequence{block: 0x0, count: 0}, 0, invalidPos, invalidPos},
+		{&sequence{block: 0x0, count: 0}, 31, invalidPos, invalidPos},
+		{&sequence{block: 0x0, count: 0}, 32, invalidPos, invalidPos},
+		{&sequence{block: 0x0, count: 1}, 0, 0, 0},
+		{&sequence{block: 0x0, count: 1}, 1, 0, 1},
+		{&sequence{block: 0x0, count: 1}, 31, 3, 7},
+		{&sequence{block: 0xF0FF0000, count: 1}, 0, 0, 4},
+		{&sequence{block: 0xF0FF0000, count: 1}, 8, 2, 0},
+		{&sequence{block: 0xFFFFFFFF, count: 1}, 0, invalidPos, invalidPos},
+		{&sequence{block: 0xFFFFFFFF, count: 1}, 16, invalidPos, invalidPos},
+		{&sequence{block: 0xFFFFFFFF, count: 1}, 31, invalidPos, invalidPos},
+		{&sequence{block: 0xFFFFFFFE, count: 1}, 0, 3, 7},
+		{&sequence{block: 0xFFFFFFFF, count: 2}, 0, invalidPos, invalidPos},
+		{&sequence{block: 0xFFFFFFFF, count: 2}, 32, invalidPos, invalidPos},
+	}
+
+	for n, i := range input {
+		b, bb, err := i.head.getAvailableBit(i.from)
+		if b != i.bytePos || bb != i.bitPos {
+			t.Fatalf("Error in sequence.getAvailableBit(%d) (%d).\nExp: (%d, %d)\nGot: (%d, %d), err: %v", i.from, n, i.bytePos, i.bitPos, b, bb, err)
+		}
+	}
+}
+
+func TestSequenceEqual(t *testing.T) {
+	input := []struct {
+		first    *sequence
+		second   *sequence
+		areEqual bool
+	}{
+		{&sequence{block: 0x0, count: 8, next: nil}, &sequence{block: 0x0, count: 8}, true},
+		{&sequence{block: 0x0, count: 0, next: nil}, &sequence{block: 0x0, count: 0}, true},
+		{&sequence{block: 0x0, count: 2, next: nil}, &sequence{block: 0x0, count: 1, next: &sequence{block: 0x0, count: 1}}, false},
+		{&sequence{block: 0x0, count: 2, next: &sequence{block: 0x1, count: 1}}, &sequence{block: 0x0, count: 2}, false},
+
+		{&sequence{block: 0x12345678, count: 8, next: nil}, &sequence{block: 0x12345678, count: 8}, true},
+		{&sequence{block: 0x12345678, count: 8, next: nil}, &sequence{block: 0x12345678, count: 9}, false},
+		{&sequence{block: 0x12345678, count: 1, next: &sequence{block: 0xFFFFFFFF, count: 1}}, &sequence{block: 0x12345678, count: 1}, false},
+		{&sequence{block: 0x12345678, count: 1}, &sequence{block: 0x12345678, count: 1, next: &sequence{block: 0xFFFFFFFF, count: 1}}, false},
+	}
+
+	for n, i := range input {
+		if i.areEqual != i.first.equal(i.second) {
+			t.Fatalf("Error in sequence.equal() (%d).\nExp: %t\nGot: %t,", n, i.areEqual, !i.areEqual)
+		}
+	}
+}
+
+func TestSequenceCopy(t *testing.T) {
+	s := getTestSequence()
+	n := s.getCopy()
+	if !s.equal(n) {
+		t.Fatal("copy of s failed")
+	}
+	if n == s {
+		t.Fatal("not true copy of s")
+	}
+}
+
+func TestGetFirstAvailable(t *testing.T) {
+	input := []struct {
+		mask    *sequence
+		bytePos uint64
+		bitPos  uint64
+		start   uint64
+	}{
+		{&sequence{block: 0xffffffff, count: 2048}, invalidPos, invalidPos, 0},
+		{&sequence{block: 0x0, count: 8}, 0, 0, 0},
+		{&sequence{block: 0x80000000, count: 8}, 0, 1, 0},
+		{&sequence{block: 0xC0000000, count: 8}, 0, 2, 0},
+		{&sequence{block: 0xE0000000, count: 8}, 0, 3, 0},
+		{&sequence{block: 0xF0000000, count: 8}, 0, 4, 0},
+		{&sequence{block: 0xF8000000, count: 8}, 0, 5, 0},
+		{&sequence{block: 0xFC000000, count: 8}, 0, 6, 0},
+		{&sequence{block: 0xFE000000, count: 8}, 0, 7, 0},
+		{&sequence{block: 0xFE000000, count: 8}, 3, 0, 24},
+
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0x00000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 4, 0, 0},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0x80000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 4, 1, 0},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xC0000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 4, 2, 0},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xE0000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 4, 3, 0},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xF0000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 4, 4, 0},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xF8000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 4, 5, 0},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFC000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 4, 6, 0},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFE000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 4, 7, 0},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0x0E000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 4, 0, 16},
+
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFF000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 5, 0, 0},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFF800000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 5, 1, 0},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFC00000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 5, 2, 0},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFE00000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 5, 3, 0},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFF00000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 5, 4, 0},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFF80000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 5, 5, 0},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFFC0000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 5, 6, 0},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFFE0000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 5, 7, 0},
+
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xfffffffe, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 7, 7, 0},
+
+		{&sequence{block: 0xffffffff, count: 2, next: &sequence{block: 0x0, count: 6}}, 8, 0, 0},
+		{&sequence{block: 0xfffcffff, count: 1, next: &sequence{block: 0x0, count: 6}}, 4, 0, 16},
+		{&sequence{block: 0xfffcffff, count: 1, next: &sequence{block: 0x0, count: 6}}, 1, 7, 15},
+		{&sequence{block: 0xfffcffff, count: 1, next: &sequence{block: 0x0, count: 6}}, 1, 6, 10},
+		{&sequence{block: 0xfffcfffe, count: 1, next: &sequence{block: 0x0, count: 6}}, 3, 7, 31},
+		{&sequence{block: 0xfffcffff, count: 1, next: &sequence{block: 0xffffffff, count: 6}}, invalidPos, invalidPos, 31},
+	}
+
+	for n, i := range input {
+		bytePos, bitPos, _ := getFirstAvailable(i.mask, i.start)
+		if bytePos != i.bytePos || bitPos != i.bitPos {
+			t.Fatalf("Error in (%d) getFirstAvailable(). Expected (%d, %d). Got (%d, %d)", n, i.bytePos, i.bitPos, bytePos, bitPos)
+		}
+	}
+}
+
+func TestFindSequence(t *testing.T) {
+	input := []struct {
+		head           *sequence
+		bytePos        uint64
+		precBlocks     uint64
+		inBlockBytePos uint64
+	}{
+		{&sequence{block: 0xffffffff, count: 0}, 0, 0, invalidPos},
+		{&sequence{block: 0xffffffff, count: 0}, 31, 0, invalidPos},
+		{&sequence{block: 0xffffffff, count: 0}, 100, 0, invalidPos},
+
+		{&sequence{block: 0x0, count: 1}, 0, 0, 0},
+		{&sequence{block: 0x0, count: 1}, 1, 0, 1},
+		{&sequence{block: 0x0, count: 1}, 31, 0, invalidPos},
+		{&sequence{block: 0x0, count: 1}, 60, 0, invalidPos},
+
+		{&sequence{block: 0xffffffff, count: 10}, 0, 0, 0},
+		{&sequence{block: 0xffffffff, count: 10}, 3, 0, 3},
+		{&sequence{block: 0xffffffff, count: 10}, 4, 1, 0},
+		{&sequence{block: 0xffffffff, count: 10}, 7, 1, 3},
+		{&sequence{block: 0xffffffff, count: 10}, 8, 2, 0},
+		{&sequence{block: 0xffffffff, count: 10}, 39, 9, 3},
+
+		{&sequence{block: 0xffffffff, count: 10, next: &sequence{block: 0xcc000000, count: 10}}, 79, 9, 3},
+		{&sequence{block: 0xffffffff, count: 10, next: &sequence{block: 0xcc000000, count: 10}}, 80, 0, invalidPos},
+	}
+
+	for n, i := range input {
+		_, _, precBlocks, inBlockBytePos := findSequence(i.head, i.bytePos)
+		if precBlocks != i.precBlocks || inBlockBytePos != i.inBlockBytePos {
+			t.Fatalf("Error in (%d) findSequence(). Expected (%d, %d). Got (%d, %d)", n, i.precBlocks, i.inBlockBytePos, precBlocks, inBlockBytePos)
+		}
+	}
+}
+
+func TestCheckIfAvailable(t *testing.T) {
+	input := []struct {
+		head    *sequence
+		ordinal uint64
+		bytePos uint64
+		bitPos  uint64
+	}{
+		{&sequence{block: 0xffffffff, count: 0}, 0, invalidPos, invalidPos},
+		{&sequence{block: 0xffffffff, count: 0}, 31, invalidPos, invalidPos},
+		{&sequence{block: 0xffffffff, count: 0}, 100, invalidPos, invalidPos},
+
+		{&sequence{block: 0x0, count: 1}, 0, 0, 0},
+		{&sequence{block: 0x0, count: 1}, 1, 0, 1},
+		{&sequence{block: 0x0, count: 1}, 31, 3, 7},
+		{&sequence{block: 0x0, count: 1}, 60, invalidPos, invalidPos},
+
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0x800000ff, count: 1}}, 31, invalidPos, invalidPos},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0x800000ff, count: 1}}, 32, invalidPos, invalidPos},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0x800000ff, count: 1}}, 33, 4, 1},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xC00000ff, count: 1}}, 33, invalidPos, invalidPos},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xC00000ff, count: 1}}, 34, 4, 2},
+
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xC00000ff, count: 1, next: &sequence{block: 0x0, count: 1}}}, 55, 6, 7},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xC00000ff, count: 1, next: &sequence{block: 0x0, count: 1}}}, 56, invalidPos, invalidPos},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xC00000ff, count: 1, next: &sequence{block: 0x0, count: 1}}}, 63, invalidPos, invalidPos},
+
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xC00000ff, count: 1, next: &sequence{block: 0x0, count: 1}}}, 64, 8, 0},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xC00000ff, count: 1, next: &sequence{block: 0x0, count: 1}}}, 95, 11, 7},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xC00000ff, count: 1, next: &sequence{block: 0x0, count: 1}}}, 96, invalidPos, invalidPos},
+	}
+
+	for n, i := range input {
+		bytePos, bitPos, err := checkIfAvailable(i.head, i.ordinal)
+		if bytePos != i.bytePos || bitPos != i.bitPos {
+			t.Fatalf("Error in (%d) checkIfAvailable(ord:%d). Expected (%d, %d). Got (%d, %d). err: %v", n, i.ordinal, i.bytePos, i.bitPos, bytePos, bitPos, err)
+		}
+	}
+}
+
+func TestMergeSequences(t *testing.T) {
+	input := []struct {
+		original *sequence
+		merged   *sequence
+	}{
+		{&sequence{block: 0xFE000000, count: 8, next: &sequence{block: 0xFE000000, count: 2}}, &sequence{block: 0xFE000000, count: 10}},
+		{&sequence{block: 0xFFFFFFFF, count: 8, next: &sequence{block: 0xFFFFFFFF, count: 1}}, &sequence{block: 0xFFFFFFFF, count: 9}},
+		{&sequence{block: 0xFFFFFFFF, count: 1, next: &sequence{block: 0xFFFFFFFF, count: 8}}, &sequence{block: 0xFFFFFFFF, count: 9}},
+
+		{&sequence{block: 0xFFFFFFF0, count: 8, next: &sequence{block: 0xFFFFFFF0, count: 1}}, &sequence{block: 0xFFFFFFF0, count: 9}},
+		{&sequence{block: 0xFFFFFFF0, count: 1, next: &sequence{block: 0xFFFFFFF0, count: 8}}, &sequence{block: 0xFFFFFFF0, count: 9}},
+
+		{&sequence{block: 0xFE, count: 8, next: &sequence{block: 0xFE, count: 1, next: &sequence{block: 0xFE, count: 5}}}, &sequence{block: 0xFE, count: 14}},
+		{&sequence{block: 0xFE, count: 8, next: &sequence{block: 0xFE, count: 1, next: &sequence{block: 0xFE, count: 5, next: &sequence{block: 0xFF, count: 1}}}},
+			&sequence{block: 0xFE, count: 14, next: &sequence{block: 0xFF, count: 1}}},
+
+		// No merge
+		{&sequence{block: 0xFE, count: 8, next: &sequence{block: 0xF8, count: 1, next: &sequence{block: 0xFE, count: 5}}},
+			&sequence{block: 0xFE, count: 8, next: &sequence{block: 0xF8, count: 1, next: &sequence{block: 0xFE, count: 5}}}},
+
+		// No merge from head: // Merge function tries to merge from passed head. If it can't merge with next, it does not reattempt with next as head
+		{&sequence{block: 0xFE, count: 8, next: &sequence{block: 0xFF, count: 1, next: &sequence{block: 0xFF, count: 5}}},
+			&sequence{block: 0xFE, count: 8, next: &sequence{block: 0xFF, count: 6}}},
+	}
+
+	for n, i := range input {
+		mergeSequences(i.original)
+		for !i.merged.equal(i.original) {
+			t.Fatalf("Error in (%d) mergeSequences().\nExp: %s\nGot: %s,", n, i.merged.toString(), i.original.toString())
+		}
+	}
+}
+
+func TestPushReservation(t *testing.T) {
+	input := []struct {
+		mask    *sequence
+		bytePos uint64
+		bitPos  uint64
+		newMask *sequence
+	}{
+		// Create first sequence and fill in 8 addresses starting from address 0
+		{&sequence{block: 0x0, count: 8, next: nil}, 0, 0, &sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x0, count: 7, next: nil}}},
+		{&sequence{block: 0x80000000, count: 8}, 0, 1, &sequence{block: 0xC0000000, count: 1, next: &sequence{block: 0x80000000, count: 7, next: nil}}},
+		{&sequence{block: 0xC0000000, count: 8}, 0, 2, &sequence{block: 0xE0000000, count: 1, next: &sequence{block: 0xC0000000, count: 7, next: nil}}},
+		{&sequence{block: 0xE0000000, count: 8}, 0, 3, &sequence{block: 0xF0000000, count: 1, next: &sequence{block: 0xE0000000, count: 7, next: nil}}},
+		{&sequence{block: 0xF0000000, count: 8}, 0, 4, &sequence{block: 0xF8000000, count: 1, next: &sequence{block: 0xF0000000, count: 7, next: nil}}},
+		{&sequence{block: 0xF8000000, count: 8}, 0, 5, &sequence{block: 0xFC000000, count: 1, next: &sequence{block: 0xF8000000, count: 7, next: nil}}},
+		{&sequence{block: 0xFC000000, count: 8}, 0, 6, &sequence{block: 0xFE000000, count: 1, next: &sequence{block: 0xFC000000, count: 7, next: nil}}},
+		{&sequence{block: 0xFE000000, count: 8}, 0, 7, &sequence{block: 0xFF000000, count: 1, next: &sequence{block: 0xFE000000, count: 7, next: nil}}},
+
+		{&sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x0, count: 7}}, 0, 1, &sequence{block: 0xC0000000, count: 1, next: &sequence{block: 0x0, count: 7, next: nil}}},
+
+		// Create second sequence and fill in 8 addresses starting from address 32
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0x00000000, count: 1, next: &sequence{block: 0xffffffff, count: 6, next: nil}}}, 4, 0,
+			&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0x80000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0x80000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 4, 1,
+			&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xC0000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xC0000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 4, 2,
+			&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xE0000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xE0000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 4, 3,
+			&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xF0000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xF0000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 4, 4,
+			&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xF8000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xF8000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 4, 5,
+			&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFC000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFC000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 4, 6,
+			&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFE000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFE000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 4, 7,
+			&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFF000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}},
+		// fill in 8 addresses starting from address 40
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFF000000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 5, 0,
+			&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFF800000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFF800000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 5, 1,
+			&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFC00000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFC00000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 5, 2,
+			&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFE00000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFE00000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 5, 3,
+			&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFF00000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFF00000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 5, 4,
+			&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFF80000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFF80000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 5, 5,
+			&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFFC0000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFFC0000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 5, 6,
+			&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFFE0000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFFE0000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}, 5, 7,
+			&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xFFFF0000, count: 1, next: &sequence{block: 0xffffffff, count: 6}}}},
+
+		// Insert new sequence
+		{&sequence{block: 0xffffffff, count: 2, next: &sequence{block: 0x0, count: 6}}, 8, 0,
+			&sequence{block: 0xffffffff, count: 2, next: &sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x0, count: 5}}}},
+		{&sequence{block: 0xffffffff, count: 2, next: &sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x0, count: 5}}}, 8, 1,
+			&sequence{block: 0xffffffff, count: 2, next: &sequence{block: 0xC0000000, count: 1, next: &sequence{block: 0x0, count: 5}}}},
+
+		// Merge affected with next
+		{&sequence{block: 0xffffffff, count: 7, next: &sequence{block: 0xfffffffe, count: 2, next: &sequence{block: 0xffffffff, count: 1}}}, 31, 7,
+			&sequence{block: 0xffffffff, count: 8, next: &sequence{block: 0xfffffffe, count: 1, next: &sequence{block: 0xffffffff, count: 1}}}},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xfffffffc, count: 1, next: &sequence{block: 0xfffffffe, count: 6}}}, 7, 6,
+			&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xfffffffe, count: 7}}},
+
+		// Merge affected with next and next.next
+		{&sequence{block: 0xffffffff, count: 7, next: &sequence{block: 0xfffffffe, count: 1, next: &sequence{block: 0xffffffff, count: 1}}}, 31, 7,
+			&sequence{block: 0xffffffff, count: 9}},
+		{&sequence{block: 0xffffffff, count: 7, next: &sequence{block: 0xfffffffe, count: 1}}, 31, 7,
+			&sequence{block: 0xffffffff, count: 8}},
+
+		// Merge affected with previous and next
+		{&sequence{block: 0xffffffff, count: 7, next: &sequence{block: 0xfffffffe, count: 1, next: &sequence{block: 0xffffffff, count: 1}}}, 31, 7,
+			&sequence{block: 0xffffffff, count: 9}},
+
+		// Redundant push: No change
+		{&sequence{block: 0xffff0000, count: 1}, 0, 0, &sequence{block: 0xffff0000, count: 1}},
+		{&sequence{block: 0xffff0000, count: 7}, 25, 7, &sequence{block: 0xffff0000, count: 7}},
+		{&sequence{block: 0xffffffff, count: 7, next: &sequence{block: 0xfffffffe, count: 1, next: &sequence{block: 0xffffffff, count: 1}}}, 7, 7,
+			&sequence{block: 0xffffffff, count: 7, next: &sequence{block: 0xfffffffe, count: 1, next: &sequence{block: 0xffffffff, count: 1}}}},
+
+		// Set last bit
+		{&sequence{block: 0x0, count: 8}, 31, 7, &sequence{block: 0x0, count: 7, next: &sequence{block: 0x1, count: 1}}},
+
+		// Set bit in a middle sequence in the first block, first bit
+		{&sequence{block: 0x40000000, count: 1, next: &sequence{block: 0x0, count: 6, next: &sequence{block: 0x1, count: 1}}}, 4, 0,
+			&sequence{block: 0x40000000, count: 1, next: &sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x0, count: 5,
+				next: &sequence{block: 0x1, count: 1}}}}},
+
+		// Set bit in a middle sequence in the first block, first bit (merge involved)
+		{&sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x0, count: 6, next: &sequence{block: 0x1, count: 1}}}, 4, 0,
+			&sequence{block: 0x80000000, count: 2, next: &sequence{block: 0x0, count: 5, next: &sequence{block: 0x1, count: 1}}}},
+
+		// Set bit in a middle sequence in the first block, last bit
+		{&sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x0, count: 6, next: &sequence{block: 0x1, count: 1}}}, 4, 31,
+			&sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x1, count: 1, next: &sequence{block: 0x0, count: 5,
+				next: &sequence{block: 0x1, count: 1}}}}},
+
+		// Set bit in a middle sequence in the first block, middle bit
+		{&sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x0, count: 6, next: &sequence{block: 0x1, count: 1}}}, 4, 16,
+			&sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x8000, count: 1, next: &sequence{block: 0x0, count: 5,
+				next: &sequence{block: 0x1, count: 1}}}}},
+
+		// Set bit in a middle sequence in a middle block, first bit
+		{&sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x0, count: 6, next: &sequence{block: 0x1, count: 1}}}, 16, 0,
+			&sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x0, count: 3, next: &sequence{block: 0x80000000, count: 1,
+				next: &sequence{block: 0x0, count: 2, next: &sequence{block: 0x1, count: 1}}}}}},
+
+		// Set bit in a middle sequence in a middle block, last bit
+		{&sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x0, count: 6, next: &sequence{block: 0x1, count: 1}}}, 16, 31,
+			&sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x0, count: 3, next: &sequence{block: 0x1, count: 1,
+				next: &sequence{block: 0x0, count: 2, next: &sequence{block: 0x1, count: 1}}}}}},
+
+		// Set bit in a middle sequence in a middle block, middle bit
+		{&sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x0, count: 6, next: &sequence{block: 0x1, count: 1}}}, 16, 15,
+			&sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x0, count: 3, next: &sequence{block: 0x10000, count: 1,
+				next: &sequence{block: 0x0, count: 2, next: &sequence{block: 0x1, count: 1}}}}}},
+
+		// Set bit in a middle sequence in the last block, first bit
+		{&sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x0, count: 6, next: &sequence{block: 0x1, count: 1}}}, 24, 0,
+			&sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x0, count: 5, next: &sequence{block: 0x80000000, count: 1,
+				next: &sequence{block: 0x1, count: 1}}}}},
+
+		// Set bit in a middle sequence in the last block, last bit
+		{&sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x0, count: 6, next: &sequence{block: 0x4, count: 1}}}, 24, 31,
+			&sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x0, count: 5, next: &sequence{block: 0x1, count: 1,
+				next: &sequence{block: 0x4, count: 1}}}}},
+
+		// Set bit in a middle sequence in the last block, last bit (merge involved)
+		{&sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x0, count: 6, next: &sequence{block: 0x1, count: 1}}}, 24, 31,
+			&sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x0, count: 5, next: &sequence{block: 0x1, count: 2}}}},
+
+		// Set bit in a middle sequence in the last block, middle bit
+		{&sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x0, count: 6, next: &sequence{block: 0x1, count: 1}}}, 24, 16,
+			&sequence{block: 0x80000000, count: 1, next: &sequence{block: 0x0, count: 5, next: &sequence{block: 0x8000, count: 1,
+				next: &sequence{block: 0x1, count: 1}}}}},
+	}
+
+	for n, i := range input {
+		mask := pushReservation(i.bytePos, i.bitPos, i.mask, false)
+		if !mask.equal(i.newMask) {
+			t.Fatalf("Error in (%d) pushReservation():\n%s + (%d,%d):\nExp: %s\nGot: %s,",
+				n, i.mask.toString(), i.bytePos, i.bitPos, i.newMask.toString(), mask.toString())
+		}
+	}
+}
+
+func TestSerializeDeserialize(t *testing.T) {
+	s := getTestSequence()
+
+	data, err := s.toByteArray()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	r := &sequence{}
+	err = r.fromByteArray(data)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !s.equal(r) {
+		t.Fatalf("Sequences are different: \n%v\n%v", s, r)
+	}
+}
+
+func getTestSequence() *sequence {
+	// Returns a custom sequence of 1024 * 32 bits
+	return &sequence{
+		block: 0xFFFFFFFF,
+		count: 100,
+		next: &sequence{
+			block: 0xFFFFFFFE,
+			count: 1,
+			next: &sequence{
+				block: 0xFF000000,
+				count: 10,
+				next: &sequence{
+					block: 0xFFFFFFFF,
+					count: 50,
+					next: &sequence{
+						block: 0xFFFFFFFC,
+						count: 1,
+						next: &sequence{
+							block: 0xFF800000,
+							count: 1,
+							next: &sequence{
+								block: 0xFFFFFFFF,
+								count: 87,
+								next: &sequence{
+									block: 0x0,
+									count: 150,
+									next: &sequence{
+										block: 0xFFFFFFFF,
+										count: 200,
+										next: &sequence{
+											block: 0x0000FFFF,
+											count: 1,
+											next: &sequence{
+												block: 0x0,
+												count: 399,
+												next: &sequence{
+													block: 0xFFFFFFFF,
+													count: 23,
+													next: &sequence{
+														block: 0x1,
+														count: 1,
+													},
+												},
+											},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
+func TestSet(t *testing.T) {
+	hnd, err := NewHandle("", nil, "", 1024*32)
+	if err != nil {
+		t.Fatal(err)
+	}
+	hnd.head = getTestSequence()
+
+	firstAv := uint64(32*100 + 31)
+	last := uint64(1024*32 - 1)
+
+	if hnd.IsSet(100000) {
+		t.Fatal("IsSet() returned wrong result")
+	}
+
+	if !hnd.IsSet(0) {
+		t.Fatal("IsSet() returned wrong result")
+	}
+
+	if hnd.IsSet(firstAv) {
+		t.Fatal("IsSet() returned wrong result")
+	}
+
+	if !hnd.IsSet(last) {
+		t.Fatal("IsSet() returned wrong result")
+	}
+
+	if err := hnd.Set(0); err == nil {
+		t.Fatal("Expected failure, but succeeded")
+	}
+
+	os, err := hnd.SetAny(false)
+	if err != nil {
+		t.Fatalf("Unexpected failure: %v", err)
+	}
+	if os != firstAv {
+		t.Fatalf("SetAny returned unexpected ordinal. Expected %d. Got %d.", firstAv, os)
+	}
+	if !hnd.IsSet(firstAv) {
+		t.Fatal("IsSet() returned wrong result")
+	}
+
+	if err := hnd.Unset(firstAv); err != nil {
+		t.Fatalf("Unexpected failure: %v", err)
+	}
+
+	if hnd.IsSet(firstAv) {
+		t.Fatal("IsSet() returned wrong result")
+	}
+
+	if err := hnd.Set(firstAv); err != nil {
+		t.Fatalf("Unexpected failure: %v", err)
+	}
+
+	if err := hnd.Set(last); err == nil {
+		t.Fatal("Expected failure, but succeeded")
+	}
+}
+
+func TestSetUnset(t *testing.T) {
+	numBits := uint64(32 * blockLen)
+	hnd, err := NewHandle("", nil, "", numBits)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := hnd.Set(uint64(32 * blockLen)); err == nil {
+		t.Fatal("Expected failure, but succeeded")
+	}
+	if err := hnd.Unset(uint64(32 * blockLen)); err == nil {
+		t.Fatal("Expected failure, but succeeded")
+	}
+
+	// set and unset all one by one
+	for hnd.Unselected() > 0 {
+		if _, err := hnd.SetAny(false); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if _, err := hnd.SetAny(false); err != ErrNoBitAvailable {
+		t.Fatal("Expected error. Got success")
+	}
+	if _, err := hnd.SetAnyInRange(10, 20, false); err != ErrNoBitAvailable {
+		t.Fatal("Expected error. Got success")
+	}
+	if err := hnd.Set(50); err != ErrBitAllocated {
+		t.Fatalf("Expected error. Got %v: %s", err, hnd)
+	}
+	i := uint64(0)
+	for hnd.Unselected() < numBits {
+		if err := hnd.Unset(i); err != nil {
+			t.Fatal(err)
+		}
+		i++
+	}
+}
+
+func TestOffsetSetUnset(t *testing.T) {
+	numBits := uint64(32 * blockLen)
+	var o uint64
+	hnd, err := NewHandle("", nil, "", numBits)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// set and unset all one by one
+	for hnd.Unselected() > 0 {
+		if _, err := hnd.SetAny(false); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	if _, err := hnd.SetAny(false); err != ErrNoBitAvailable {
+		t.Fatal("Expected error. Got success")
+	}
+
+	if _, err := hnd.SetAnyInRange(10, 20, false); err != ErrNoBitAvailable {
+		t.Fatal("Expected error. Got success")
+	}
+
+	if err := hnd.Unset(288); err != nil {
+		t.Fatal(err)
+	}
+
+	//At this point sequence is (0xffffffff, 9)->(0x7fffffff, 1)->(0xffffffff, 22)->end
+	if o, err = hnd.SetAnyInRange(32, 500, false); err != nil {
+		t.Fatal(err)
+	}
+
+	if o != 288 {
+		t.Fatalf("Expected ordinal not received, Received:%d", o)
+	}
+}
+
+func TestSetInRange(t *testing.T) {
+	numBits := uint64(1024 * blockLen)
+	hnd, err := NewHandle("", nil, "", numBits)
+	if err != nil {
+		t.Fatal(err)
+	}
+	hnd.head = getTestSequence()
+
+	firstAv := uint64(100*blockLen + blockLen - 1)
+
+	if o, err := hnd.SetAnyInRange(4, 3, false); err == nil {
+		t.Fatalf("Expected failure. Got success with ordinal:%d", o)
+	}
+
+	if o, err := hnd.SetAnyInRange(0, numBits, false); err == nil {
+		t.Fatalf("Expected failure. Got success with ordinal:%d", o)
+	}
+
+	o, err := hnd.SetAnyInRange(100*uint64(blockLen), 101*uint64(blockLen), false)
+	if err != nil {
+		t.Fatalf("Unexpected failure: (%d, %v)", o, err)
+	}
+	if o != firstAv {
+		t.Fatalf("Unexpected ordinal: %d", o)
+	}
+
+	if o, err := hnd.SetAnyInRange(0, uint64(blockLen), false); err == nil {
+		t.Fatalf("Expected failure. Got success with ordinal:%d", o)
+	}
+
+	if o, err := hnd.SetAnyInRange(0, firstAv-1, false); err == nil {
+		t.Fatalf("Expected failure. Got success with ordinal:%d", o)
+	}
+
+	if o, err := hnd.SetAnyInRange(111*uint64(blockLen), 161*uint64(blockLen), false); err == nil {
+		t.Fatalf("Expected failure. Got success with ordinal:%d", o)
+	}
+
+	o, err = hnd.SetAnyInRange(161*uint64(blockLen), 162*uint64(blockLen), false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if o != 161*uint64(blockLen)+30 {
+		t.Fatalf("Unexpected ordinal: %d", o)
+	}
+
+	o, err = hnd.SetAnyInRange(161*uint64(blockLen), 162*uint64(blockLen), false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if o != 161*uint64(blockLen)+31 {
+		t.Fatalf("Unexpected ordinal: %d", o)
+	}
+
+	o, err = hnd.SetAnyInRange(161*uint64(blockLen), 162*uint64(blockLen), false)
+	if err == nil {
+		t.Fatalf("Expected failure. Got success with ordinal:%d", o)
+	}
+
+	if _, err := hnd.SetAnyInRange(0, numBits-1, false); err != nil {
+		t.Fatalf("Unexpected failure: %v", err)
+	}
+
+	// set one bit using the set range with 1 bit size range
+	if _, err := hnd.SetAnyInRange(uint64(163*blockLen-1), uint64(163*blockLen-1), false); err != nil {
+		t.Fatal(err)
+	}
+
+	// create a non multiple of 32 mask
+	hnd, err = NewHandle("", nil, "", 30)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// set all bit in the first range
+	for hnd.Unselected() > 22 {
+		if o, err := hnd.SetAnyInRange(0, 7, false); err != nil {
+			t.Fatalf("Unexpected failure: (%d, %v)", o, err)
+		}
+	}
+	// try one more set, which should fail
+	o, err = hnd.SetAnyInRange(0, 7, false)
+	if err == nil {
+		t.Fatalf("Expected failure. Got success with ordinal:%d", o)
+	}
+	if err != ErrNoBitAvailable {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+
+	// set all bit in a second range
+	for hnd.Unselected() > 14 {
+		if o, err := hnd.SetAnyInRange(8, 15, false); err != nil {
+			t.Fatalf("Unexpected failure: (%d, %v)", o, err)
+		}
+	}
+
+	// try one more set, which should fail
+	o, err = hnd.SetAnyInRange(0, 15, false)
+	if err == nil {
+		t.Fatalf("Expected failure. Got success with ordinal:%d", o)
+	}
+	if err != ErrNoBitAvailable {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+
+	// set all bit in a range which includes the last bit
+	for hnd.Unselected() > 12 {
+		if o, err := hnd.SetAnyInRange(28, 29, false); err != nil {
+			t.Fatalf("Unexpected failure: (%d, %v)", o, err)
+		}
+	}
+	o, err = hnd.SetAnyInRange(28, 29, false)
+	if err == nil {
+		t.Fatalf("Expected failure. Got success with ordinal:%d", o)
+	}
+	if err != ErrNoBitAvailable {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+}
+
+// This one tests an allocation pattern which unveiled an issue in pushReservation
+// Specifically a failure in detecting when we are in the (B) case (the bit to set
+// belongs to the last block of the current sequence). Because of a bug, code
+// was assuming the bit belonged to a block in the middle of the current sequence.
+// Which in turn caused an incorrect allocation when requesting a bit which is not
+// in the first or last sequence block.
+func TestSetAnyInRange(t *testing.T) {
+	numBits := uint64(8 * blockLen)
+	hnd, err := NewHandle("", nil, "", numBits)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := hnd.Set(0); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := hnd.Set(255); err != nil {
+		t.Fatal(err)
+	}
+
+	o, err := hnd.SetAnyInRange(128, 255, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if o != 128 {
+		t.Fatalf("Unexpected ordinal: %d", o)
+	}
+
+	o, err = hnd.SetAnyInRange(128, 255, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if o != 129 {
+		t.Fatalf("Unexpected ordinal: %d", o)
+	}
+
+	o, err = hnd.SetAnyInRange(246, 255, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if o != 246 {
+		t.Fatalf("Unexpected ordinal: %d", o)
+	}
+
+	o, err = hnd.SetAnyInRange(246, 255, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if o != 247 {
+		t.Fatalf("Unexpected ordinal: %d", o)
+	}
+}
+
+func TestMethods(t *testing.T) {
+	numBits := uint64(256 * blockLen)
+	hnd, err := NewHandle("path/to/data", nil, "sequence1", numBits)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if hnd.Bits() != numBits {
+		t.Fatalf("Unexpected bit number: %d", hnd.Bits())
+	}
+
+	if hnd.Unselected() != numBits {
+		t.Fatalf("Unexpected bit number: %d", hnd.Unselected())
+	}
+
+	exp := "(0x0, 256)->end"
+	if hnd.head.toString() != exp {
+		t.Fatalf("Unexpected sequence string: %s", hnd.head.toString())
+	}
+
+	for i := 0; i < 192; i++ {
+		_, err := hnd.SetAny(false)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	exp = "(0xffffffff, 6)->(0x0, 250)->end"
+	if hnd.head.toString() != exp {
+		t.Fatalf("Unexpected sequence string: %s", hnd.head.toString())
+	}
+}
+
+func TestRandomAllocateDeallocate(t *testing.T) {
+	ds, err := randomLocalStore()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	numBits := int(16 * blockLen)
+	hnd, err := NewHandle("bitseq-test/data/", ds, "test1", uint64(numBits))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	seed := time.Now().Unix()
+	rand.Seed(seed)
+
+	// Allocate all bits using a random pattern
+	pattern := rand.Perm(numBits)
+	for _, bit := range pattern {
+		err := hnd.Set(uint64(bit))
+		if err != nil {
+			t.Fatalf("Unexpected failure on allocation of %d: %v.\nSeed: %d.\n%s", bit, err, seed, hnd)
+		}
+	}
+	if hnd.Unselected() != 0 {
+		t.Fatalf("Expected full sequence. Instead found %d free bits. Seed: %d.\n%s", hnd.unselected, seed, hnd)
+	}
+	if hnd.head.toString() != "(0xffffffff, 16)->end" {
+		t.Fatalf("Unexpected db: %s", hnd.head.toString())
+	}
+
+	// Deallocate all bits using a random pattern
+	pattern = rand.Perm(numBits)
+	for _, bit := range pattern {
+		err := hnd.Unset(uint64(bit))
+		if err != nil {
+			t.Fatalf("Unexpected failure on deallocation of %d: %v.\nSeed: %d.\n%s", bit, err, seed, hnd)
+		}
+	}
+	if hnd.Unselected() != uint64(numBits) {
+		t.Fatalf("Expected full sequence. Instead found %d free bits. Seed: %d.\n%s", hnd.unselected, seed, hnd)
+	}
+	if hnd.head.toString() != "(0x0, 16)->end" {
+		t.Fatalf("Unexpected db: %s", hnd.head.toString())
+	}
+
+	err = hnd.Destroy()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestAllocateRandomDeallocate(t *testing.T) {
+	ds, err := randomLocalStore()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	numBlocks := uint32(8)
+	numBits := int(numBlocks * blockLen)
+	hnd, err := NewHandle(filepath.Join("bitseq", "test", "data"), ds, "test1", uint64(numBits))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expected := &sequence{block: 0xffffffff, count: uint64(numBlocks / 2), next: &sequence{block: 0x0, count: uint64(numBlocks / 2)}}
+
+	// Allocate first half of the bits
+	for i := 0; i < numBits/2; i++ {
+		_, err := hnd.SetAny(false)
+		if err != nil {
+			t.Fatalf("Unexpected failure on allocation %d: %v\n%s", i, err, hnd)
+		}
+	}
+	if hnd.Unselected() != uint64(numBits/2) {
+		t.Fatalf("Expected full sequence. Instead found %d free bits. %s", hnd.unselected, hnd)
+	}
+	if !hnd.head.equal(expected) {
+		t.Fatalf("Unexpected sequence. Got:\n%s", hnd)
+	}
+
+	seed := time.Now().Unix()
+	rand.Seed(seed)
+
+	// Deallocate half of the allocated bits following a random pattern
+	pattern := rand.Perm(numBits / 2)
+	for i := 0; i < numBits/4; i++ {
+		bit := pattern[i]
+		err := hnd.Unset(uint64(bit))
+		if err != nil {
+			t.Fatalf("Unexpected failure on deallocation of %d: %v.\nSeed: %d.\n%s", bit, err, seed, hnd)
+		}
+	}
+	if hnd.Unselected() != uint64(3*numBits/4) {
+		t.Fatalf("Expected full sequence. Instead found %d free bits.\nSeed: %d.\n%s", hnd.unselected, seed, hnd)
+	}
+
+	// Request a quarter of bits
+	for i := 0; i < numBits/4; i++ {
+		_, err := hnd.SetAny(false)
+		if err != nil {
+			t.Fatalf("Unexpected failure on allocation %d: %v\nSeed: %d\n%s", i, err, seed, hnd)
+		}
+	}
+	if hnd.Unselected() != uint64(numBits/2) {
+		t.Fatalf("Expected half sequence. Instead found %d free bits.\nSeed: %d\n%s", hnd.unselected, seed, hnd)
+	}
+	if !hnd.head.equal(expected) {
+		t.Fatalf("Unexpected sequence. Got:\n%s", hnd)
+	}
+
+	err = hnd.Destroy()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestAllocateRandomDeallocateSerialize(t *testing.T) {
+	ds, err := randomLocalStore()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	numBlocks := uint32(8)
+	numBits := int(numBlocks * blockLen)
+	hnd, err := NewHandle("bitseq-test/data/", ds, "test1", uint64(numBits))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expected := &sequence{block: 0xffffffff, count: uint64(numBlocks / 2), next: &sequence{block: 0x0, count: uint64(numBlocks / 2)}}
+
+	// Allocate first half of the bits
+	for i := 0; i < numBits/2; i++ {
+		_, err := hnd.SetAny(true)
+		if err != nil {
+			t.Fatalf("Unexpected failure on allocation %d: %v\n%s", i, err, hnd)
+		}
+	}
+
+	if hnd.Unselected() != uint64(numBits/2) {
+		t.Fatalf("Expected full sequence. Instead found %d free bits. %s", hnd.unselected, hnd)
+	}
+	if !hnd.head.equal(expected) {
+		t.Fatalf("Unexpected sequence. Got:\n%s", hnd)
+	}
+
+	seed := time.Now().Unix()
+	rand.Seed(seed)
+
+	// Deallocate half of the allocated bits following a random pattern
+	pattern := rand.Perm(numBits / 2)
+	for i := 0; i < numBits/4; i++ {
+		bit := pattern[i]
+		err := hnd.Unset(uint64(bit))
+		if err != nil {
+			t.Fatalf("Unexpected failure on deallocation of %d: %v.\nSeed: %d.\n%s", bit, err, seed, hnd)
+		}
+	}
+	if hnd.Unselected() != uint64(3*numBits/4) {
+		t.Fatalf("Expected full sequence. Instead found %d free bits.\nSeed: %d.\n%s", hnd.unselected, seed, hnd)
+	}
+
+	// Request a quarter of bits
+	for i := 0; i < numBits/4; i++ {
+		_, err := hnd.SetAny(true)
+		if err != nil {
+			t.Fatalf("Unexpected failure on allocation %d: %v\nSeed: %d\n%s", i, err, seed, hnd)
+		}
+	}
+	if hnd.Unselected() != uint64(numBits/2) {
+		t.Fatalf("Expected half sequence. Instead found %d free bits.\nSeed: %d\n%s", hnd.unselected, seed, hnd)
+	}
+
+	err = hnd.Destroy()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestRetrieveFromStore(t *testing.T) {
+	ds, err := randomLocalStore()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	numBits := int(8 * blockLen)
+	hnd, err := NewHandle("bitseq-test/data/", ds, "test1", uint64(numBits))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Allocate first half of the bits
+	for i := 0; i < numBits/2; i++ {
+		_, err := hnd.SetAny(false)
+		if err != nil {
+			t.Fatalf("Unexpected failure on allocation %d: %v\n%s", i, err, hnd)
+		}
+	}
+	hnd0 := hnd.String()
+
+	// Retrieve same handle
+	hnd, err = NewHandle("bitseq-test/data/", ds, "test1", uint64(numBits))
+	if err != nil {
+		t.Fatal(err)
+	}
+	hnd1 := hnd.String()
+
+	if hnd1 != hnd0 {
+		t.Fatalf("%v\n%v", hnd0, hnd1)
+	}
+
+	err = hnd.Destroy()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestIsCorrupted(t *testing.T) {
+	ds, err := randomLocalStore()
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Negative test
+	hnd, err := NewHandle("bitseq-test/data/", ds, "test_corrupted", 1024)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if hnd.runConsistencyCheck() {
+		t.Fatalf("Unexpected corrupted for %s", hnd)
+	}
+
+	if err := hnd.CheckConsistency(); err != nil {
+		t.Fatal(err)
+	}
+
+	hnd.Set(0)
+	if hnd.runConsistencyCheck() {
+		t.Fatalf("Unexpected corrupted for %s", hnd)
+	}
+
+	hnd.Set(1023)
+	if hnd.runConsistencyCheck() {
+		t.Fatalf("Unexpected corrupted for %s", hnd)
+	}
+
+	if err := hnd.CheckConsistency(); err != nil {
+		t.Fatal(err)
+	}
+
+	// Try real corrupted ipam handles found in the local store files reported by three docker users,
+	// plus a generic ipam handle from docker 1.9.1. This last will fail as well, because of how the
+	// last node in the sequence is expressed (This is true for IPAM handle only, because of the broadcast
+	// address reservation: last bit). This will allow an application using bitseq that runs a consistency
+	// check to detect and replace the 1.9.0/1 old vulnerable handle with the new one.
+	input := []*Handle{
+		{
+			id:         "LocalDefault/172.17.0.0/16",
+			bits:       65536,
+			unselected: 65412,
+			head: &sequence{
+				block: 0xffffffff,
+				count: 3,
+				next: &sequence{
+					block: 0xffffffbf,
+					count: 0,
+					next: &sequence{
+						block: 0xfe98816e,
+						count: 1,
+						next: &sequence{
+							block: 0xffffffff,
+							count: 0,
+							next: &sequence{
+								block: 0xe3bc0000,
+								count: 1,
+								next: &sequence{
+									block: 0x0,
+									count: 2042,
+									next: &sequence{
+										block: 0x1, count: 1,
+										next: &sequence{
+											block: 0x0, count: 0,
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			id:         "LocalDefault/172.17.0.0/16",
+			bits:       65536,
+			unselected: 65319,
+			head: &sequence{
+				block: 0xffffffff,
+				count: 7,
+				next: &sequence{
+					block: 0xffffff7f,
+					count: 0,
+					next: &sequence{
+						block: 0xffffffff,
+						count: 0,
+						next: &sequence{
+							block: 0x2000000,
+							count: 1,
+							next: &sequence{
+								block: 0x0,
+								count: 2039,
+								next: &sequence{
+									block: 0x1,
+									count: 1,
+									next: &sequence{
+										block: 0x0,
+										count: 0,
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			id:         "LocalDefault/172.17.0.0/16",
+			bits:       65536,
+			unselected: 65456,
+			head: &sequence{
+				block: 0xffffffff, count: 2,
+				next: &sequence{
+					block: 0xfffbffff, count: 0,
+					next: &sequence{
+						block: 0xffd07000, count: 1,
+						next: &sequence{
+							block: 0x0, count: 333,
+							next: &sequence{
+								block: 0x40000000, count: 1,
+								next: &sequence{
+									block: 0x0, count: 1710,
+									next: &sequence{
+										block: 0x1, count: 1,
+										next: &sequence{
+											block: 0x0, count: 0,
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for idx, hnd := range input {
+		if !hnd.runConsistencyCheck() {
+			t.Fatalf("Expected corrupted for (%d): %s", idx, hnd)
+		}
+		if hnd.runConsistencyCheck() {
+			t.Fatalf("Sequence still marked corrupted (%d): %s", idx, hnd)
+		}
+	}
+}
+
+func testSetRollover(t *testing.T, serial bool) {
+	ds, err := randomLocalStore()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	numBlocks := uint32(8)
+	numBits := int(numBlocks * blockLen)
+	hnd, err := NewHandle("bitseq-test/data/", ds, "test1", uint64(numBits))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Allocate first half of the bits
+	for i := 0; i < numBits/2; i++ {
+		_, err := hnd.SetAny(serial)
+		if err != nil {
+			t.Fatalf("Unexpected failure on allocation %d: %v\n%s", i, err, hnd)
+		}
+	}
+
+	if hnd.Unselected() != uint64(numBits/2) {
+		t.Fatalf("Expected full sequence. Instead found %d free bits. %s", hnd.unselected, hnd)
+	}
+
+	seed := time.Now().Unix()
+	rand.Seed(seed)
+
+	// Deallocate half of the allocated bits following a random pattern
+	pattern := rand.Perm(numBits / 2)
+	for i := 0; i < numBits/4; i++ {
+		bit := pattern[i]
+		err := hnd.Unset(uint64(bit))
+		if err != nil {
+			t.Fatalf("Unexpected failure on deallocation of %d: %v.\nSeed: %d.\n%s", bit, err, seed, hnd)
+		}
+	}
+	if hnd.Unselected() != uint64(3*numBits/4) {
+		t.Fatalf("Unexpected free bits: found %d free bits.\nSeed: %d.\n%s", hnd.unselected, seed, hnd)
+	}
+
+	//request to allocate for remaining half of the bits
+	for i := 0; i < numBits/2; i++ {
+		_, err := hnd.SetAny(serial)
+		if err != nil {
+			t.Fatalf("Unexpected failure on allocation %d: %v\nSeed: %d\n%s", i, err, seed, hnd)
+		}
+	}
+
+	//At this point all the bits must be allocated except the randomly unallocated bits
+	//which were unallocated in the first half of the bit sequence
+	if hnd.Unselected() != uint64(numBits/4) {
+		t.Fatalf("Unexpected number of unselected bits %d, Expected %d", hnd.Unselected(), numBits/4)
+	}
+
+	for i := 0; i < numBits/4; i++ {
+		_, err := hnd.SetAny(serial)
+		if err != nil {
+			t.Fatalf("Unexpected failure on allocation %d: %v\nSeed: %d\n%s", i, err, seed, hnd)
+		}
+	}
+	//Now requesting to allocate the unallocated random bits (qurter of the number of bits) should
+	//leave no more bits that can be allocated.
+	if hnd.Unselected() != 0 {
+		t.Fatalf("Unexpected number of unselected bits %d, Expected %d", hnd.Unselected(), 0)
+	}
+
+	err = hnd.Destroy()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestSetRollover(t *testing.T) {
+	testSetRollover(t, false)
+}
+
+func TestSetRolloverSerial(t *testing.T) {
+	testSetRollover(t, true)
+}
+
+func TestGetFirstAvailableFromCurrent(t *testing.T) {
+	input := []struct {
+		mask    *sequence
+		bytePos uint64
+		bitPos  uint64
+		start   uint64
+		curr    uint64
+		end     uint64
+	}{
+		{&sequence{block: 0xffffffff, count: 2048}, invalidPos, invalidPos, 0, 0, 65536},
+		{&sequence{block: 0x0, count: 8}, 0, 0, 0, 0, 256},
+		{&sequence{block: 0x80000000, count: 8}, 1, 0, 0, 8, 256},
+		{&sequence{block: 0xC0000000, count: 8}, 0, 2, 0, 2, 256},
+		{&sequence{block: 0xE0000000, count: 8}, 0, 3, 0, 0, 256},
+		{&sequence{block: 0xFFFB1FFF, count: 8}, 2, 0, 14, 0, 256},
+		{&sequence{block: 0xFFFFFFFE, count: 8}, 3, 7, 0, 0, 256},
+
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0x00000000, count: 1, next: &sequence{block: 0xffffffff, count: 14}}}, 4, 0, 0, 32, 512},
+		{&sequence{block: 0xfffeffff, count: 1, next: &sequence{block: 0xffffffff, count: 15}}, 1, 7, 0, 16, 512},
+		{&sequence{block: 0xfffeffff, count: 15, next: &sequence{block: 0xffffffff, count: 1}}, 5, 7, 0, 16, 512},
+		{&sequence{block: 0xfffeffff, count: 15, next: &sequence{block: 0xffffffff, count: 1}}, 9, 7, 0, 48, 512},
+		{&sequence{block: 0xffffffff, count: 2, next: &sequence{block: 0xffffffef, count: 14}}, 19, 3, 0, 124, 512},
+		{&sequence{block: 0xfffeffff, count: 15, next: &sequence{block: 0x0fffffff, count: 1}}, 60, 0, 0, 480, 512},
+		{&sequence{block: 0xffffffff, count: 1, next: &sequence{block: 0xfffeffff, count: 14, next: &sequence{block: 0xffffffff, count: 1}}}, 17, 7, 0, 124, 512},
+		{&sequence{block: 0xfffffffb, count: 1, next: &sequence{block: 0xffffffff, count: 14, next: &sequence{block: 0xffffffff, count: 1}}}, 3, 5, 0, 124, 512},
+		{&sequence{block: 0xfffffffb, count: 1, next: &sequence{block: 0xfffeffff, count: 14, next: &sequence{block: 0xffffffff, count: 1}}}, 13, 7, 0, 80, 512},
+	}
+
+	for n, i := range input {
+		bytePos, bitPos, _ := getAvailableFromCurrent(i.mask, i.start, i.curr, i.end)
+		if bytePos != i.bytePos || bitPos != i.bitPos {
+			t.Fatalf("Error in (%d) getFirstAvailable(). Expected (%d, %d). Got (%d, %d)", n, i.bytePos, i.bitPos, bytePos, bitPos)
+		}
+	}
+}
+
+func TestMarshalJSON(t *testing.T) {
+	const expectedID = "my-bitseq"
+	expected := []byte("hello libnetwork")
+	hnd, err := NewHandle("", nil, expectedID, uint64(len(expected)*8))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for i, c := range expected {
+		for j := 0; j < 8; j++ {
+			if c&(1<<j) == 0 {
+				continue
+			}
+			if err := hnd.Set(uint64(i*8 + j)); err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
+
+	hstr := hnd.String()
+	t.Log(hstr)
+	marshaled, err := hnd.MarshalJSON()
+	if err != nil {
+		t.Fatalf("MarshalJSON() err = %v", err)
+	}
+	t.Logf("%s", marshaled)
+
+	// Serializations of hnd as would be marshaled by versions of the code
+	// found in the wild. We need to support unmarshaling old versions to
+	// maintain backwards compatibility with sequences persisted on disk.
+	const (
+		goldenV0 = `{"id":"my-bitseq","sequence":"AAAAAAAAAIAAAAAAAAAAPRamNjYAAAAAAAAAAfYENpYAAAAAAAAAAUZ2pi4AAAAAAAAAAe72TtYAAAAAAAAAAQ=="}`
+	)
+
+	if string(marshaled) != goldenV0 {
+		t.Errorf("MarshalJSON() output differs from golden. Please add a new golden case to this test.")
+	}
+
+	for _, tt := range []struct {
+		name string
+		data []byte
+	}{
+		{name: "Live", data: marshaled},
+		{name: "Golden-v0", data: []byte(goldenV0)},
+	} {
+		tt := tt
+		t.Run("UnmarshalJSON="+tt.name, func(t *testing.T) {
+			hnd2, err := NewHandle("", nil, "", 0)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if err := hnd2.UnmarshalJSON(tt.data); err != nil {
+				t.Errorf("UnmarshalJSON() err = %v", err)
+			}
+
+			h2str := hnd2.String()
+			t.Log(h2str)
+			if hstr != h2str {
+				t.Errorf("Unmarshaled a different bitseq: want %q, got %q", hstr, h2str)
+			}
+		})
+	}
+}

libnetwork/bitmap/store.go

@@ -0,0 +1,130 @@
+package bitseq
+
+import (
+	"encoding/json"
+
+	"github.com/docker/docker/libnetwork/datastore"
+	"github.com/docker/docker/libnetwork/types"
+)
+
+// Key provides the Key to be used in KV Store
+func (h *Handle) Key() []string {
+	h.Lock()
+	defer h.Unlock()
+	return []string{h.app, h.id}
+}
+
+// KeyPrefix returns the immediate parent key that can be used for tree walk
+func (h *Handle) KeyPrefix() []string {
+	h.Lock()
+	defer h.Unlock()
+	return []string{h.app}
+}
+
+// Value marshals the data to be stored in the KV store
+func (h *Handle) Value() []byte {
+	b, err := json.Marshal(h)
+	if err != nil {
+		return nil
+	}
+	return b
+}
+
+// SetValue unmarshals the data from the KV store
+func (h *Handle) SetValue(value []byte) error {
+	return json.Unmarshal(value, h)
+}
+
+// Index returns the latest DB Index as seen by this object
+func (h *Handle) Index() uint64 {
+	h.Lock()
+	defer h.Unlock()
+	return h.dbIndex
+}
+
+// SetIndex method allows the datastore to store the latest DB Index into this object
+func (h *Handle) SetIndex(index uint64) {
+	h.Lock()
+	h.dbIndex = index
+	h.dbExists = true
+	h.Unlock()
+}
+
+// Exists method is true if this object has been stored in the DB.
+func (h *Handle) Exists() bool {
+	h.Lock()
+	defer h.Unlock()
+	return h.dbExists
+}
+
+// New method returns a handle based on the receiver handle
+func (h *Handle) New() datastore.KVObject {
+	h.Lock()
+	defer h.Unlock()
+
+	return &Handle{
+		app:   h.app,
+		store: h.store,
+	}
+}
+
+// CopyTo deep copies the handle into the passed destination object
+func (h *Handle) CopyTo(o datastore.KVObject) error {
+	h.Lock()
+	defer h.Unlock()
+
+	dstH := o.(*Handle)
+	if h == dstH {
+		return nil
+	}
+	dstH.Lock()
+	dstH.bits = h.bits
+	dstH.unselected = h.unselected
+	dstH.head = h.head.getCopy()
+	dstH.app = h.app
+	dstH.id = h.id
+	dstH.dbIndex = h.dbIndex
+	dstH.dbExists = h.dbExists
+	dstH.store = h.store
+	dstH.curr = h.curr
+	dstH.Unlock()
+
+	return nil
+}
+
+// Skip provides a way for a KV Object to avoid persisting it in the KV Store
+func (h *Handle) Skip() bool {
+	return false
+}
+
+// DataScope method returns the storage scope of the datastore
+func (h *Handle) DataScope() string {
+	h.Lock()
+	defer h.Unlock()
+
+	return h.store.Scope()
+}
+
+func (h *Handle) writeToStore() error {
+	h.Lock()
+	store := h.store
+	h.Unlock()
+	if store == nil {
+		return nil
+	}
+	err := store.PutObjectAtomic(h)
+	if err == datastore.ErrKeyModified {
+		return types.RetryErrorf("failed to perform atomic write (%v). Retry might fix the error", err)
+	}
+	return err
+}
+
+func (h *Handle) deleteFromStore() error {
+	h.Lock()
+	store := h.store
+	h.Unlock()
+	if store == nil {
+		return nil
+	}
+	return store.DeleteObjectAtomic(h)
+}