|
|
@@ -34,7 +34,7 @@ type match struct {
|
|
|
est int32
|
|
|
}
|
|
|
|
|
|
-const highScore = 25000
|
|
|
+const highScore = maxMatchLen * 8
|
|
|
|
|
|
// estBits will estimate output bits from predefined tables.
|
|
|
func (m *match) estBits(bitsPerByte int32) {
|
|
|
@@ -159,7 +159,6 @@ func (e *bestFastEncoder) Encode(blk *blockEnc, src []byte) {
|
|
|
|
|
|
// nextEmit is where in src the next emitLiteral should start from.
|
|
|
nextEmit := s
|
|
|
- cv := load6432(src, s)
|
|
|
|
|
|
// Relative offsets
|
|
|
offset1 := int32(blk.recentOffsets[0])
|
|
|
@@ -173,7 +172,6 @@ func (e *bestFastEncoder) Encode(blk *blockEnc, src []byte) {
|
|
|
blk.literals = append(blk.literals, src[nextEmit:until]...)
|
|
|
s.litLen = uint32(until - nextEmit)
|
|
|
}
|
|
|
- _ = addLiterals
|
|
|
|
|
|
if debugEncoder {
|
|
|
println("recent offsets:", blk.recentOffsets)
|
|
|
@@ -188,7 +186,9 @@ encodeLoop:
|
|
|
panic("offset0 was 0")
|
|
|
}
|
|
|
|
|
|
- const goodEnough = 100
|
|
|
+ const goodEnough = 250
|
|
|
+
|
|
|
+ cv := load6432(src, s)
|
|
|
|
|
|
nextHashL := hashLen(cv, bestLongTableBits, bestLongLen)
|
|
|
nextHashS := hashLen(cv, bestShortTableBits, bestShortLen)
|
|
|
@@ -201,11 +201,45 @@ encodeLoop:
|
|
|
return
|
|
|
}
|
|
|
if debugAsserts {
|
|
|
+ if offset <= 0 {
|
|
|
+ panic(offset)
|
|
|
+ }
|
|
|
if !bytes.Equal(src[s:s+4], src[offset:offset+4]) {
|
|
|
panic(fmt.Sprintf("first match mismatch: %v != %v, first: %08x", src[s:s+4], src[offset:offset+4], first))
|
|
|
}
|
|
|
}
|
|
|
- cand := match{offset: offset, s: s, length: 4 + e.matchlen(s+4, offset+4, src), rep: rep}
|
|
|
+ // Try to quick reject if we already have a long match.
|
|
|
+ if m.length > 16 {
|
|
|
+ left := len(src) - int(m.s+m.length)
|
|
|
+ // If we are too close to the end, keep as is.
|
|
|
+ if left <= 0 {
|
|
|
+ return
|
|
|
+ }
|
|
|
+ checkLen := m.length - (s - m.s) - 8
|
|
|
+ if left > 2 && checkLen > 4 {
|
|
|
+ // Check 4 bytes, 4 bytes from the end of the current match.
|
|
|
+ a := load3232(src, offset+checkLen)
|
|
|
+ b := load3232(src, s+checkLen)
|
|
|
+ if a != b {
|
|
|
+ return
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ l := 4 + e.matchlen(s+4, offset+4, src)
|
|
|
+ if rep < 0 {
|
|
|
+ // Extend candidate match backwards as far as possible.
|
|
|
+ tMin := s - e.maxMatchOff
|
|
|
+ if tMin < 0 {
|
|
|
+ tMin = 0
|
|
|
+ }
|
|
|
+ for offset > tMin && s > nextEmit && src[offset-1] == src[s-1] && l < maxMatchLength {
|
|
|
+ s--
|
|
|
+ offset--
|
|
|
+ l++
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ cand := match{offset: offset, s: s, length: l, rep: rep}
|
|
|
cand.estBits(bitsPerByte)
|
|
|
if m.est >= highScore || cand.est-m.est+(cand.s-m.s)*bitsPerByte>>10 < 0 {
|
|
|
*m = cand
|
|
|
@@ -219,17 +253,29 @@ encodeLoop:
|
|
|
improve(&best, candidateS.prev-e.cur, s, uint32(cv), -1)
|
|
|
|
|
|
if canRepeat && best.length < goodEnough {
|
|
|
- cv32 := uint32(cv >> 8)
|
|
|
- spp := s + 1
|
|
|
- improve(&best, spp-offset1, spp, cv32, 1)
|
|
|
- improve(&best, spp-offset2, spp, cv32, 2)
|
|
|
- improve(&best, spp-offset3, spp, cv32, 3)
|
|
|
- if best.length > 0 {
|
|
|
- cv32 = uint32(cv >> 24)
|
|
|
- spp += 2
|
|
|
+ if s == nextEmit {
|
|
|
+ // Check repeats straight after a match.
|
|
|
+ improve(&best, s-offset2, s, uint32(cv), 1|4)
|
|
|
+ improve(&best, s-offset3, s, uint32(cv), 2|4)
|
|
|
+ if offset1 > 1 {
|
|
|
+ improve(&best, s-(offset1-1), s, uint32(cv), 3|4)
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ // If either no match or a non-repeat match, check at + 1
|
|
|
+ if best.rep <= 0 {
|
|
|
+ cv32 := uint32(cv >> 8)
|
|
|
+ spp := s + 1
|
|
|
improve(&best, spp-offset1, spp, cv32, 1)
|
|
|
improve(&best, spp-offset2, spp, cv32, 2)
|
|
|
improve(&best, spp-offset3, spp, cv32, 3)
|
|
|
+ if best.rep < 0 {
|
|
|
+ cv32 = uint32(cv >> 24)
|
|
|
+ spp += 2
|
|
|
+ improve(&best, spp-offset1, spp, cv32, 1)
|
|
|
+ improve(&best, spp-offset2, spp, cv32, 2)
|
|
|
+ improve(&best, spp-offset3, spp, cv32, 3)
|
|
|
+ }
|
|
|
}
|
|
|
}
|
|
|
// Load next and check...
|
|
|
@@ -244,41 +290,44 @@ encodeLoop:
|
|
|
if s >= sLimit {
|
|
|
break encodeLoop
|
|
|
}
|
|
|
- cv = load6432(src, s)
|
|
|
continue
|
|
|
}
|
|
|
|
|
|
- s++
|
|
|
candidateS = e.table[hashLen(cv>>8, bestShortTableBits, bestShortLen)]
|
|
|
- cv = load6432(src, s)
|
|
|
- cv2 := load6432(src, s+1)
|
|
|
+ cv = load6432(src, s+1)
|
|
|
+ cv2 := load6432(src, s+2)
|
|
|
candidateL = e.longTable[hashLen(cv, bestLongTableBits, bestLongLen)]
|
|
|
candidateL2 := e.longTable[hashLen(cv2, bestLongTableBits, bestLongLen)]
|
|
|
|
|
|
// Short at s+1
|
|
|
- improve(&best, candidateS.offset-e.cur, s, uint32(cv), -1)
|
|
|
+ improve(&best, candidateS.offset-e.cur, s+1, uint32(cv), -1)
|
|
|
// Long at s+1, s+2
|
|
|
- improve(&best, candidateL.offset-e.cur, s, uint32(cv), -1)
|
|
|
- improve(&best, candidateL.prev-e.cur, s, uint32(cv), -1)
|
|
|
- improve(&best, candidateL2.offset-e.cur, s+1, uint32(cv2), -1)
|
|
|
- improve(&best, candidateL2.prev-e.cur, s+1, uint32(cv2), -1)
|
|
|
+ improve(&best, candidateL.offset-e.cur, s+1, uint32(cv), -1)
|
|
|
+ improve(&best, candidateL.prev-e.cur, s+1, uint32(cv), -1)
|
|
|
+ improve(&best, candidateL2.offset-e.cur, s+2, uint32(cv2), -1)
|
|
|
+ improve(&best, candidateL2.prev-e.cur, s+2, uint32(cv2), -1)
|
|
|
if false {
|
|
|
// Short at s+3.
|
|
|
// Too often worse...
|
|
|
- improve(&best, e.table[hashLen(cv2>>8, bestShortTableBits, bestShortLen)].offset-e.cur, s+2, uint32(cv2>>8), -1)
|
|
|
+ improve(&best, e.table[hashLen(cv2>>8, bestShortTableBits, bestShortLen)].offset-e.cur, s+3, uint32(cv2>>8), -1)
|
|
|
}
|
|
|
- // See if we can find a better match by checking where the current best ends.
|
|
|
- // Use that offset to see if we can find a better full match.
|
|
|
- if sAt := best.s + best.length; sAt < sLimit {
|
|
|
- nextHashL := hashLen(load6432(src, sAt), bestLongTableBits, bestLongLen)
|
|
|
- candidateEnd := e.longTable[nextHashL]
|
|
|
- // Start check at a fixed offset to allow for a few mismatches.
|
|
|
- // For this compression level 2 yields the best results.
|
|
|
- const skipBeginning = 2
|
|
|
- if pos := candidateEnd.offset - e.cur - best.length + skipBeginning; pos >= 0 {
|
|
|
- improve(&best, pos, best.s+skipBeginning, load3232(src, best.s+skipBeginning), -1)
|
|
|
- if pos := candidateEnd.prev - e.cur - best.length + skipBeginning; pos >= 0 {
|
|
|
- improve(&best, pos, best.s+skipBeginning, load3232(src, best.s+skipBeginning), -1)
|
|
|
+
|
|
|
+ // Start check at a fixed offset to allow for a few mismatches.
|
|
|
+ // For this compression level 2 yields the best results.
|
|
|
+ // We cannot do this if we have already indexed this position.
|
|
|
+ const skipBeginning = 2
|
|
|
+ if best.s > s-skipBeginning {
|
|
|
+ // See if we can find a better match by checking where the current best ends.
|
|
|
+ // Use that offset to see if we can find a better full match.
|
|
|
+ if sAt := best.s + best.length; sAt < sLimit {
|
|
|
+ nextHashL := hashLen(load6432(src, sAt), bestLongTableBits, bestLongLen)
|
|
|
+ candidateEnd := e.longTable[nextHashL]
|
|
|
+
|
|
|
+ if off := candidateEnd.offset - e.cur - best.length + skipBeginning; off >= 0 {
|
|
|
+ improve(&best, off, best.s+skipBeginning, load3232(src, best.s+skipBeginning), -1)
|
|
|
+ if off := candidateEnd.prev - e.cur - best.length + skipBeginning; off >= 0 {
|
|
|
+ improve(&best, off, best.s+skipBeginning, load3232(src, best.s+skipBeginning), -1)
|
|
|
+ }
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
@@ -292,51 +341,34 @@ encodeLoop:
|
|
|
|
|
|
// We have a match, we can store the forward value
|
|
|
if best.rep > 0 {
|
|
|
- s = best.s
|
|
|
var seq seq
|
|
|
seq.matchLen = uint32(best.length - zstdMinMatch)
|
|
|
-
|
|
|
- // We might be able to match backwards.
|
|
|
- // Extend as long as we can.
|
|
|
- start := best.s
|
|
|
- // We end the search early, so we don't risk 0 literals
|
|
|
- // and have to do special offset treatment.
|
|
|
- startLimit := nextEmit + 1
|
|
|
-
|
|
|
- tMin := s - e.maxMatchOff
|
|
|
- if tMin < 0 {
|
|
|
- tMin = 0
|
|
|
+ if debugAsserts && s <= nextEmit {
|
|
|
+ panic("s <= nextEmit")
|
|
|
}
|
|
|
- repIndex := best.offset
|
|
|
- for repIndex > tMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch-1 {
|
|
|
- repIndex--
|
|
|
- start--
|
|
|
- seq.matchLen++
|
|
|
- }
|
|
|
- addLiterals(&seq, start)
|
|
|
+ addLiterals(&seq, best.s)
|
|
|
|
|
|
- // rep 0
|
|
|
- seq.offset = uint32(best.rep)
|
|
|
+ // Repeat. If bit 4 is set, this is a non-lit repeat.
|
|
|
+ seq.offset = uint32(best.rep & 3)
|
|
|
if debugSequences {
|
|
|
println("repeat sequence", seq, "next s:", s)
|
|
|
}
|
|
|
blk.sequences = append(blk.sequences, seq)
|
|
|
|
|
|
- // Index match start+1 (long) -> s - 1
|
|
|
- index0 := s
|
|
|
+ // Index old s + 1 -> s - 1
|
|
|
+ index0 := s + 1
|
|
|
s = best.s + best.length
|
|
|
|
|
|
nextEmit = s
|
|
|
if s >= sLimit {
|
|
|
if debugEncoder {
|
|
|
println("repeat ended", s, best.length)
|
|
|
-
|
|
|
}
|
|
|
break encodeLoop
|
|
|
}
|
|
|
// Index skipped...
|
|
|
off := index0 + e.cur
|
|
|
- for index0 < s-1 {
|
|
|
+ for index0 < s {
|
|
|
cv0 := load6432(src, index0)
|
|
|
h0 := hashLen(cv0, bestLongTableBits, bestLongLen)
|
|
|
h1 := hashLen(cv0, bestShortTableBits, bestShortLen)
|
|
|
@@ -346,17 +378,19 @@ encodeLoop:
|
|
|
index0++
|
|
|
}
|
|
|
switch best.rep {
|
|
|
- case 2:
|
|
|
+ case 2, 4 | 1:
|
|
|
offset1, offset2 = offset2, offset1
|
|
|
- case 3:
|
|
|
+ case 3, 4 | 2:
|
|
|
offset1, offset2, offset3 = offset3, offset1, offset2
|
|
|
+ case 4 | 3:
|
|
|
+ offset1, offset2, offset3 = offset1-1, offset1, offset2
|
|
|
}
|
|
|
- cv = load6432(src, s)
|
|
|
continue
|
|
|
}
|
|
|
|
|
|
// A 4-byte match has been found. Update recent offsets.
|
|
|
// We'll later see if more than 4 bytes.
|
|
|
+ index0 := s + 1
|
|
|
s = best.s
|
|
|
t := best.offset
|
|
|
offset1, offset2, offset3 = s-t, offset1, offset2
|
|
|
@@ -369,22 +403,9 @@ encodeLoop:
|
|
|
panic("invalid offset")
|
|
|
}
|
|
|
|
|
|
- // Extend the n-byte match as long as possible.
|
|
|
- l := best.length
|
|
|
-
|
|
|
- // Extend backwards
|
|
|
- tMin := s - e.maxMatchOff
|
|
|
- if tMin < 0 {
|
|
|
- tMin = 0
|
|
|
- }
|
|
|
- for t > tMin && s > nextEmit && src[t-1] == src[s-1] && l < maxMatchLength {
|
|
|
- s--
|
|
|
- t--
|
|
|
- l++
|
|
|
- }
|
|
|
-
|
|
|
// Write our sequence
|
|
|
var seq seq
|
|
|
+ l := best.length
|
|
|
seq.litLen = uint32(s - nextEmit)
|
|
|
seq.matchLen = uint32(l - zstdMinMatch)
|
|
|
if seq.litLen > 0 {
|
|
|
@@ -401,10 +422,8 @@ encodeLoop:
|
|
|
break encodeLoop
|
|
|
}
|
|
|
|
|
|
- // Index match start+1 (long) -> s - 1
|
|
|
- index0 := s - l + 1
|
|
|
- // every entry
|
|
|
- for index0 < s-1 {
|
|
|
+ // Index old s + 1 -> s - 1
|
|
|
+ for index0 < s {
|
|
|
cv0 := load6432(src, index0)
|
|
|
h0 := hashLen(cv0, bestLongTableBits, bestLongLen)
|
|
|
h1 := hashLen(cv0, bestShortTableBits, bestShortLen)
|
|
|
@@ -413,50 +432,6 @@ encodeLoop:
|
|
|
e.table[h1] = prevEntry{offset: off, prev: e.table[h1].offset}
|
|
|
index0++
|
|
|
}
|
|
|
-
|
|
|
- cv = load6432(src, s)
|
|
|
- if !canRepeat {
|
|
|
- continue
|
|
|
- }
|
|
|
-
|
|
|
- // Check offset 2
|
|
|
- for {
|
|
|
- o2 := s - offset2
|
|
|
- if load3232(src, o2) != uint32(cv) {
|
|
|
- // Do regular search
|
|
|
- break
|
|
|
- }
|
|
|
-
|
|
|
- // Store this, since we have it.
|
|
|
- nextHashS := hashLen(cv, bestShortTableBits, bestShortLen)
|
|
|
- nextHashL := hashLen(cv, bestLongTableBits, bestLongLen)
|
|
|
-
|
|
|
- // We have at least 4 byte match.
|
|
|
- // No need to check backwards. We come straight from a match
|
|
|
- l := 4 + e.matchlen(s+4, o2+4, src)
|
|
|
-
|
|
|
- e.longTable[nextHashL] = prevEntry{offset: s + e.cur, prev: e.longTable[nextHashL].offset}
|
|
|
- e.table[nextHashS] = prevEntry{offset: s + e.cur, prev: e.table[nextHashS].offset}
|
|
|
- seq.matchLen = uint32(l) - zstdMinMatch
|
|
|
- seq.litLen = 0
|
|
|
-
|
|
|
- // Since litlen is always 0, this is offset 1.
|
|
|
- seq.offset = 1
|
|
|
- s += l
|
|
|
- nextEmit = s
|
|
|
- if debugSequences {
|
|
|
- println("sequence", seq, "next s:", s)
|
|
|
- }
|
|
|
- blk.sequences = append(blk.sequences, seq)
|
|
|
-
|
|
|
- // Swap offset 1 and 2.
|
|
|
- offset1, offset2 = offset2, offset1
|
|
|
- if s >= sLimit {
|
|
|
- // Finished
|
|
|
- break encodeLoop
|
|
|
- }
|
|
|
- cv = load6432(src, s)
|
|
|
- }
|
|
|
}
|
|
|
|
|
|
if int(nextEmit) < len(src) {
|
Sebastiaan van Stijn