beenull/moby
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

compression: add support for the zstd algorithm

Browse source 
zstd is a compression algorithm that has a very fast decoder, while
providing also good compression ratios.  The fast decoder makes it
suitable for container images, as decompressing the tarballs is a very
expensive operation.

https://github.com/opencontainers/image-spec/pull/788 added support
for zstd to the OCI image specs.

Signed-off-by: Giuseppe Scrivano <gscrivan@redhat.com>
This commit is contained in:
Giuseppe Scrivano 2020-12-08 10:56:32 +01:00
parent 5176095455
commit e187eb2bb5
No known key found for this signature in database
GPG key ID: E4730F97F60286ED
3 changed files with 29 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
Dockerfile
View file

@ -276,7 +276,8 @@ RUN --mount=type=cache,sharing=locked,id=moby-dev-aptlib,target=/var/lib/apt \
vim-common \
xfsprogs \
xz-utils \
zip
zip \
zstd
# Switch to use iptables instead of nftables (to match the CI hosts)

13
pkg/archive/archive.go
View file

@ -22,6 +22,7 @@ import (
"github.com/docker/docker/pkg/ioutils"
"github.com/docker/docker/pkg/pools"
"github.com/docker/docker/pkg/system"
"github.com/klauspost/compress/zstd"
"github.com/sirupsen/logrus"
exec "golang.org/x/sys/execabs"
)
@ -83,6 +84,8 @@ const (
Gzip
// Xz is xz compression algorithm.
Xz
// Zstd is zstd compression algorithm.
Zstd
)
const (
@ -127,6 +130,7 @@ func DetectCompression(source []byte) Compression {
Bzip2: {0x42, 0x5A, 0x68},
Gzip: {0x1F, 0x8B, 0x08},
Xz: {0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00},
Zstd: {0x28, 0xb5, 0x2f, 0xfd},
} {
if bytes.HasPrefix(source, m) {
return compression
@ -215,6 +219,13 @@ func DecompressStream(archive io.Reader) (io.ReadCloser, error) {
}
readBufWrapper := p.NewReadCloserWrapper(buf, xzReader)
return wrapReadCloser(readBufWrapper, cancel), nil
case Zstd:
zstdReader, err := zstd.NewReader(buf)
if err != nil {
return nil, err
}
readBufWrapper := p.NewReadCloserWrapper(buf, zstdReader)
return readBufWrapper, nil
default:
return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension())
}
@ -341,6 +352,8 @@ func (compression *Compression) Extension() string {
return "tar.gz"
case Xz:
return "tar.xz"
case Zstd:
return "tar.zst"
}
return ""
}

14
pkg/archive/archive_test.go
View file

@ -135,6 +135,13 @@ func TestDecompressStreamXz(t *testing.T) {
testDecompressStream(t, "xz", "xz -f")
}
func TestDecompressStreamZstd(t *testing.T) {
if _, err := exec.LookPath("zstd"); err != nil {
t.Skip("zstd not installed")
}
testDecompressStream(t, "zst", "zstd -f")
}
func TestCompressStreamXzUnsupported(t *testing.T) {
dest, err := os.Create(tmp + "dest")
if err != nil {
@ -210,6 +217,13 @@ func TestExtensionXz(t *testing.T) {
t.Fatalf("The extension of a xz archive should be 'tar.xz'")
}
}
func TestExtensionZstd(t *testing.T) {
compression := Zstd
output := compression.Extension()
if output != "tar.zst" {
t.Fatalf("The extension of a zstd archive should be 'tar.zst'")
}
}
func TestCmdStreamLargeStderr(t *testing.T) {
cmd := exec.Command("sh", "-c", "dd if=/dev/zero bs=1k count=1000 of=/dev/stderr; echo hello")