0ct0pu5/moby
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

vendor: github.com/containerd/nydus-snapshotter v0.13.7

Browse source 
Update to the latest patch release, which contains changes from v0.13.5 to
remove the reference package from "github.com/docker/distribution", which
is now a separate module.

full diff: https://github.com/containerd/nydus-snapshotter/compare/v0.8.2...v0.13.7

Signed-off-by: Sebastiaan van Stijn <github@gone.nl>
This commit is contained in:
Sebastiaan van Stijn 2024-02-12 09:10:02 +01:00
parent 6932939326
commit ef7766304c
No known key found for this signature in database
GPG key ID: 76698F39D527CE8C
10 changed files with 304 additions and 148 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
vendor.mod
View file

@ -137,7 +137,7 @@ require (
github.com/containerd/console v1.0.3 // indirect
github.com/containerd/go-cni v1.1.9 // indirect
github.com/containerd/go-runc v1.1.0 // indirect
github.com/containerd/nydus-snapshotter v0.8.2 // indirect
github.com/containerd/nydus-snapshotter v0.13.7 // indirect
github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
github.com/containerd/ttrpc v1.2.2 // indirect
github.com/containernetworking/cni v1.1.2 // indirect

4
vendor.sum
View file

@ -325,8 +325,8 @@ github.com/containerd/go-runc v1.1.0 h1:OX4f+/i2y5sUT7LhmcJH7GYrjjhHa1QI4e8yO0gG
github.com/containerd/go-runc v1.1.0/go.mod h1:xJv2hFF7GvHtTJd9JqTS2UVxMkULUYw4JN5XAUZqH5U=
github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
github.com/containerd/nydus-snapshotter v0.8.2 h1:7SOrMU2YmLzfbsr5J7liMZJlNi5WT6vtIOxLGv+iz7E=
github.com/containerd/nydus-snapshotter v0.8.2/go.mod h1:UJILTN5LVBRY+dt8BGJbp72Xy729hUZsOugObEI3/O8=
github.com/containerd/nydus-snapshotter v0.13.7 h1:x7DHvGnzJOu1ZPwPYkeOPk5MjZZYbdddygEjaSDoFTk=
github.com/containerd/nydus-snapshotter v0.13.7/go.mod h1:VPVKQ3jmHFIcUIV2yiQ1kImZuBFS3GXDohKs9mRABVE=
github.com/containerd/stargz-snapshotter v0.0.0-20201027054423-3a04e4c2c116/go.mod h1:o59b3PCKVAf9jjiKtCc/9hLAd+5p/rfhBfm6aBcTEr4=
github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSkDwbkEETK84kQgEeFwDC+62k=
github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o=

3
vendor/github.com/containerd/nydus-snapshotter/pkg/converter/constant.go generated vendored
View file

@ -19,6 +19,9 @@ const (
LayerAnnotationNydusBlobSize = "containerd.io/snapshot/nydus-blob-size"
LayerAnnotationNydusBootstrap = "containerd.io/snapshot/nydus-bootstrap"
LayerAnnotationNydusSourceChainID = "containerd.io/snapshot/nydus-source-chainid"
LayerAnnotationNydusEncryptedBlob = "containerd.io/snapshot/nydus-encrypted-blob"
LayerAnnotationNydusSourceDigest = "containerd.io/snapshot/nydus-source-digest"
LayerAnnotationNydusTargetDigest = "containerd.io/snapshot/nydus-target-digest"
LayerAnnotationNydusReferenceBlobIDs = "containerd.io/snapshot/nydus-reference-blob-ids"

193
vendor/github.com/containerd/nydus-snapshotter/pkg/converter/convert_unix.go generated vendored
View file

@ -122,7 +122,7 @@ func unpackOciTar(ctx context.Context, dst string, reader io.Reader) error {
// unpackNydusBlob unpacks a Nydus formatted tar stream into a directory.
// unpackBlob indicates whether to unpack blob data.
func unpackNydusBlob(bootDst, blobDst string, ra content.ReaderAt, unpackBlob bool) error {
boot, err := os.OpenFile(bootDst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
boot, err := os.OpenFile(bootDst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0640)
if err != nil {
return errors.Wrapf(err, "write to bootstrap %s", bootDst)
}
@ -133,7 +133,7 @@ func unpackNydusBlob(bootDst, blobDst string, ra content.ReaderAt, unpackBlob bo
}
if unpackBlob {
blob, err := os.OpenFile(blobDst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
blob, err := os.OpenFile(blobDst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0640)
if err != nil {
return errors.Wrapf(err, "write to blob %s", blobDst)
}
@ -152,7 +152,7 @@ func unpackNydusBlob(bootDst, blobDst string, ra content.ReaderAt, unpackBlob bo
return nil
}
func seekFileByTarHeader(ra content.ReaderAt, targetName string, handle func(io.Reader, *tar.Header) error) error {
func seekFileByTarHeader(ra content.ReaderAt, targetName string, maxSize *int64, handle func(io.Reader, *tar.Header) error) error {
const headerSize = 512
if headerSize > ra.Size() {
@ -183,6 +183,10 @@ func seekFileByTarHeader(ra content.ReaderAt, targetName string, handle func(io.
}
if hdr.Name == targetName {
if maxSize != nil && hdr.Size > *maxSize {
return fmt.Errorf("invalid nydus tar size %d", ra.Size())
}
// Try to seek the part of tar data.
_, err = reader.Seek(cur-hdr.Size, io.SeekStart)
if err != nil {
@ -208,9 +212,10 @@ func seekFileByTarHeader(ra content.ReaderAt, targetName string, handle func(io.
func seekFileByTOC(ra content.ReaderAt, targetName string, handle func(io.Reader, *tar.Header) error) (*TOCEntry, error) {
entrySize := 128
maxSize := int64(1 << 20)
var tocEntry *TOCEntry
err := seekFileByTarHeader(ra, EntryTOC, func(tocEntryDataReader io.Reader, _ *tar.Header) error {
err := seekFileByTarHeader(ra, EntryTOC, &maxSize, func(tocEntryDataReader io.Reader, _ *tar.Header) error {
entryData, err := io.ReadAll(tocEntryDataReader)
if err != nil {
return errors.Wrap(err, "read toc entries")
@ -296,7 +301,7 @@ func seekFile(ra content.ReaderAt, targetName string, handle func(io.Reader, *ta
}
// Seek target data by tar header, ensure compatible with old rafs blob format.
return nil, seekFileByTarHeader(ra, targetName, handle)
return nil, seekFileByTarHeader(ra, targetName, nil, handle)
}
// Pack converts an OCI tar stream to nydus formatted stream with a tar-like
@ -316,7 +321,20 @@ func Pack(ctx context.Context, dest io.Writer, opt PackOption) (io.WriteCloser,
}
builderPath := getBuilder(opt.BuilderPath)
opt.features = tool.DetectFeatures(builderPath, []tool.Feature{tool.FeatureTar2Rafs})
requiredFeatures := tool.NewFeatures(tool.FeatureTar2Rafs)
if opt.BatchSize != "" && opt.BatchSize != "0" {
requiredFeatures.Add(tool.FeatureBatchSize)
}
if opt.Encrypt {
requiredFeatures.Add(tool.FeatureEncrypt)
}
detectedFeatures, err := tool.DetectFeatures(builderPath, requiredFeatures, tool.GetHelp)
if err != nil {
return nil, err
}
opt.features = detectedFeatures
if opt.OCIRef {
if opt.FsVersion == "6" {
@ -325,10 +343,18 @@ func Pack(ctx context.Context, dest io.Writer, opt PackOption) (io.WriteCloser,
return nil, fmt.Errorf("oci ref can only be supported by fs version 6")
}
if opt.features.Contains(tool.FeatureBatchSize) && opt.FsVersion != "6" {
return nil, fmt.Errorf("'--batch-size' can only be supported by fs version 6")
}
if opt.features.Contains(tool.FeatureTar2Rafs) {
return packFromTar(ctx, dest, opt)
}
return packFromDirectory(ctx, dest, opt, builderPath)
}
func packFromDirectory(ctx context.Context, dest io.Writer, opt PackOption, builderPath string) (io.WriteCloser, error) {
workDir, err := ensureWorkDir(opt.WorkDir)
if err != nil {
return nil, errors.Wrap(err, "ensure work directory")
@ -365,7 +391,7 @@ func Pack(ctx context.Context, dest io.Writer, opt PackOption) (io.WriteCloser,
<-unpackDone
blobPath := filepath.Join(workDir, "blob")
blobFifo, err := fifo.OpenFifo(ctx, blobPath, syscall.O_CREAT|syscall.O_RDONLY|syscall.O_NONBLOCK, 0644)
blobFifo, err := fifo.OpenFifo(ctx, blobPath, syscall.O_CREAT|syscall.O_RDONLY|syscall.O_NONBLOCK, 0640)
if err != nil {
return errors.Wrapf(err, "create fifo file")
}
@ -382,8 +408,10 @@ func Pack(ctx context.Context, dest io.Writer, opt PackOption) (io.WriteCloser,
PrefetchPatterns: opt.PrefetchPatterns,
AlignedChunk: opt.AlignedChunk,
ChunkSize: opt.ChunkSize,
BatchSize: opt.BatchSize,
Compressor: opt.Compressor,
Timeout: opt.Timeout,
Encrypt: opt.Encrypt,
Features: opt.features,
})
@ -417,13 +445,13 @@ func packFromTar(ctx context.Context, dest io.Writer, opt PackOption) (io.WriteC
}()
rafsBlobPath := filepath.Join(workDir, "blob.rafs")
rafsBlobFifo, err := fifo.OpenFifo(ctx, rafsBlobPath, syscall.O_CREAT|syscall.O_RDONLY|syscall.O_NONBLOCK, 0644)
rafsBlobFifo, err := fifo.OpenFifo(ctx, rafsBlobPath, syscall.O_CREAT|syscall.O_RDONLY|syscall.O_NONBLOCK, 0640)
if err != nil {
return nil, errors.Wrapf(err, "create fifo file")
}
tarBlobPath := filepath.Join(workDir, "blob.targz")
tarBlobFifo, err := fifo.OpenFifo(ctx, tarBlobPath, syscall.O_CREAT|syscall.O_WRONLY|syscall.O_NONBLOCK, 0644)
tarBlobFifo, err := fifo.OpenFifo(ctx, tarBlobPath, syscall.O_CREAT|syscall.O_WRONLY|syscall.O_NONBLOCK, 0640)
if err != nil {
defer rafsBlobFifo.Close()
return nil, errors.Wrapf(err, "create fifo file")
@ -487,6 +515,7 @@ func packFromTar(ctx context.Context, dest io.Writer, opt PackOption) (io.WriteC
BatchSize: opt.BatchSize,
Compressor: opt.Compressor,
Timeout: opt.Timeout,
Encrypt: opt.Encrypt,
Features: opt.features,
})
@ -503,8 +532,9 @@ func packFromTar(ctx context.Context, dest io.Writer, opt PackOption) (io.WriteC
}
func calcBlobTOCDigest(ra content.ReaderAt) (*digest.Digest, error) {
maxSize := int64(1 << 20)
digester := digest.Canonical.Digester()
if err := seekFileByTarHeader(ra, EntryTOC, func(tocData io.Reader, _ *tar.Header) error {
if err := seekFileByTarHeader(ra, EntryTOC, &maxSize, func(tocData io.Reader, _ *tar.Header) error {
if _, err := io.Copy(digester.Hash(), tocData); err != nil {
return errors.Wrap(err, "calc toc data and header digest")
}
@ -543,13 +573,13 @@ func Merge(ctx context.Context, layers []Layer, dest io.Writer, opt MergeOption)
for idx := range layers {
sourceBootstrapPaths = append(sourceBootstrapPaths, getBootstrapPath(idx))
if layers[idx].OriginalDigest != nil {
rafsBlobDigests = append(rafsBlobDigests, layers[idx].Digest.Hex())
rafsBlobSizes = append(rafsBlobSizes, layers[idx].ReaderAt.Size())
rafsBlobTOCDigest, err := calcBlobTOCDigest(layers[idx].ReaderAt)
if err != nil {
return nil, errors.Wrapf(err, "calc blob toc digest for layer %s", layers[idx].Digest)
}
rafsBlobTOCDigests = append(rafsBlobTOCDigests, rafsBlobTOCDigest.Hex())
rafsBlobDigests = append(rafsBlobDigests, layers[idx].Digest.Hex())
rafsBlobSizes = append(rafsBlobSizes, layers[idx].ReaderAt.Size())
}
eg.Go(func(idx int) func() error {
return func() error {
@ -632,7 +662,7 @@ func Unpack(ctx context.Context, ra content.ReaderAt, dest io.Writer, opt Unpack
}
tarPath := filepath.Join(workDir, "oci.tar")
blobFifo, err := fifo.OpenFifo(ctx, tarPath, syscall.O_CREAT|syscall.O_RDONLY|syscall.O_NONBLOCK, 0644)
blobFifo, err := fifo.OpenFifo(ctx, tarPath, syscall.O_CREAT|syscall.O_RDONLY|syscall.O_NONBLOCK, 0640)
if err != nil {
return errors.Wrapf(err, "create fifo file")
}
@ -656,7 +686,7 @@ func Unpack(ctx context.Context, ra content.ReaderAt, dest io.Writer, opt Unpack
// generate backend config file
backendConfigStr := fmt.Sprintf(`{"version":2,"backend":{"type":"http-proxy","http-proxy":{"addr":"%s"}}}`, proxy.socketPath)
backendConfigPath := filepath.Join(workDir, "backend-config.json")
if err := os.WriteFile(backendConfigPath, []byte(backendConfigStr), 0644); err != nil {
if err := os.WriteFile(backendConfigPath, []byte(backendConfigStr), 0640); err != nil {
return errors.Wrap(err, "write backend config")
}
unpackOpt.BlobPath = ""
@ -716,6 +746,60 @@ func IsNydusBootstrap(desc ocispec.Descriptor) bool {
return hasAnno
}
// isNydusImage checks if the last layer is nydus bootstrap,
// so that we can ensure it is a nydus image.
func isNydusImage(manifest *ocispec.Manifest) bool {
layers := manifest.Layers
if len(layers) != 0 {
desc := layers[len(layers)-1]
if IsNydusBootstrap(desc) {
return true
}
}
return false
}
// makeBlobDesc returns a ocispec.Descriptor by the given information.
func makeBlobDesc(ctx context.Context, cs content.Store, opt PackOption, sourceDigest, targetDigest digest.Digest) (*ocispec.Descriptor, error) {
targetInfo, err := cs.Info(ctx, targetDigest)
if err != nil {
return nil, errors.Wrapf(err, "get target blob info %s", targetDigest)
}
if targetInfo.Labels == nil {
targetInfo.Labels = map[string]string{}
}
// Write a diff id label of layer in content store for simplifying
// diff id calculation to speed up the conversion.
// See: https://github.com/containerd/containerd/blob/e4fefea5544d259177abb85b64e428702ac49c97/images/diffid.go#L49
targetInfo.Labels[labels.LabelUncompressed] = targetDigest.String()
_, err = cs.Update(ctx, targetInfo)
if err != nil {
return nil, errors.Wrap(err, "update layer label")
}
targetDesc := ocispec.Descriptor{
Digest: targetDigest,
Size: targetInfo.Size,
MediaType: MediaTypeNydusBlob,
Annotations: map[string]string{
// Use `containerd.io/uncompressed` to generate DiffID of
// layer defined in OCI spec.
LayerAnnotationUncompressed: targetDigest.String(),
LayerAnnotationNydusBlob: "true",
},
}
if opt.OCIRef {
targetDesc.Annotations[label.NydusRefLayer] = sourceDigest.String()
}
if opt.Encrypt {
targetDesc.Annotations[LayerAnnotationNydusEncryptedBlob] = "true"
}
return &targetDesc, nil
}
// LayerConvertFunc returns a function which converts an OCI image layer to
// a nydus blob layer, and set the media type to "application/vnd.oci.image.layer.nydus.blob.v1".
func LayerConvertFunc(opt PackOption) converter.ConvertFunc {
@ -729,6 +813,15 @@ func LayerConvertFunc(opt PackOption) converter.ConvertFunc {
return nil, nil
}
// Use remote cache to avoid unnecessary conversion
info, err := cs.Info(ctx, desc.Digest)
if err != nil {
return nil, errors.Wrapf(err, "get blob info %s", desc.Digest)
}
if targetDigest := digest.Digest(info.Labels[LayerAnnotationNydusTargetDigest]); targetDigest.Validate() == nil {
return makeBlobDesc(ctx, cs, opt, desc.Digest, targetDigest)
}
ra, err := cs.ReaderAt(ctx, desc)
if err != nil {
return nil, errors.Wrap(err, "get source blob reader")
@ -783,45 +876,18 @@ func LayerConvertFunc(opt PackOption) converter.ConvertFunc {
}
blobDigest := digester.Digest()
info, err := cs.Info(ctx, blobDigest)
newDesc, err := makeBlobDesc(ctx, cs, opt, desc.Digest, blobDigest)
if err != nil {
return nil, errors.Wrapf(err, "get blob info %s", blobDigest)
}
if info.Labels == nil {
info.Labels = map[string]string{}
}
// Write a diff id label of layer in content store for simplifying
// diff id calculation to speed up the conversion.
// See: https://github.com/containerd/containerd/blob/e4fefea5544d259177abb85b64e428702ac49c97/images/diffid.go#L49
info.Labels[labels.LabelUncompressed] = blobDigest.String()
_, err = cs.Update(ctx, info)
if err != nil {
return nil, errors.Wrap(err, "update layer label")
}
newDesc := ocispec.Descriptor{
Digest: blobDigest,
Size: info.Size,
MediaType: MediaTypeNydusBlob,
Annotations: map[string]string{
// Use `containerd.io/uncompressed` to generate DiffID of
// layer defined in OCI spec.
LayerAnnotationUncompressed: blobDigest.String(),
LayerAnnotationNydusBlob: "true",
},
}
if opt.OCIRef {
newDesc.Annotations[label.NydusRefLayer] = desc.Digest.String()
return nil, err
}
if opt.Backend != nil {
if err := opt.Backend.Push(ctx, cs, newDesc); err != nil {
if err := opt.Backend.Push(ctx, cs, *newDesc); err != nil {
return nil, errors.Wrap(err, "push to storage backend")
}
}
return &newDesc, nil
return newDesc, nil
}
}
@ -890,19 +956,6 @@ func convertIndex(ctx context.Context, cs content.Store, orgDesc ocispec.Descrip
return newIndexDesc, nil
}
// isNydusImage checks if the last layer is nydus bootstrap,
// so that we can ensure it is a nydus image.
func isNydusImage(manifest *ocispec.Manifest) bool {
layers := manifest.Layers
if len(layers) != 0 {
desc := layers[len(layers)-1]
if IsNydusBootstrap(desc) {
return true
}
}
return false
}
// convertManifest merges all the nydus blob layers into a
// nydus bootstrap layer, update the image config,
// and modify the image manifest.
@ -927,7 +980,7 @@ func convertManifest(ctx context.Context, cs content.Store, oldDesc ocispec.Desc
opt.OCI = true
}
// Append bootstrap layer to manifest.
// Append bootstrap layer to manifest, encrypt bootstrap layer if needed.
bootstrapDesc, blobDescs, err := MergeLayers(ctx, cs, manifest.Layers, opt)
if err != nil {
return nil, errors.Wrap(err, "merge nydus layers")
@ -985,10 +1038,12 @@ func convertManifest(ctx context.Context, cs content.Store, oldDesc ocispec.Desc
// Update the config gc label
manifestLabels[configGCLabelKey] = newConfigDesc.Digest.String()
// Associate a reference to the original OCI manifest.
// See the `subject` field description in
// https://github.com/opencontainers/image-spec/blob/main/manifest.md#image-manifest-property-descriptions
manifest.Subject = &oldDesc
if opt.WithReferrer {
// Associate a reference to the original OCI manifest.
// See the `subject` field description in
// https://github.com/opencontainers/image-spec/blob/main/manifest.md#image-manifest-property-descriptions
manifest.Subject = &oldDesc
}
// Update image manifest in content store.
newManifestDesc, err := writeJSON(ctx, cs, manifest, manifestDesc, manifestLabels)
@ -1110,6 +1165,11 @@ func MergeLayers(ctx context.Context, cs content.Store, descs []ocispec.Descript
if opt.OCIRef {
blobDesc.Annotations[label.NydusRefLayer] = layers[idx].OriginalDigest.String()
}
if opt.Encrypt != nil {
blobDesc.Annotations[LayerAnnotationNydusEncryptedBlob] = "true"
}
blobDescs = append(blobDescs, blobDesc)
}
@ -1133,5 +1193,12 @@ func MergeLayers(ctx context.Context, cs content.Store, descs []ocispec.Descript
},
}
if opt.Encrypt != nil {
// Encrypt the Nydus bootstrap layer.
bootstrapDesc, err = opt.Encrypt(ctx, cs, bootstrapDesc)
if err != nil {
return nil, nil, errors.Wrap(err, "encrypt bootstrap layer")
}
}
return &bootstrapDesc, blobDescs, nil
}

13
vendor/github.com/containerd/nydus-snapshotter/pkg/converter/convert_windows.go generated vendored
View file

@ -15,6 +15,7 @@ import (
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/images/converter"
"github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)
@ -22,11 +23,11 @@ func Pack(ctx context.Context, dest io.Writer, opt PackOption) (io.WriteCloser,
panic("not implemented")
}
func Merge(ctx context.Context, layers []Layer, dest io.Writer, opt MergeOption) error {
func Merge(ctx context.Context, layers []Layer, dest io.Writer, opt MergeOption) ([]digest.Digest, error) {
panic("not implemented")
}
func Unpack(ctx context.Context, ia content.ReaderAt, dest io.Writer, opt UnpackOption) error {
func Unpack(ctx context.Context, ra content.ReaderAt, dest io.Writer, opt UnpackOption) error {
panic("not implemented")
}
@ -34,7 +35,11 @@ func IsNydusBlobAndExists(ctx context.Context, cs content.Store, desc ocispec.De
panic("not implemented")
}
func IsNydusBlob(ctx context.Context, desc ocispec.Descriptor) bool {
func IsNydusBlob(desc ocispec.Descriptor) bool {
panic("not implemented")
}
func IsNydusBootstrap(desc ocispec.Descriptor) bool {
panic("not implemented")
}
@ -46,6 +51,6 @@ func ConvertHookFunc(opt MergeOption) converter.ConvertHookFunc {
panic("not implemented")
}
func MergeLayers(ctx context.Context, cs content.Store, descs []ocispec.Descriptor, opt MergeOption) (*ocispec.Descriptor, error) {
func MergeLayers(ctx context.Context, cs content.Store, descs []ocispec.Descriptor, opt MergeOption) (*ocispec.Descriptor, []ocispec.Descriptor, error) {
panic("not implemented")
}

6
vendor/github.com/containerd/nydus-snapshotter/pkg/converter/tool/builder.go generated vendored
View file

@ -40,6 +40,7 @@ type PackOption struct {
AlignedChunk bool
ChunkSize string
BatchSize string
Encrypt bool
Timeout *time.Duration
Features Features
@ -133,9 +134,12 @@ func buildPackArgs(option PackOption) []string {
if option.ChunkSize != "" {
args = append(args, "--chunk-size", option.ChunkSize)
}
if option.BatchSize != "" {
if option.Features.Contains(FeatureBatchSize) {
args = append(args, "--batch-size", option.BatchSize)
}
if option.Encrypt {
args = append(args, "--encrypt")
}
args = append(args, option.SourcePath)
return args

169
vendor/github.com/containerd/nydus-snapshotter/pkg/converter/tool/feature.go generated vendored
View file

@ -8,20 +8,19 @@ package tool
import (
"context"
"fmt"
"os"
"os/exec"
"regexp"
"strings"
"sync"
"github.com/sirupsen/logrus"
"golang.org/x/mod/semver"
)
const envNydusDisableTar2Rafs = "NYDUS_DISABLE_TAR2RAFS"
type Feature string
type Features map[Feature]struct{}
var currentVersion string
var currentVersionDetectOnce sync.Once
var disableTar2Rafs = os.Getenv(envNydusDisableTar2Rafs) != ""
const envNydusDisableTar2Rafs string = "NYDUS_DISABLE_TAR2RAFS"
const (
// The option `--type tar-rafs` enables converting OCI tar blob
@ -29,85 +28,119 @@ const (
// need to decompress it to a local directory first, thus greatly
// accelerating the pack process.
FeatureTar2Rafs Feature = "--type tar-rafs"
// The option `--batch-size` enables merging multiple small chunks
// into a big batch chunk, which can reduce the the size of the image
// and accelerate the runtime file loading.
FeatureBatchSize Feature = "--batch-size"
// The option `--encrypt` enables converting directories, tar files
// or OCI images into encrypted nydus blob.
FeatureEncrypt Feature = "--encrypt"
)
var featureMap = map[Feature]string{
FeatureTar2Rafs: "v2.2",
var requiredFeatures Features
var detectedFeatures Features
var detectFeaturesOnce sync.Once
var disableTar2Rafs = os.Getenv(envNydusDisableTar2Rafs) != ""
func NewFeatures(items ...Feature) Features {
features := Features{}
features.Add(items...)
return features
}
type Feature string
type Features []Feature
func (features *Features) Add(items ...Feature) {
for _, item := range items {
(*features)[item] = struct{}{}
}
}
func (features *Features) Remove(items ...Feature) {
for _, item := range items {
delete(*features, item)
}
}
func (features *Features) Contains(feature Feature) bool {
for _, feat := range *features {
if feat == feature {
_, ok := (*features)[feature]
return ok
}
func (features *Features) Equals(other Features) bool {
if len(*features) != len(other) {
return false
}
for f := range *features {
if !other.Contains(f) {
return false
}
}
return true
}
// GetHelp returns the help message of `nydus-image create`.
func GetHelp(builder string) []byte {
cmd := exec.CommandContext(context.Background(), builder, "create", "-h")
output, err := cmd.Output()
if err != nil {
return nil
}
return output
}
// detectFeature returns true if the feature is detected in the help message.
func detectFeature(msg []byte, feature Feature) bool {
if feature == "" {
return false
}
if strings.Contains(string(msg), string(feature)) {
return true
}
if parts := strings.Split(string(feature), " "); len(parts) == 2 {
// Check each part of the feature.
// e.g., "--type tar-rafs" -> ["--type", "tar-rafs"]
if strings.Contains(string(msg), parts[0]) && strings.Contains(string(msg), parts[1]) {
return true
}
}
return false
}
func (features *Features) Remove(feature Feature) {
found := -1
for idx, feat := range *features {
if feat == feature {
found = idx
break
}
}
if found != -1 {
*features = append((*features)[:found], (*features)[found+1:]...)
}
}
func detectVersion(msg []byte) string {
re := regexp.MustCompile(`Version:\s*v*(\d+.\d+.\d+)`)
matches := re.FindSubmatch(msg)
if len(matches) > 1 {
return string(matches[1])
}
return ""
}
// DetectFeatures returns supported feature list from required feature list.
func DetectFeatures(builder string, required Features) Features {
currentVersionDetectOnce.Do(func() {
if required.Contains(FeatureTar2Rafs) && disableTar2Rafs {
logrus.Warnf("the feature '%s' is disabled by env '%s'", FeatureTar2Rafs, envNydusDisableTar2Rafs)
}
// The supported feature list is detected from the help message of `nydus-image create`.
func DetectFeatures(builder string, required Features, getHelp func(string) []byte) (Features, error) {
detectFeaturesOnce.Do(func() {
requiredFeatures = required
detectedFeatures = Features{}
cmd := exec.CommandContext(context.Background(), builder, "--version")
output, err := cmd.Output()
if err != nil {
return
}
helpMsg := getHelp(builder)
currentVersion = detectVersion(output)
for feature := range required {
// The feature is supported by current version of nydus-image.
supported := detectFeature(helpMsg, feature)
if supported {
// It is an experimental feature, so we still provide an env
// variable to allow users to disable it.
if feature == FeatureTar2Rafs && disableTar2Rafs {
logrus.Warnf("the feature '%s' is disabled by env '%s'", FeatureTar2Rafs, envNydusDisableTar2Rafs)
continue
}
detectedFeatures.Add(feature)
} else {
logrus.Warnf("the feature '%s' is ignored, it requires higher version of nydus-image", feature)
}
}
})
if currentVersion == "" {
return Features{}
// Return Error if required features changed in different calls.
if !requiredFeatures.Equals(required) {
return nil, fmt.Errorf("features changed: %v -> %v", requiredFeatures, required)
}
detectedFeatures := Features{}
for _, feature := range required {
requiredVersion := featureMap[feature]
if requiredVersion == "" {
detectedFeatures = append(detectedFeatures, feature)
continue
}
// The feature is supported by current version
supported := semver.Compare(requiredVersion, "v"+currentVersion) <= 0
if supported {
// It is an experimental feature, so we still provide an env
// variable to allow users to disable it.
if feature == FeatureTar2Rafs && disableTar2Rafs {
continue
}
detectedFeatures = append(detectedFeatures, feature)
}
}
return detectedFeatures
return detectedFeatures, nil
}

30
vendor/github.com/containerd/nydus-snapshotter/pkg/converter/types.go generated vendored
View file

@ -21,9 +21,13 @@ import (
type Compressor = uint32
type Encrypter = func(context.Context, content.Store, ocispec.Descriptor) (ocispec.Descriptor, error)
const (
CompressorNone Compressor = 0x0001
CompressorZstd Compressor = 0x0002
CompressorNone Compressor = 0x0000_0001
CompressorZstd Compressor = 0x0000_0002
CompressorLz4Block Compressor = 0x0000_0004
CompressorMask Compressor = 0x0000_000f
)
var (
@ -77,6 +81,8 @@ type PackOption struct {
Backend Backend
// Timeout cancels execution once exceed the specified time.
Timeout *time.Duration
// Whether the generated Nydus blobs should be encrypted.
Encrypt bool
// Features keeps a feature list supported by newer version of builder,
// It is detected automatically, so don't export it.
@ -103,10 +109,22 @@ type MergeOption struct {
OCI bool
// OCIRef enables converting OCI tar(.gz) blob to nydus referenced blob.
OCIRef bool
// WithReferrer associates a reference to the original OCI manifest.
// See the `subject` field description in
// https://github.com/opencontainers/image-spec/blob/main/manifest.md#image-manifest-property-descriptions
//
// With this association, we can track all nydus images associated with
// an OCI image. For example, in Harbor we can cascade to show nydus
// images linked to an OCI image, deleting the OCI image can also delete
// the corresponding nydus images. At runtime, nydus snapshotter can also
// automatically upgrade an OCI image run to nydus image.
WithReferrer bool
// Backend uploads blobs generated by nydus-image builder to a backend storage.
Backend Backend
// Timeout cancels execution once exceed the specified time.
Timeout *time.Duration
// Encrypt encrypts the bootstrap layer if it's specified.
Encrypt Encrypter
}
type UnpackOption struct {
@ -139,11 +157,13 @@ type TOCEntry struct {
}
func (entry *TOCEntry) GetCompressor() (Compressor, error) {
switch {
case entry.Flags&CompressorNone == CompressorNone:
switch entry.Flags & CompressorMask {
case CompressorNone:
return CompressorNone, nil
case entry.Flags&CompressorZstd == CompressorZstd:
case CompressorZstd:
return CompressorZstd, nil
case CompressorLz4Block:
return CompressorLz4Block, nil
}
return 0, fmt.Errorf("unsupported compressor, entry flags %x", entry.Flags)
}

30
vendor/github.com/containerd/nydus-snapshotter/pkg/label/label.go generated vendored
View file

@ -34,10 +34,18 @@ const (
NydusMetaLayer = "containerd.io/snapshot/nydus-bootstrap"
// The referenced blob sha256 in format of `sha256:xxx`, set by image builders.
NydusRefLayer = "containerd.io/snapshot/nydus-ref"
// The blobID of associated layer, also marking the layer as a nydus tarfs, set by the snapshotter
NydusTarfsLayer = "containerd.io/snapshot/nydus-tarfs"
// Dm-verity information for image block device
NydusImageBlockInfo = "containerd.io/snapshot/nydus-image-block"
// Dm-verity information for layer block device
NydusLayerBlockInfo = "containerd.io/snapshot/nydus-layer-block"
// Annotation containing secret to pull images from registry, set by the snapshotter.
NydusImagePullSecret = "containerd.io/snapshot/pullsecret"
// Annotation containing username to pull images from registry, set by the snapshotter.
NydusImagePullUsername = "containerd.io/snapshot/pullusername"
// Proxy image pull actions to other agents.
NydusProxyMode = "containerd.io/snapshot/nydus-proxy-mode"
// A bool flag to enable integrity verification of meta data blob
NydusSignature = "containerd.io/snapshot/nydus-signature"
@ -48,6 +56,10 @@ const (
// If this optional label of a snapshot is specified, when mounted to rootdir
// this snapshot will include volatile option
OverlayfsVolatileOpt = "containerd.io/snapshot/overlay.volatile"
// A bool flag to mark it is recommended to run this image with tarfs mode, set by image builders.
// runtime can decide whether to rely on this annotation
TarfsHint = "containerd.io/snapshot/tarfs-hint"
)
func IsNydusDataLayer(labels map[string]string) bool {
@ -56,9 +68,21 @@ func IsNydusDataLayer(labels map[string]string) bool {
}
func IsNydusMetaLayer(labels map[string]string) bool {
if labels == nil {
return false
}
_, ok := labels[NydusMetaLayer]
return ok
}
func IsTarfsDataLayer(labels map[string]string) bool {
_, ok := labels[NydusTarfsLayer]
return ok
}
func IsNydusProxyMode(labels map[string]string) bool {
_, ok := labels[NydusProxyMode]
return ok
}
func HasTarfsHint(labels map[string]string) bool {
_, ok := labels[TarfsHint]
return ok
}

2
vendor/modules.txt vendored
View file

@ -365,7 +365,7 @@ github.com/containerd/go-runc
## explicit; go 1.20
github.com/containerd/log
github.com/containerd/log/logtest
# github.com/containerd/nydus-snapshotter v0.8.2
# github.com/containerd/nydus-snapshotter v0.13.7
## explicit; go 1.19
github.com/containerd/nydus-snapshotter/pkg/converter
github.com/containerd/nydus-snapshotter/pkg/converter/tool