vendor: github.com/containerd/nydus-snapshotter v0.13.7

Update to the latest patch release, which contains changes from v0.13.5 to
remove the reference package from "github.com/docker/distribution", which
is now a separate module.

full diff: https://github.com/containerd/nydus-snapshotter/compare/v0.8.2...v0.13.7

Signed-off-by: Sebastiaan van Stijn <github@gone.nl>

vendor.mod

@@ -137,7 +137,7 @@ require (
 	github.com/containerd/console v1.0.3 // indirect
 	github.com/containerd/go-cni v1.1.9 // indirect
 	github.com/containerd/go-runc v1.1.0 // indirect
-	github.com/containerd/nydus-snapshotter v0.8.2 // indirect
+	github.com/containerd/nydus-snapshotter v0.13.7 // indirect
 	github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
 	github.com/containerd/ttrpc v1.2.2 // indirect
 	github.com/containernetworking/cni v1.1.2 // indirect

vendor.sum

@@ -325,8 +325,8 @@ github.com/containerd/go-runc v1.1.0 h1:OX4f+/i2y5sUT7LhmcJH7GYrjjhHa1QI4e8yO0gG
 github.com/containerd/go-runc v1.1.0/go.mod h1:xJv2hFF7GvHtTJd9JqTS2UVxMkULUYw4JN5XAUZqH5U=
 github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
 github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
-github.com/containerd/nydus-snapshotter v0.8.2 h1:7SOrMU2YmLzfbsr5J7liMZJlNi5WT6vtIOxLGv+iz7E=
-github.com/containerd/nydus-snapshotter v0.8.2/go.mod h1:UJILTN5LVBRY+dt8BGJbp72Xy729hUZsOugObEI3/O8=
+github.com/containerd/nydus-snapshotter v0.13.7 h1:x7DHvGnzJOu1ZPwPYkeOPk5MjZZYbdddygEjaSDoFTk=
+github.com/containerd/nydus-snapshotter v0.13.7/go.mod h1:VPVKQ3jmHFIcUIV2yiQ1kImZuBFS3GXDohKs9mRABVE=
 github.com/containerd/stargz-snapshotter v0.0.0-20201027054423-3a04e4c2c116/go.mod h1:o59b3PCKVAf9jjiKtCc/9hLAd+5p/rfhBfm6aBcTEr4=
 github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSkDwbkEETK84kQgEeFwDC+62k=
 github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o=

vendor/github.com/containerd/nydus-snapshotter/pkg/converter/constant.go

@@ -19,6 +19,9 @@ const (
 	LayerAnnotationNydusBlobSize      = "containerd.io/snapshot/nydus-blob-size"
 	LayerAnnotationNydusBootstrap     = "containerd.io/snapshot/nydus-bootstrap"
 	LayerAnnotationNydusSourceChainID = "containerd.io/snapshot/nydus-source-chainid"
+	LayerAnnotationNydusEncryptedBlob = "containerd.io/snapshot/nydus-encrypted-blob"
+	LayerAnnotationNydusSourceDigest  = "containerd.io/snapshot/nydus-source-digest"
+	LayerAnnotationNydusTargetDigest  = "containerd.io/snapshot/nydus-target-digest"
 
 	LayerAnnotationNydusReferenceBlobIDs = "containerd.io/snapshot/nydus-reference-blob-ids"
 

vendor/github.com/containerd/nydus-snapshotter/pkg/converter/convert_unix.go

@@ -122,7 +122,7 @@ func unpackOciTar(ctx context.Context, dst string, reader io.Reader) error {
 // unpackNydusBlob unpacks a Nydus formatted tar stream into a directory.
 // unpackBlob indicates whether to unpack blob data.
 func unpackNydusBlob(bootDst, blobDst string, ra content.ReaderAt, unpackBlob bool) error {
-	boot, err := os.OpenFile(bootDst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
+	boot, err := os.OpenFile(bootDst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0640)
 	if err != nil {
 		return errors.Wrapf(err, "write to bootstrap %s", bootDst)
 	}
@@ -133,7 +133,7 @@ func unpackNydusBlob(bootDst, blobDst string, ra content.ReaderAt, unpackBlob bo
 	}
 
 	if unpackBlob {
-		blob, err := os.OpenFile(blobDst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
+		blob, err := os.OpenFile(blobDst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0640)
 		if err != nil {
 			return errors.Wrapf(err, "write to blob %s", blobDst)
 		}
@@ -152,7 +152,7 @@ func unpackNydusBlob(bootDst, blobDst string, ra content.ReaderAt, unpackBlob bo
 	return nil
 }
 
-func seekFileByTarHeader(ra content.ReaderAt, targetName string, handle func(io.Reader, *tar.Header) error) error {
+func seekFileByTarHeader(ra content.ReaderAt, targetName string, maxSize *int64, handle func(io.Reader, *tar.Header) error) error {
 	const headerSize = 512
 
 	if headerSize > ra.Size() {
@@ -183,6 +183,10 @@ func seekFileByTarHeader(ra content.ReaderAt, targetName string, handle func(io.
 		}
 
 		if hdr.Name == targetName {
+			if maxSize != nil && hdr.Size > *maxSize {
+				return fmt.Errorf("invalid nydus tar size %d", ra.Size())
+			}
+
 			// Try to seek the part of tar data.
 			_, err = reader.Seek(cur-hdr.Size, io.SeekStart)
 			if err != nil {
@@ -208,9 +212,10 @@ func seekFileByTarHeader(ra content.ReaderAt, targetName string, handle func(io.
 
 func seekFileByTOC(ra content.ReaderAt, targetName string, handle func(io.Reader, *tar.Header) error) (*TOCEntry, error) {
 	entrySize := 128
+	maxSize := int64(1 << 20)
 	var tocEntry *TOCEntry
 
-	err := seekFileByTarHeader(ra, EntryTOC, func(tocEntryDataReader io.Reader, _ *tar.Header) error {
+	err := seekFileByTarHeader(ra, EntryTOC, &maxSize, func(tocEntryDataReader io.Reader, _ *tar.Header) error {
 		entryData, err := io.ReadAll(tocEntryDataReader)
 		if err != nil {
 			return errors.Wrap(err, "read toc entries")
@@ -296,7 +301,7 @@ func seekFile(ra content.ReaderAt, targetName string, handle func(io.Reader, *ta
 	}
 
 	// Seek target data by tar header, ensure compatible with old rafs blob format.
-	return nil, seekFileByTarHeader(ra, targetName, handle)
+	return nil, seekFileByTarHeader(ra, targetName, nil, handle)
 }
 
 // Pack converts an OCI tar stream to nydus formatted stream with a tar-like
@@ -316,7 +321,20 @@ func Pack(ctx context.Context, dest io.Writer, opt PackOption) (io.WriteCloser,
 	}
 
 	builderPath := getBuilder(opt.BuilderPath)
-	opt.features = tool.DetectFeatures(builderPath, []tool.Feature{tool.FeatureTar2Rafs})
+
+	requiredFeatures := tool.NewFeatures(tool.FeatureTar2Rafs)
+	if opt.BatchSize != "" && opt.BatchSize != "0" {
+		requiredFeatures.Add(tool.FeatureBatchSize)
+	}
+	if opt.Encrypt {
+		requiredFeatures.Add(tool.FeatureEncrypt)
+	}
+
+	detectedFeatures, err := tool.DetectFeatures(builderPath, requiredFeatures, tool.GetHelp)
+	if err != nil {
+		return nil, err
+	}
+	opt.features = detectedFeatures
 
 	if opt.OCIRef {
 		if opt.FsVersion == "6" {
@@ -325,10 +343,18 @@ func Pack(ctx context.Context, dest io.Writer, opt PackOption) (io.WriteCloser,
 		return nil, fmt.Errorf("oci ref can only be supported by fs version 6")
 	}
 
+	if opt.features.Contains(tool.FeatureBatchSize) && opt.FsVersion != "6" {
+		return nil, fmt.Errorf("'--batch-size' can only be supported by fs version 6")
+	}
+
 	if opt.features.Contains(tool.FeatureTar2Rafs) {
 		return packFromTar(ctx, dest, opt)
 	}
 
+	return packFromDirectory(ctx, dest, opt, builderPath)
+}
+
+func packFromDirectory(ctx context.Context, dest io.Writer, opt PackOption, builderPath string) (io.WriteCloser, error) {
 	workDir, err := ensureWorkDir(opt.WorkDir)
 	if err != nil {
 		return nil, errors.Wrap(err, "ensure work directory")
@@ -365,7 +391,7 @@ func Pack(ctx context.Context, dest io.Writer, opt PackOption) (io.WriteCloser,
 		<-unpackDone
 
 		blobPath := filepath.Join(workDir, "blob")
-		blobFifo, err := fifo.OpenFifo(ctx, blobPath, syscall.O_CREAT|syscall.O_RDONLY|syscall.O_NONBLOCK, 0644)
+		blobFifo, err := fifo.OpenFifo(ctx, blobPath, syscall.O_CREAT|syscall.O_RDONLY|syscall.O_NONBLOCK, 0640)
 		if err != nil {
 			return errors.Wrapf(err, "create fifo file")
 		}
@@ -382,8 +408,10 @@ func Pack(ctx context.Context, dest io.Writer, opt PackOption) (io.WriteCloser,
 				PrefetchPatterns: opt.PrefetchPatterns,
 				AlignedChunk:     opt.AlignedChunk,
 				ChunkSize:        opt.ChunkSize,
+				BatchSize:        opt.BatchSize,
 				Compressor:       opt.Compressor,
 				Timeout:          opt.Timeout,
+				Encrypt:          opt.Encrypt,
 
 				Features: opt.features,
 			})
@@ -417,13 +445,13 @@ func packFromTar(ctx context.Context, dest io.Writer, opt PackOption) (io.WriteC
 	}()
 
 	rafsBlobPath := filepath.Join(workDir, "blob.rafs")
-	rafsBlobFifo, err := fifo.OpenFifo(ctx, rafsBlobPath, syscall.O_CREAT|syscall.O_RDONLY|syscall.O_NONBLOCK, 0644)
+	rafsBlobFifo, err := fifo.OpenFifo(ctx, rafsBlobPath, syscall.O_CREAT|syscall.O_RDONLY|syscall.O_NONBLOCK, 0640)
 	if err != nil {
 		return nil, errors.Wrapf(err, "create fifo file")
 	}
 
 	tarBlobPath := filepath.Join(workDir, "blob.targz")
-	tarBlobFifo, err := fifo.OpenFifo(ctx, tarBlobPath, syscall.O_CREAT|syscall.O_WRONLY|syscall.O_NONBLOCK, 0644)
+	tarBlobFifo, err := fifo.OpenFifo(ctx, tarBlobPath, syscall.O_CREAT|syscall.O_WRONLY|syscall.O_NONBLOCK, 0640)
 	if err != nil {
 		defer rafsBlobFifo.Close()
 		return nil, errors.Wrapf(err, "create fifo file")
@@ -487,6 +515,7 @@ func packFromTar(ctx context.Context, dest io.Writer, opt PackOption) (io.WriteC
 				BatchSize:        opt.BatchSize,
 				Compressor:       opt.Compressor,
 				Timeout:          opt.Timeout,
+				Encrypt:          opt.Encrypt,
 
 				Features: opt.features,
 			})
@@ -503,8 +532,9 @@ func packFromTar(ctx context.Context, dest io.Writer, opt PackOption) (io.WriteC
 }
 
 func calcBlobTOCDigest(ra content.ReaderAt) (*digest.Digest, error) {
+	maxSize := int64(1 << 20)
 	digester := digest.Canonical.Digester()
-	if err := seekFileByTarHeader(ra, EntryTOC, func(tocData io.Reader, _ *tar.Header) error {
+	if err := seekFileByTarHeader(ra, EntryTOC, &maxSize, func(tocData io.Reader, _ *tar.Header) error {
 		if _, err := io.Copy(digester.Hash(), tocData); err != nil {
 			return errors.Wrap(err, "calc toc data and header digest")
 		}
@@ -543,13 +573,13 @@ func Merge(ctx context.Context, layers []Layer, dest io.Writer, opt MergeOption)
 	for idx := range layers {
 		sourceBootstrapPaths = append(sourceBootstrapPaths, getBootstrapPath(idx))
 		if layers[idx].OriginalDigest != nil {
-			rafsBlobDigests = append(rafsBlobDigests, layers[idx].Digest.Hex())
-			rafsBlobSizes = append(rafsBlobSizes, layers[idx].ReaderAt.Size())
 			rafsBlobTOCDigest, err := calcBlobTOCDigest(layers[idx].ReaderAt)
 			if err != nil {
 				return nil, errors.Wrapf(err, "calc blob toc digest for layer %s", layers[idx].Digest)
 			}
 			rafsBlobTOCDigests = append(rafsBlobTOCDigests, rafsBlobTOCDigest.Hex())
+			rafsBlobDigests = append(rafsBlobDigests, layers[idx].Digest.Hex())
+			rafsBlobSizes = append(rafsBlobSizes, layers[idx].ReaderAt.Size())
 		}
 		eg.Go(func(idx int) func() error {
 			return func() error {
@@ -632,7 +662,7 @@ func Unpack(ctx context.Context, ra content.ReaderAt, dest io.Writer, opt Unpack
 	}
 
 	tarPath := filepath.Join(workDir, "oci.tar")
-	blobFifo, err := fifo.OpenFifo(ctx, tarPath, syscall.O_CREAT|syscall.O_RDONLY|syscall.O_NONBLOCK, 0644)
+	blobFifo, err := fifo.OpenFifo(ctx, tarPath, syscall.O_CREAT|syscall.O_RDONLY|syscall.O_NONBLOCK, 0640)
 	if err != nil {
 		return errors.Wrapf(err, "create fifo file")
 	}
@@ -656,7 +686,7 @@ func Unpack(ctx context.Context, ra content.ReaderAt, dest io.Writer, opt Unpack
 		// generate backend config file
 		backendConfigStr := fmt.Sprintf(`{"version":2,"backend":{"type":"http-proxy","http-proxy":{"addr":"%s"}}}`, proxy.socketPath)
 		backendConfigPath := filepath.Join(workDir, "backend-config.json")
-		if err := os.WriteFile(backendConfigPath, []byte(backendConfigStr), 0644); err != nil {
+		if err := os.WriteFile(backendConfigPath, []byte(backendConfigStr), 0640); err != nil {
 			return errors.Wrap(err, "write backend config")
 		}
 		unpackOpt.BlobPath = ""
@@ -716,6 +746,60 @@ func IsNydusBootstrap(desc ocispec.Descriptor) bool {
 	return hasAnno
 }
 
+// isNydusImage checks if the last layer is nydus bootstrap,
+// so that we can ensure it is a nydus image.
+func isNydusImage(manifest *ocispec.Manifest) bool {
+	layers := manifest.Layers
+	if len(layers) != 0 {
+		desc := layers[len(layers)-1]
+		if IsNydusBootstrap(desc) {
+			return true
+		}
+	}
+	return false
+}
+
+// makeBlobDesc returns a ocispec.Descriptor by the given information.
+func makeBlobDesc(ctx context.Context, cs content.Store, opt PackOption, sourceDigest, targetDigest digest.Digest) (*ocispec.Descriptor, error) {
+	targetInfo, err := cs.Info(ctx, targetDigest)
+	if err != nil {
+		return nil, errors.Wrapf(err, "get target blob info %s", targetDigest)
+	}
+	if targetInfo.Labels == nil {
+		targetInfo.Labels = map[string]string{}
+	}
+	// Write a diff id label of layer in content store for simplifying
+	// diff id calculation to speed up the conversion.
+	// See: https://github.com/containerd/containerd/blob/e4fefea5544d259177abb85b64e428702ac49c97/images/diffid.go#L49
+	targetInfo.Labels[labels.LabelUncompressed] = targetDigest.String()
+	_, err = cs.Update(ctx, targetInfo)
+	if err != nil {
+		return nil, errors.Wrap(err, "update layer label")
+	}
+
+	targetDesc := ocispec.Descriptor{
+		Digest:    targetDigest,
+		Size:      targetInfo.Size,
+		MediaType: MediaTypeNydusBlob,
+		Annotations: map[string]string{
+			// Use `containerd.io/uncompressed` to generate DiffID of
+			// layer defined in OCI spec.
+			LayerAnnotationUncompressed: targetDigest.String(),
+			LayerAnnotationNydusBlob:    "true",
+		},
+	}
+
+	if opt.OCIRef {
+		targetDesc.Annotations[label.NydusRefLayer] = sourceDigest.String()
+	}
+
+	if opt.Encrypt {
+		targetDesc.Annotations[LayerAnnotationNydusEncryptedBlob] = "true"
+	}
+
+	return &targetDesc, nil
+}
+
 // LayerConvertFunc returns a function which converts an OCI image layer to
 // a nydus blob layer, and set the media type to "application/vnd.oci.image.layer.nydus.blob.v1".
 func LayerConvertFunc(opt PackOption) converter.ConvertFunc {
@@ -729,6 +813,15 @@ func LayerConvertFunc(opt PackOption) converter.ConvertFunc {
 			return nil, nil
 		}
 
+		// Use remote cache to avoid unnecessary conversion
+		info, err := cs.Info(ctx, desc.Digest)
+		if err != nil {
+			return nil, errors.Wrapf(err, "get blob info %s", desc.Digest)
+		}
+		if targetDigest := digest.Digest(info.Labels[LayerAnnotationNydusTargetDigest]); targetDigest.Validate() == nil {
+			return makeBlobDesc(ctx, cs, opt, desc.Digest, targetDigest)
+		}
+
 		ra, err := cs.ReaderAt(ctx, desc)
 		if err != nil {
 			return nil, errors.Wrap(err, "get source blob reader")
@@ -783,45 +876,18 @@ func LayerConvertFunc(opt PackOption) converter.ConvertFunc {
 		}
 
 		blobDigest := digester.Digest()
-		info, err := cs.Info(ctx, blobDigest)
-		if err != nil {
-			return nil, errors.Wrapf(err, "get blob info %s", blobDigest)
-		}
-		if info.Labels == nil {
-			info.Labels = map[string]string{}
-		}
-		// Write a diff id label of layer in content store for simplifying
-		// diff id calculation to speed up the conversion.
-		// See: https://github.com/containerd/containerd/blob/e4fefea5544d259177abb85b64e428702ac49c97/images/diffid.go#L49
-		info.Labels[labels.LabelUncompressed] = blobDigest.String()
-		_, err = cs.Update(ctx, info)
+		newDesc, err := makeBlobDesc(ctx, cs, opt, desc.Digest, blobDigest)
 		if err != nil {
-			return nil, errors.Wrap(err, "update layer label")
-		}
-
-		newDesc := ocispec.Descriptor{
-			Digest:    blobDigest,
-			Size:      info.Size,
-			MediaType: MediaTypeNydusBlob,
-			Annotations: map[string]string{
-				// Use `containerd.io/uncompressed` to generate DiffID of
-				// layer defined in OCI spec.
-				LayerAnnotationUncompressed: blobDigest.String(),
-				LayerAnnotationNydusBlob:    "true",
-			},
-		}
-
-		if opt.OCIRef {
-			newDesc.Annotations[label.NydusRefLayer] = desc.Digest.String()
+			return nil, err
 		}
 
 		if opt.Backend != nil {
-			if err := opt.Backend.Push(ctx, cs, newDesc); err != nil {
+			if err := opt.Backend.Push(ctx, cs, *newDesc); err != nil {
 				return nil, errors.Wrap(err, "push to storage backend")
 			}
 		}
 
-		return &newDesc, nil
+		return newDesc, nil
 	}
 }
 
@@ -890,19 +956,6 @@ func convertIndex(ctx context.Context, cs content.Store, orgDesc ocispec.Descrip
 	return newIndexDesc, nil
 }
 
-// isNydusImage checks if the last layer is nydus bootstrap,
-// so that we can ensure it is a nydus image.
-func isNydusImage(manifest *ocispec.Manifest) bool {
-	layers := manifest.Layers
-	if len(layers) != 0 {
-		desc := layers[len(layers)-1]
-		if IsNydusBootstrap(desc) {
-			return true
-		}
-	}
-	return false
-}
-
 // convertManifest merges all the nydus blob layers into a
 // nydus bootstrap layer, update the image config,
 // and modify the image manifest.
@@ -927,7 +980,7 @@ func convertManifest(ctx context.Context, cs content.Store, oldDesc ocispec.Desc
 		opt.OCI = true
 	}
 
-	// Append bootstrap layer to manifest.
+	// Append bootstrap layer to manifest, encrypt bootstrap layer if needed.
 	bootstrapDesc, blobDescs, err := MergeLayers(ctx, cs, manifest.Layers, opt)
 	if err != nil {
 		return nil, errors.Wrap(err, "merge nydus layers")
@@ -985,10 +1038,12 @@ func convertManifest(ctx context.Context, cs content.Store, oldDesc ocispec.Desc
 	// Update the config gc label
 	manifestLabels[configGCLabelKey] = newConfigDesc.Digest.String()
 
-	// Associate a reference to the original OCI manifest.
-	// See the `subject` field description in
-	// https://github.com/opencontainers/image-spec/blob/main/manifest.md#image-manifest-property-descriptions
-	manifest.Subject = &oldDesc
+	if opt.WithReferrer {
+		// Associate a reference to the original OCI manifest.
+		// See the `subject` field description in
+		// https://github.com/opencontainers/image-spec/blob/main/manifest.md#image-manifest-property-descriptions
+		manifest.Subject = &oldDesc
+	}
 
 	// Update image manifest in content store.
 	newManifestDesc, err := writeJSON(ctx, cs, manifest, manifestDesc, manifestLabels)
@@ -1110,6 +1165,11 @@ func MergeLayers(ctx context.Context, cs content.Store, descs []ocispec.Descript
 		if opt.OCIRef {
 			blobDesc.Annotations[label.NydusRefLayer] = layers[idx].OriginalDigest.String()
 		}
+
+		if opt.Encrypt != nil {
+			blobDesc.Annotations[LayerAnnotationNydusEncryptedBlob] = "true"
+		}
+
 		blobDescs = append(blobDescs, blobDesc)
 	}
 
@@ -1133,5 +1193,12 @@ func MergeLayers(ctx context.Context, cs content.Store, descs []ocispec.Descript
 		},
 	}
 
+	if opt.Encrypt != nil {
+		// Encrypt the Nydus bootstrap layer.
+		bootstrapDesc, err = opt.Encrypt(ctx, cs, bootstrapDesc)
+		if err != nil {
+			return nil, nil, errors.Wrap(err, "encrypt bootstrap layer")
+		}
+	}
 	return &bootstrapDesc, blobDescs, nil
 }

vendor/github.com/containerd/nydus-snapshotter/pkg/converter/convert_windows.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/containerd/containerd/content"
 	"github.com/containerd/containerd/images/converter"
+	"github.com/opencontainers/go-digest"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
@@ -22,11 +23,11 @@ func Pack(ctx context.Context, dest io.Writer, opt PackOption) (io.WriteCloser,
 	panic("not implemented")
 }
 
-func Merge(ctx context.Context, layers []Layer, dest io.Writer, opt MergeOption) error {
+func Merge(ctx context.Context, layers []Layer, dest io.Writer, opt MergeOption) ([]digest.Digest, error) {
 	panic("not implemented")
 }
 
-func Unpack(ctx context.Context, ia content.ReaderAt, dest io.Writer, opt UnpackOption) error {
+func Unpack(ctx context.Context, ra content.ReaderAt, dest io.Writer, opt UnpackOption) error {
 	panic("not implemented")
 }
 
@@ -34,7 +35,11 @@ func IsNydusBlobAndExists(ctx context.Context, cs content.Store, desc ocispec.De
 	panic("not implemented")
 }
 
-func IsNydusBlob(ctx context.Context, desc ocispec.Descriptor) bool {
+func IsNydusBlob(desc ocispec.Descriptor) bool {
+	panic("not implemented")
+}
+
+func IsNydusBootstrap(desc ocispec.Descriptor) bool {
 	panic("not implemented")
 }
 
@@ -46,6 +51,6 @@ func ConvertHookFunc(opt MergeOption) converter.ConvertHookFunc {
 	panic("not implemented")
 }
 
-func MergeLayers(ctx context.Context, cs content.Store, descs []ocispec.Descriptor, opt MergeOption) (*ocispec.Descriptor, error) {
+func MergeLayers(ctx context.Context, cs content.Store, descs []ocispec.Descriptor, opt MergeOption) (*ocispec.Descriptor, []ocispec.Descriptor, error) {
 	panic("not implemented")
 }

vendor/github.com/containerd/nydus-snapshotter/pkg/converter/tool/builder.go

@@ -40,6 +40,7 @@ type PackOption struct {
 	AlignedChunk     bool
 	ChunkSize        string
 	BatchSize        string
+	Encrypt          bool
 	Timeout          *time.Duration
 
 	Features Features
@@ -133,9 +134,12 @@ func buildPackArgs(option PackOption) []string {
 	if option.ChunkSize != "" {
 		args = append(args, "--chunk-size", option.ChunkSize)
 	}
-	if option.BatchSize != "" {
+	if option.Features.Contains(FeatureBatchSize) {
 		args = append(args, "--batch-size", option.BatchSize)
 	}
+	if option.Encrypt {
+		args = append(args, "--encrypt")
+	}
 	args = append(args, option.SourcePath)
 
 	return args

vendor/github.com/containerd/nydus-snapshotter/pkg/converter/tool/feature.go

@@ -8,20 +8,19 @@ package tool
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"os/exec"
-	"regexp"
+	"strings"
 	"sync"
 
 	"github.com/sirupsen/logrus"
-	"golang.org/x/mod/semver"
 )
 
-const envNydusDisableTar2Rafs = "NYDUS_DISABLE_TAR2RAFS"
+type Feature string
+type Features map[Feature]struct{}
 
-var currentVersion string
-var currentVersionDetectOnce sync.Once
-var disableTar2Rafs = os.Getenv(envNydusDisableTar2Rafs) != ""
+const envNydusDisableTar2Rafs string = "NYDUS_DISABLE_TAR2RAFS"
 
 const (
 	// The option `--type tar-rafs` enables converting OCI tar blob
@@ -29,85 +28,119 @@ const (
 	// need to decompress it to a local directory first, thus greatly
 	// accelerating the pack process.
 	FeatureTar2Rafs Feature = "--type tar-rafs"
+	// The option `--batch-size` enables merging multiple small chunks
+	// into a big batch chunk, which can reduce the the size of the image
+	// and accelerate the runtime file loading.
+	FeatureBatchSize Feature = "--batch-size"
+	// The option `--encrypt` enables converting directories, tar files
+	// or OCI images into encrypted nydus blob.
+	FeatureEncrypt Feature = "--encrypt"
 )
 
-var featureMap = map[Feature]string{
-	FeatureTar2Rafs: "v2.2",
-}
+var requiredFeatures Features
+var detectedFeatures Features
+var detectFeaturesOnce sync.Once
+var disableTar2Rafs = os.Getenv(envNydusDisableTar2Rafs) != ""
 
-type Feature string
-type Features []Feature
+func NewFeatures(items ...Feature) Features {
+	features := Features{}
+	features.Add(items...)
+	return features
+}
 
-func (features *Features) Contains(feature Feature) bool {
-	for _, feat := range *features {
-		if feat == feature {
-			return true
-		}
+func (features *Features) Add(items ...Feature) {
+	for _, item := range items {
+		(*features)[item] = struct{}{}
 	}
-	return false
 }
 
-func (features *Features) Remove(feature Feature) {
-	found := -1
-	for idx, feat := range *features {
-		if feat == feature {
-			found = idx
-			break
-		}
-	}
-	if found != -1 {
-		*features = append((*features)[:found], (*features)[found+1:]...)
+func (features *Features) Remove(items ...Feature) {
+	for _, item := range items {
+		delete(*features, item)
 	}
 }
 
-func detectVersion(msg []byte) string {
-	re := regexp.MustCompile(`Version:\s*v*(\d+.\d+.\d+)`)
-	matches := re.FindSubmatch(msg)
-	if len(matches) > 1 {
-		return string(matches[1])
-	}
-	return ""
+func (features *Features) Contains(feature Feature) bool {
+	_, ok := (*features)[feature]
+	return ok
 }
 
-// DetectFeatures returns supported feature list from required feature list.
-func DetectFeatures(builder string, required Features) Features {
-	currentVersionDetectOnce.Do(func() {
-		if required.Contains(FeatureTar2Rafs) && disableTar2Rafs {
-			logrus.Warnf("the feature '%s' is disabled by env '%s'", FeatureTar2Rafs, envNydusDisableTar2Rafs)
-		}
+func (features *Features) Equals(other Features) bool {
+	if len(*features) != len(other) {
+		return false
+	}
 
-		cmd := exec.CommandContext(context.Background(), builder, "--version")
-		output, err := cmd.Output()
-		if err != nil {
-			return
+	for f := range *features {
+		if !other.Contains(f) {
+			return false
 		}
+	}
 
-		currentVersion = detectVersion(output)
-	})
+	return true
+}
+
+// GetHelp returns the help message of `nydus-image create`.
+func GetHelp(builder string) []byte {
+	cmd := exec.CommandContext(context.Background(), builder, "create", "-h")
+	output, err := cmd.Output()
+	if err != nil {
+		return nil
+	}
+
+	return output
+}
+
+// detectFeature returns true if the feature is detected in the help message.
+func detectFeature(msg []byte, feature Feature) bool {
+	if feature == "" {
+		return false
+	}
 
-	if currentVersion == "" {
-		return Features{}
+	if strings.Contains(string(msg), string(feature)) {
+		return true
 	}
 
-	detectedFeatures := Features{}
-	for _, feature := range required {
-		requiredVersion := featureMap[feature]
-		if requiredVersion == "" {
-			detectedFeatures = append(detectedFeatures, feature)
-			continue
+	if parts := strings.Split(string(feature), " "); len(parts) == 2 {
+		// Check each part of the feature.
+		// e.g., "--type tar-rafs" -> ["--type", "tar-rafs"]
+		if strings.Contains(string(msg), parts[0]) && strings.Contains(string(msg), parts[1]) {
+			return true
 		}
+	}
+
+	return false
+}
 
-		// The feature is supported by current version
-		supported := semver.Compare(requiredVersion, "v"+currentVersion) <= 0
-		if supported {
-			// It is an experimental feature, so we still provide an env
-			// variable to allow users to disable it.
-			if feature == FeatureTar2Rafs && disableTar2Rafs {
-				continue
+// DetectFeatures returns supported feature list from required feature list.
+// The supported feature list is detected from the help message of `nydus-image create`.
+func DetectFeatures(builder string, required Features, getHelp func(string) []byte) (Features, error) {
+	detectFeaturesOnce.Do(func() {
+		requiredFeatures = required
+		detectedFeatures = Features{}
+
+		helpMsg := getHelp(builder)
+
+		for feature := range required {
+			// The feature is supported by current version of nydus-image.
+			supported := detectFeature(helpMsg, feature)
+			if supported {
+				// It is an experimental feature, so we still provide an env
+				// variable to allow users to disable it.
+				if feature == FeatureTar2Rafs && disableTar2Rafs {
+					logrus.Warnf("the feature '%s' is disabled by env '%s'", FeatureTar2Rafs, envNydusDisableTar2Rafs)
+					continue
+				}
+				detectedFeatures.Add(feature)
+			} else {
+				logrus.Warnf("the feature '%s' is ignored, it requires higher version of nydus-image", feature)
 			}
-			detectedFeatures = append(detectedFeatures, feature)
 		}
+	})
+
+	// Return Error if required features changed in different calls.
+	if !requiredFeatures.Equals(required) {
+		return nil, fmt.Errorf("features changed: %v -> %v", requiredFeatures, required)
 	}
 
-	return detectedFeatures
+	return detectedFeatures, nil
 }

vendor/github.com/containerd/nydus-snapshotter/pkg/converter/types.go

@@ -21,9 +21,13 @@ import (
 
 type Compressor = uint32
 
+type Encrypter = func(context.Context, content.Store, ocispec.Descriptor) (ocispec.Descriptor, error)
+
 const (
-	CompressorNone Compressor = 0x0001
-	CompressorZstd Compressor = 0x0002
+	CompressorNone     Compressor = 0x0000_0001
+	CompressorZstd     Compressor = 0x0000_0002
+	CompressorLz4Block Compressor = 0x0000_0004
+	CompressorMask     Compressor = 0x0000_000f
 )
 
 var (
@@ -77,6 +81,8 @@ type PackOption struct {
 	Backend Backend
 	// Timeout cancels execution once exceed the specified time.
 	Timeout *time.Duration
+	// Whether the generated Nydus blobs should be encrypted.
+	Encrypt bool
 
 	// Features keeps a feature list supported by newer version of builder,
 	// It is detected automatically, so don't export it.
@@ -103,10 +109,22 @@ type MergeOption struct {
 	OCI bool
 	// OCIRef enables converting OCI tar(.gz) blob to nydus referenced blob.
 	OCIRef bool
+	// WithReferrer associates a reference to the original OCI manifest.
+	// See the `subject` field description in
+	// https://github.com/opencontainers/image-spec/blob/main/manifest.md#image-manifest-property-descriptions
+	//
+	// With this association, we can track all nydus images associated with
+	// an OCI image. For example, in Harbor we can cascade to show nydus
+	// images linked to an OCI image, deleting the OCI image can also delete
+	// the corresponding nydus images. At runtime, nydus snapshotter can also
+	// automatically upgrade an OCI image run to nydus image.
+	WithReferrer bool
 	// Backend uploads blobs generated by nydus-image builder to a backend storage.
 	Backend Backend
 	// Timeout cancels execution once exceed the specified time.
 	Timeout *time.Duration
+	// Encrypt encrypts the bootstrap layer if it's specified.
+	Encrypt Encrypter
 }
 
 type UnpackOption struct {
@@ -139,11 +157,13 @@ type TOCEntry struct {
 }
 
 func (entry *TOCEntry) GetCompressor() (Compressor, error) {
-	switch {
-	case entry.Flags&CompressorNone == CompressorNone:
+	switch entry.Flags & CompressorMask {
+	case CompressorNone:
 		return CompressorNone, nil
-	case entry.Flags&CompressorZstd == CompressorZstd:
+	case CompressorZstd:
 		return CompressorZstd, nil
+	case CompressorLz4Block:
+		return CompressorLz4Block, nil
 	}
 	return 0, fmt.Errorf("unsupported compressor, entry flags %x", entry.Flags)
 }

vendor/github.com/containerd/nydus-snapshotter/pkg/label/label.go

@@ -34,10 +34,18 @@ const (
 	NydusMetaLayer = "containerd.io/snapshot/nydus-bootstrap"
 	// The referenced blob sha256 in format of `sha256:xxx`, set by image builders.
 	NydusRefLayer = "containerd.io/snapshot/nydus-ref"
+	// The blobID of associated layer, also marking the layer as a nydus tarfs, set by the snapshotter
+	NydusTarfsLayer = "containerd.io/snapshot/nydus-tarfs"
+	// Dm-verity information for image block device
+	NydusImageBlockInfo = "containerd.io/snapshot/nydus-image-block"
+	// Dm-verity information for layer block device
+	NydusLayerBlockInfo = "containerd.io/snapshot/nydus-layer-block"
 	// Annotation containing secret to pull images from registry, set by the snapshotter.
 	NydusImagePullSecret = "containerd.io/snapshot/pullsecret"
 	// Annotation containing username to pull images from registry, set by the snapshotter.
 	NydusImagePullUsername = "containerd.io/snapshot/pullusername"
+	// Proxy image pull actions to other agents.
+	NydusProxyMode = "containerd.io/snapshot/nydus-proxy-mode"
 	// A bool flag to enable integrity verification of meta data blob
 	NydusSignature = "containerd.io/snapshot/nydus-signature"
 
@@ -48,6 +56,10 @@ const (
 	// If this optional label of a snapshot is specified, when mounted to rootdir
 	// this snapshot will include volatile option
 	OverlayfsVolatileOpt = "containerd.io/snapshot/overlay.volatile"
+
+	// A bool flag to mark it is recommended to run this image with tarfs mode, set by image builders.
+	// runtime can decide whether to rely on this annotation
+	TarfsHint = "containerd.io/snapshot/tarfs-hint"
 )
 
 func IsNydusDataLayer(labels map[string]string) bool {
@@ -56,9 +68,21 @@ func IsNydusDataLayer(labels map[string]string) bool {
 }
 
 func IsNydusMetaLayer(labels map[string]string) bool {
-	if labels == nil {
-		return false
-	}
 	_, ok := labels[NydusMetaLayer]
 	return ok
 }
+
+func IsTarfsDataLayer(labels map[string]string) bool {
+	_, ok := labels[NydusTarfsLayer]
+	return ok
+}
+
+func IsNydusProxyMode(labels map[string]string) bool {
+	_, ok := labels[NydusProxyMode]
+	return ok
+}
+
+func HasTarfsHint(labels map[string]string) bool {
+	_, ok := labels[TarfsHint]
+	return ok
+}

vendor/modules.txt

@@ -365,7 +365,7 @@ github.com/containerd/go-runc
 ## explicit; go 1.20
 github.com/containerd/log
 github.com/containerd/log/logtest
-# github.com/containerd/nydus-snapshotter v0.8.2
+# github.com/containerd/nydus-snapshotter v0.13.7
 ## explicit; go 1.19
 github.com/containerd/nydus-snapshotter/pkg/converter
 github.com/containerd/nydus-snapshotter/pkg/converter/tool