pull_v2.go 28 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941
  1. package distribution // import "github.com/docker/docker/distribution"
  2. import (
  3. "context"
  4. "encoding/json"
  5. "fmt"
  6. "io"
  7. "io/ioutil"
  8. "net/url"
  9. "os"
  10. "runtime"
  11. "strings"
  12. "github.com/docker/distribution"
  13. "github.com/docker/distribution/manifest/manifestlist"
  14. "github.com/docker/distribution/manifest/schema1"
  15. "github.com/docker/distribution/manifest/schema2"
  16. "github.com/docker/distribution/reference"
  17. "github.com/docker/distribution/registry/api/errcode"
  18. "github.com/docker/distribution/registry/client/auth"
  19. "github.com/docker/distribution/registry/client/transport"
  20. "github.com/docker/docker/distribution/metadata"
  21. "github.com/docker/docker/distribution/xfer"
  22. "github.com/docker/docker/image"
  23. "github.com/docker/docker/image/v1"
  24. "github.com/docker/docker/layer"
  25. "github.com/docker/docker/pkg/ioutils"
  26. "github.com/docker/docker/pkg/progress"
  27. "github.com/docker/docker/pkg/stringid"
  28. "github.com/docker/docker/pkg/system"
  29. refstore "github.com/docker/docker/reference"
  30. "github.com/docker/docker/registry"
  31. "github.com/opencontainers/go-digest"
  32. specs "github.com/opencontainers/image-spec/specs-go/v1"
  33. "github.com/pkg/errors"
  34. "github.com/sirupsen/logrus"
  35. )
  36. var (
  37. errRootFSMismatch = errors.New("layers from manifest don't match image configuration")
  38. errRootFSInvalid = errors.New("invalid rootfs in image configuration")
  39. )
  40. // ImageConfigPullError is an error pulling the image config blob
  41. // (only applies to schema2).
  42. type ImageConfigPullError struct {
  43. Err error
  44. }
  45. // Error returns the error string for ImageConfigPullError.
  46. func (e ImageConfigPullError) Error() string {
  47. return "error pulling image configuration: " + e.Err.Error()
  48. }
  49. type v2Puller struct {
  50. V2MetadataService metadata.V2MetadataService
  51. endpoint registry.APIEndpoint
  52. config *ImagePullConfig
  53. repoInfo *registry.RepositoryInfo
  54. repo distribution.Repository
  55. // confirmedV2 is set to true if we confirm we're talking to a v2
  56. // registry. This is used to limit fallbacks to the v1 protocol.
  57. confirmedV2 bool
  58. }
  59. func (p *v2Puller) Pull(ctx context.Context, ref reference.Named, os string) (err error) {
  60. // TODO(tiborvass): was ReceiveTimeout
  61. p.repo, p.confirmedV2, err = NewV2Repository(ctx, p.repoInfo, p.endpoint, p.config.MetaHeaders, p.config.AuthConfig, "pull")
  62. if err != nil {
  63. logrus.Warnf("Error getting v2 registry: %v", err)
  64. return err
  65. }
  66. if err = p.pullV2Repository(ctx, ref, os); err != nil {
  67. if _, ok := err.(fallbackError); ok {
  68. return err
  69. }
  70. if continueOnError(err, p.endpoint.Mirror) {
  71. return fallbackError{
  72. err: err,
  73. confirmedV2: p.confirmedV2,
  74. transportOK: true,
  75. }
  76. }
  77. }
  78. return err
  79. }
  80. func (p *v2Puller) pullV2Repository(ctx context.Context, ref reference.Named, os string) (err error) {
  81. var layersDownloaded bool
  82. if !reference.IsNameOnly(ref) {
  83. layersDownloaded, err = p.pullV2Tag(ctx, ref, os)
  84. if err != nil {
  85. return err
  86. }
  87. } else {
  88. tags, err := p.repo.Tags(ctx).All(ctx)
  89. if err != nil {
  90. // If this repository doesn't exist on V2, we should
  91. // permit a fallback to V1.
  92. return allowV1Fallback(err)
  93. }
  94. // The v2 registry knows about this repository, so we will not
  95. // allow fallback to the v1 protocol even if we encounter an
  96. // error later on.
  97. p.confirmedV2 = true
  98. for _, tag := range tags {
  99. tagRef, err := reference.WithTag(ref, tag)
  100. if err != nil {
  101. return err
  102. }
  103. pulledNew, err := p.pullV2Tag(ctx, tagRef, os)
  104. if err != nil {
  105. // Since this is the pull-all-tags case, don't
  106. // allow an error pulling a particular tag to
  107. // make the whole pull fall back to v1.
  108. if fallbackErr, ok := err.(fallbackError); ok {
  109. return fallbackErr.err
  110. }
  111. return err
  112. }
  113. // pulledNew is true if either new layers were downloaded OR if existing images were newly tagged
  114. // TODO(tiborvass): should we change the name of `layersDownload`? What about message in WriteStatus?
  115. layersDownloaded = layersDownloaded || pulledNew
  116. }
  117. }
  118. writeStatus(reference.FamiliarString(ref), p.config.ProgressOutput, layersDownloaded)
  119. return nil
  120. }
  121. type v2LayerDescriptor struct {
  122. digest digest.Digest
  123. diffID layer.DiffID
  124. repoInfo *registry.RepositoryInfo
  125. repo distribution.Repository
  126. V2MetadataService metadata.V2MetadataService
  127. tmpFile *os.File
  128. verifier digest.Verifier
  129. src distribution.Descriptor
  130. }
  131. func (ld *v2LayerDescriptor) Key() string {
  132. return "v2:" + ld.digest.String()
  133. }
  134. func (ld *v2LayerDescriptor) ID() string {
  135. return stringid.TruncateID(ld.digest.String())
  136. }
  137. func (ld *v2LayerDescriptor) DiffID() (layer.DiffID, error) {
  138. if ld.diffID != "" {
  139. return ld.diffID, nil
  140. }
  141. return ld.V2MetadataService.GetDiffID(ld.digest)
  142. }
  143. func (ld *v2LayerDescriptor) Download(ctx context.Context, progressOutput progress.Output) (io.ReadCloser, int64, error) {
  144. logrus.Debugf("pulling blob %q", ld.digest)
  145. var (
  146. err error
  147. offset int64
  148. )
  149. if ld.tmpFile == nil {
  150. ld.tmpFile, err = createDownloadFile()
  151. if err != nil {
  152. return nil, 0, xfer.DoNotRetry{Err: err}
  153. }
  154. } else {
  155. offset, err = ld.tmpFile.Seek(0, os.SEEK_END)
  156. if err != nil {
  157. logrus.Debugf("error seeking to end of download file: %v", err)
  158. offset = 0
  159. ld.tmpFile.Close()
  160. if err := os.Remove(ld.tmpFile.Name()); err != nil {
  161. logrus.Errorf("Failed to remove temp file: %s", ld.tmpFile.Name())
  162. }
  163. ld.tmpFile, err = createDownloadFile()
  164. if err != nil {
  165. return nil, 0, xfer.DoNotRetry{Err: err}
  166. }
  167. } else if offset != 0 {
  168. logrus.Debugf("attempting to resume download of %q from %d bytes", ld.digest, offset)
  169. }
  170. }
  171. tmpFile := ld.tmpFile
  172. layerDownload, err := ld.open(ctx)
  173. if err != nil {
  174. logrus.Errorf("Error initiating layer download: %v", err)
  175. return nil, 0, retryOnError(err)
  176. }
  177. if offset != 0 {
  178. _, err := layerDownload.Seek(offset, os.SEEK_SET)
  179. if err != nil {
  180. if err := ld.truncateDownloadFile(); err != nil {
  181. return nil, 0, xfer.DoNotRetry{Err: err}
  182. }
  183. return nil, 0, err
  184. }
  185. }
  186. size, err := layerDownload.Seek(0, os.SEEK_END)
  187. if err != nil {
  188. // Seek failed, perhaps because there was no Content-Length
  189. // header. This shouldn't fail the download, because we can
  190. // still continue without a progress bar.
  191. size = 0
  192. } else {
  193. if size != 0 && offset > size {
  194. logrus.Debug("Partial download is larger than full blob. Starting over")
  195. offset = 0
  196. if err := ld.truncateDownloadFile(); err != nil {
  197. return nil, 0, xfer.DoNotRetry{Err: err}
  198. }
  199. }
  200. // Restore the seek offset either at the beginning of the
  201. // stream, or just after the last byte we have from previous
  202. // attempts.
  203. _, err = layerDownload.Seek(offset, os.SEEK_SET)
  204. if err != nil {
  205. return nil, 0, err
  206. }
  207. }
  208. reader := progress.NewProgressReader(ioutils.NewCancelReadCloser(ctx, layerDownload), progressOutput, size-offset, ld.ID(), "Downloading")
  209. defer reader.Close()
  210. if ld.verifier == nil {
  211. ld.verifier = ld.digest.Verifier()
  212. }
  213. _, err = io.Copy(tmpFile, io.TeeReader(reader, ld.verifier))
  214. if err != nil {
  215. if err == transport.ErrWrongCodeForByteRange {
  216. if err := ld.truncateDownloadFile(); err != nil {
  217. return nil, 0, xfer.DoNotRetry{Err: err}
  218. }
  219. return nil, 0, err
  220. }
  221. return nil, 0, retryOnError(err)
  222. }
  223. progress.Update(progressOutput, ld.ID(), "Verifying Checksum")
  224. if !ld.verifier.Verified() {
  225. err = fmt.Errorf("filesystem layer verification failed for digest %s", ld.digest)
  226. logrus.Error(err)
  227. // Allow a retry if this digest verification error happened
  228. // after a resumed download.
  229. if offset != 0 {
  230. if err := ld.truncateDownloadFile(); err != nil {
  231. return nil, 0, xfer.DoNotRetry{Err: err}
  232. }
  233. return nil, 0, err
  234. }
  235. return nil, 0, xfer.DoNotRetry{Err: err}
  236. }
  237. progress.Update(progressOutput, ld.ID(), "Download complete")
  238. logrus.Debugf("Downloaded %s to tempfile %s", ld.ID(), tmpFile.Name())
  239. _, err = tmpFile.Seek(0, os.SEEK_SET)
  240. if err != nil {
  241. tmpFile.Close()
  242. if err := os.Remove(tmpFile.Name()); err != nil {
  243. logrus.Errorf("Failed to remove temp file: %s", tmpFile.Name())
  244. }
  245. ld.tmpFile = nil
  246. ld.verifier = nil
  247. return nil, 0, xfer.DoNotRetry{Err: err}
  248. }
  249. // hand off the temporary file to the download manager, so it will only
  250. // be closed once
  251. ld.tmpFile = nil
  252. return ioutils.NewReadCloserWrapper(tmpFile, func() error {
  253. tmpFile.Close()
  254. err := os.RemoveAll(tmpFile.Name())
  255. if err != nil {
  256. logrus.Errorf("Failed to remove temp file: %s", tmpFile.Name())
  257. }
  258. return err
  259. }), size, nil
  260. }
  261. func (ld *v2LayerDescriptor) Close() {
  262. if ld.tmpFile != nil {
  263. ld.tmpFile.Close()
  264. if err := os.RemoveAll(ld.tmpFile.Name()); err != nil {
  265. logrus.Errorf("Failed to remove temp file: %s", ld.tmpFile.Name())
  266. }
  267. }
  268. }
  269. func (ld *v2LayerDescriptor) truncateDownloadFile() error {
  270. // Need a new hash context since we will be redoing the download
  271. ld.verifier = nil
  272. if _, err := ld.tmpFile.Seek(0, os.SEEK_SET); err != nil {
  273. logrus.Errorf("error seeking to beginning of download file: %v", err)
  274. return err
  275. }
  276. if err := ld.tmpFile.Truncate(0); err != nil {
  277. logrus.Errorf("error truncating download file: %v", err)
  278. return err
  279. }
  280. return nil
  281. }
  282. func (ld *v2LayerDescriptor) Registered(diffID layer.DiffID) {
  283. // Cache mapping from this layer's DiffID to the blobsum
  284. ld.V2MetadataService.Add(diffID, metadata.V2Metadata{Digest: ld.digest, SourceRepository: ld.repoInfo.Name.Name()})
  285. }
  286. func (p *v2Puller) pullV2Tag(ctx context.Context, ref reference.Named, os string) (tagUpdated bool, err error) {
  287. manSvc, err := p.repo.Manifests(ctx)
  288. if err != nil {
  289. return false, err
  290. }
  291. var (
  292. manifest distribution.Manifest
  293. tagOrDigest string // Used for logging/progress only
  294. )
  295. if digested, isDigested := ref.(reference.Canonical); isDigested {
  296. manifest, err = manSvc.Get(ctx, digested.Digest())
  297. if err != nil {
  298. return false, err
  299. }
  300. tagOrDigest = digested.Digest().String()
  301. } else if tagged, isTagged := ref.(reference.NamedTagged); isTagged {
  302. manifest, err = manSvc.Get(ctx, "", distribution.WithTag(tagged.Tag()))
  303. if err != nil {
  304. return false, allowV1Fallback(err)
  305. }
  306. tagOrDigest = tagged.Tag()
  307. } else {
  308. return false, fmt.Errorf("internal error: reference has neither a tag nor a digest: %s", reference.FamiliarString(ref))
  309. }
  310. if manifest == nil {
  311. return false, fmt.Errorf("image manifest does not exist for tag or digest %q", tagOrDigest)
  312. }
  313. if m, ok := manifest.(*schema2.DeserializedManifest); ok {
  314. var allowedMediatype bool
  315. for _, t := range p.config.Schema2Types {
  316. if m.Manifest.Config.MediaType == t {
  317. allowedMediatype = true
  318. break
  319. }
  320. }
  321. if !allowedMediatype {
  322. configClass := mediaTypeClasses[m.Manifest.Config.MediaType]
  323. if configClass == "" {
  324. configClass = "unknown"
  325. }
  326. return false, invalidManifestClassError{m.Manifest.Config.MediaType, configClass}
  327. }
  328. }
  329. // If manSvc.Get succeeded, we can be confident that the registry on
  330. // the other side speaks the v2 protocol.
  331. p.confirmedV2 = true
  332. logrus.Debugf("Pulling ref from V2 registry: %s", reference.FamiliarString(ref))
  333. progress.Message(p.config.ProgressOutput, tagOrDigest, "Pulling from "+reference.FamiliarName(p.repo.Named()))
  334. var (
  335. id digest.Digest
  336. manifestDigest digest.Digest
  337. )
  338. switch v := manifest.(type) {
  339. case *schema1.SignedManifest:
  340. if p.config.RequireSchema2 {
  341. return false, fmt.Errorf("invalid manifest: not schema2")
  342. }
  343. id, manifestDigest, err = p.pullSchema1(ctx, ref, v, os)
  344. if err != nil {
  345. return false, err
  346. }
  347. case *schema2.DeserializedManifest:
  348. id, manifestDigest, err = p.pullSchema2(ctx, ref, v, os)
  349. if err != nil {
  350. return false, err
  351. }
  352. case *manifestlist.DeserializedManifestList:
  353. id, manifestDigest, err = p.pullManifestList(ctx, ref, v, os)
  354. if err != nil {
  355. return false, err
  356. }
  357. default:
  358. return false, invalidManifestFormatError{}
  359. }
  360. progress.Message(p.config.ProgressOutput, "", "Digest: "+manifestDigest.String())
  361. if p.config.ReferenceStore != nil {
  362. oldTagID, err := p.config.ReferenceStore.Get(ref)
  363. if err == nil {
  364. if oldTagID == id {
  365. return false, addDigestReference(p.config.ReferenceStore, ref, manifestDigest, id)
  366. }
  367. } else if err != refstore.ErrDoesNotExist {
  368. return false, err
  369. }
  370. if canonical, ok := ref.(reference.Canonical); ok {
  371. if err = p.config.ReferenceStore.AddDigest(canonical, id, true); err != nil {
  372. return false, err
  373. }
  374. } else {
  375. if err = addDigestReference(p.config.ReferenceStore, ref, manifestDigest, id); err != nil {
  376. return false, err
  377. }
  378. if err = p.config.ReferenceStore.AddTag(ref, id, true); err != nil {
  379. return false, err
  380. }
  381. }
  382. }
  383. return true, nil
  384. }
  385. func (p *v2Puller) pullSchema1(ctx context.Context, ref reference.Reference, unverifiedManifest *schema1.SignedManifest, requestedOS string) (id digest.Digest, manifestDigest digest.Digest, err error) {
  386. var verifiedManifest *schema1.Manifest
  387. verifiedManifest, err = verifySchema1Manifest(unverifiedManifest, ref)
  388. if err != nil {
  389. return "", "", err
  390. }
  391. rootFS := image.NewRootFS()
  392. // remove duplicate layers and check parent chain validity
  393. err = fixManifestLayers(verifiedManifest)
  394. if err != nil {
  395. return "", "", err
  396. }
  397. var descriptors []xfer.DownloadDescriptor
  398. // Image history converted to the new format
  399. var history []image.History
  400. // Note that the order of this loop is in the direction of bottom-most
  401. // to top-most, so that the downloads slice gets ordered correctly.
  402. for i := len(verifiedManifest.FSLayers) - 1; i >= 0; i-- {
  403. blobSum := verifiedManifest.FSLayers[i].BlobSum
  404. var throwAway struct {
  405. ThrowAway bool `json:"throwaway,omitempty"`
  406. }
  407. if err := json.Unmarshal([]byte(verifiedManifest.History[i].V1Compatibility), &throwAway); err != nil {
  408. return "", "", err
  409. }
  410. h, err := v1.HistoryFromConfig([]byte(verifiedManifest.History[i].V1Compatibility), throwAway.ThrowAway)
  411. if err != nil {
  412. return "", "", err
  413. }
  414. history = append(history, h)
  415. if throwAway.ThrowAway {
  416. continue
  417. }
  418. layerDescriptor := &v2LayerDescriptor{
  419. digest: blobSum,
  420. repoInfo: p.repoInfo,
  421. repo: p.repo,
  422. V2MetadataService: p.V2MetadataService,
  423. }
  424. descriptors = append(descriptors, layerDescriptor)
  425. }
  426. // The v1 manifest itself doesn't directly contain an OS. However,
  427. // the history does, but unfortunately that's a string, so search through
  428. // all the history until hopefully we find one which indicates the OS.
  429. // supertest2014/nyan is an example of a registry image with schemav1.
  430. configOS := runtime.GOOS
  431. if system.LCOWSupported() {
  432. type config struct {
  433. Os string `json:"os,omitempty"`
  434. }
  435. for _, v := range verifiedManifest.History {
  436. var c config
  437. if err := json.Unmarshal([]byte(v.V1Compatibility), &c); err == nil {
  438. if c.Os != "" {
  439. configOS = c.Os
  440. break
  441. }
  442. }
  443. }
  444. }
  445. // Early bath if the requested OS doesn't match that of the configuration.
  446. // This avoids doing the download, only to potentially fail later.
  447. if !strings.EqualFold(configOS, requestedOS) {
  448. return "", "", fmt.Errorf("cannot download image with operating system %q when requesting %q", configOS, requestedOS)
  449. }
  450. resultRootFS, release, err := p.config.DownloadManager.Download(ctx, *rootFS, configOS, descriptors, p.config.ProgressOutput)
  451. if err != nil {
  452. return "", "", err
  453. }
  454. defer release()
  455. config, err := v1.MakeConfigFromV1Config([]byte(verifiedManifest.History[0].V1Compatibility), &resultRootFS, history)
  456. if err != nil {
  457. return "", "", err
  458. }
  459. imageID, err := p.config.ImageStore.Put(config)
  460. if err != nil {
  461. return "", "", err
  462. }
  463. manifestDigest = digest.FromBytes(unverifiedManifest.Canonical)
  464. return imageID, manifestDigest, nil
  465. }
  466. func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *schema2.DeserializedManifest, requestedOS string) (id digest.Digest, manifestDigest digest.Digest, err error) {
  467. manifestDigest, err = schema2ManifestDigest(ref, mfst)
  468. if err != nil {
  469. return "", "", err
  470. }
  471. target := mfst.Target()
  472. if _, err := p.config.ImageStore.Get(target.Digest); err == nil {
  473. // If the image already exists locally, no need to pull
  474. // anything.
  475. return target.Digest, manifestDigest, nil
  476. }
  477. var descriptors []xfer.DownloadDescriptor
  478. // Note that the order of this loop is in the direction of bottom-most
  479. // to top-most, so that the downloads slice gets ordered correctly.
  480. for _, d := range mfst.Layers {
  481. layerDescriptor := &v2LayerDescriptor{
  482. digest: d.Digest,
  483. repo: p.repo,
  484. repoInfo: p.repoInfo,
  485. V2MetadataService: p.V2MetadataService,
  486. src: d,
  487. }
  488. descriptors = append(descriptors, layerDescriptor)
  489. }
  490. configChan := make(chan []byte, 1)
  491. configErrChan := make(chan error, 1)
  492. layerErrChan := make(chan error, 1)
  493. downloadsDone := make(chan struct{})
  494. var cancel func()
  495. ctx, cancel = context.WithCancel(ctx)
  496. defer cancel()
  497. // Pull the image config
  498. go func() {
  499. configJSON, err := p.pullSchema2Config(ctx, target.Digest)
  500. if err != nil {
  501. configErrChan <- ImageConfigPullError{Err: err}
  502. cancel()
  503. return
  504. }
  505. configChan <- configJSON
  506. }()
  507. var (
  508. configJSON []byte // raw serialized image config
  509. downloadedRootFS *image.RootFS // rootFS from registered layers
  510. configRootFS *image.RootFS // rootFS from configuration
  511. release func() // release resources from rootFS download
  512. configPlatform *specs.Platform // for LCOW when registering downloaded layers
  513. )
  514. // https://github.com/docker/docker/issues/24766 - Err on the side of caution,
  515. // explicitly blocking images intended for linux from the Windows daemon. On
  516. // Windows, we do this before the attempt to download, effectively serialising
  517. // the download slightly slowing it down. We have to do it this way, as
  518. // chances are the download of layers itself would fail due to file names
  519. // which aren't suitable for NTFS. At some point in the future, if a similar
  520. // check to block Windows images being pulled on Linux is implemented, it
  521. // may be necessary to perform the same type of serialisation.
  522. if runtime.GOOS == "windows" {
  523. configJSON, configRootFS, configPlatform, err = receiveConfig(p.config.ImageStore, configChan, configErrChan)
  524. if err != nil {
  525. return "", "", err
  526. }
  527. if configRootFS == nil {
  528. return "", "", errRootFSInvalid
  529. }
  530. if err := checkImageCompatibility(configPlatform.OS, configPlatform.OSVersion); err != nil {
  531. return "", "", err
  532. }
  533. if len(descriptors) != len(configRootFS.DiffIDs) {
  534. return "", "", errRootFSMismatch
  535. }
  536. // Early bath if the requested OS doesn't match that of the configuration.
  537. // This avoids doing the download, only to potentially fail later.
  538. if !strings.EqualFold(configPlatform.OS, requestedOS) {
  539. return "", "", fmt.Errorf("cannot download image with operating system %q when requesting %q", configPlatform.OS, requestedOS)
  540. }
  541. // Populate diff ids in descriptors to avoid downloading foreign layers
  542. // which have been side loaded
  543. for i := range descriptors {
  544. descriptors[i].(*v2LayerDescriptor).diffID = configRootFS.DiffIDs[i]
  545. }
  546. }
  547. if p.config.DownloadManager != nil {
  548. go func() {
  549. var (
  550. err error
  551. rootFS image.RootFS
  552. )
  553. downloadRootFS := *image.NewRootFS()
  554. rootFS, release, err = p.config.DownloadManager.Download(ctx, downloadRootFS, requestedOS, descriptors, p.config.ProgressOutput)
  555. if err != nil {
  556. // Intentionally do not cancel the config download here
  557. // as the error from config download (if there is one)
  558. // is more interesting than the layer download error
  559. layerErrChan <- err
  560. return
  561. }
  562. downloadedRootFS = &rootFS
  563. close(downloadsDone)
  564. }()
  565. } else {
  566. // We have nothing to download
  567. close(downloadsDone)
  568. }
  569. if configJSON == nil {
  570. configJSON, configRootFS, _, err = receiveConfig(p.config.ImageStore, configChan, configErrChan)
  571. if err == nil && configRootFS == nil {
  572. err = errRootFSInvalid
  573. }
  574. if err != nil {
  575. cancel()
  576. select {
  577. case <-downloadsDone:
  578. case <-layerErrChan:
  579. }
  580. return "", "", err
  581. }
  582. }
  583. select {
  584. case <-downloadsDone:
  585. case err = <-layerErrChan:
  586. return "", "", err
  587. }
  588. if release != nil {
  589. defer release()
  590. }
  591. if downloadedRootFS != nil {
  592. // The DiffIDs returned in rootFS MUST match those in the config.
  593. // Otherwise the image config could be referencing layers that aren't
  594. // included in the manifest.
  595. if len(downloadedRootFS.DiffIDs) != len(configRootFS.DiffIDs) {
  596. return "", "", errRootFSMismatch
  597. }
  598. for i := range downloadedRootFS.DiffIDs {
  599. if downloadedRootFS.DiffIDs[i] != configRootFS.DiffIDs[i] {
  600. return "", "", errRootFSMismatch
  601. }
  602. }
  603. }
  604. imageID, err := p.config.ImageStore.Put(configJSON)
  605. if err != nil {
  606. return "", "", err
  607. }
  608. return imageID, manifestDigest, nil
  609. }
  610. func receiveConfig(s ImageConfigStore, configChan <-chan []byte, errChan <-chan error) ([]byte, *image.RootFS, *specs.Platform, error) {
  611. select {
  612. case configJSON := <-configChan:
  613. rootfs, err := s.RootFSFromConfig(configJSON)
  614. if err != nil {
  615. return nil, nil, nil, err
  616. }
  617. platform, err := s.PlatformFromConfig(configJSON)
  618. if err != nil {
  619. return nil, nil, nil, err
  620. }
  621. return configJSON, rootfs, platform, nil
  622. case err := <-errChan:
  623. return nil, nil, nil, err
  624. // Don't need a case for ctx.Done in the select because cancellation
  625. // will trigger an error in p.pullSchema2ImageConfig.
  626. }
  627. }
  628. // pullManifestList handles "manifest lists" which point to various
  629. // platform-specific manifests.
  630. func (p *v2Puller) pullManifestList(ctx context.Context, ref reference.Named, mfstList *manifestlist.DeserializedManifestList, os string) (id digest.Digest, manifestListDigest digest.Digest, err error) {
  631. manifestListDigest, err = schema2ManifestDigest(ref, mfstList)
  632. if err != nil {
  633. return "", "", err
  634. }
  635. logrus.Debugf("%s resolved to a manifestList object with %d entries; looking for a %s/%s match", ref, len(mfstList.Manifests), os, runtime.GOARCH)
  636. manifestMatches := filterManifests(mfstList.Manifests, os)
  637. if len(manifestMatches) == 0 {
  638. errMsg := fmt.Sprintf("no matching manifest for %s/%s in the manifest list entries", os, runtime.GOARCH)
  639. logrus.Debugf(errMsg)
  640. return "", "", errors.New(errMsg)
  641. }
  642. if len(manifestMatches) > 1 {
  643. logrus.Debugf("found multiple matches in manifest list, choosing best match %s", manifestMatches[0].Digest.String())
  644. }
  645. manifestDigest := manifestMatches[0].Digest
  646. if err := checkImageCompatibility(manifestMatches[0].Platform.OS, manifestMatches[0].Platform.OSVersion); err != nil {
  647. return "", "", err
  648. }
  649. manSvc, err := p.repo.Manifests(ctx)
  650. if err != nil {
  651. return "", "", err
  652. }
  653. manifest, err := manSvc.Get(ctx, manifestDigest)
  654. if err != nil {
  655. return "", "", err
  656. }
  657. manifestRef, err := reference.WithDigest(reference.TrimNamed(ref), manifestDigest)
  658. if err != nil {
  659. return "", "", err
  660. }
  661. switch v := manifest.(type) {
  662. case *schema1.SignedManifest:
  663. id, _, err = p.pullSchema1(ctx, manifestRef, v, os)
  664. if err != nil {
  665. return "", "", err
  666. }
  667. case *schema2.DeserializedManifest:
  668. id, _, err = p.pullSchema2(ctx, manifestRef, v, os)
  669. if err != nil {
  670. return "", "", err
  671. }
  672. default:
  673. return "", "", errors.New("unsupported manifest format")
  674. }
  675. return id, manifestListDigest, err
  676. }
  677. func (p *v2Puller) pullSchema2Config(ctx context.Context, dgst digest.Digest) (configJSON []byte, err error) {
  678. blobs := p.repo.Blobs(ctx)
  679. configJSON, err = blobs.Get(ctx, dgst)
  680. if err != nil {
  681. return nil, err
  682. }
  683. // Verify image config digest
  684. verifier := dgst.Verifier()
  685. if _, err := verifier.Write(configJSON); err != nil {
  686. return nil, err
  687. }
  688. if !verifier.Verified() {
  689. err := fmt.Errorf("image config verification failed for digest %s", dgst)
  690. logrus.Error(err)
  691. return nil, err
  692. }
  693. return configJSON, nil
  694. }
  695. // schema2ManifestDigest computes the manifest digest, and, if pulling by
  696. // digest, ensures that it matches the requested digest.
  697. func schema2ManifestDigest(ref reference.Named, mfst distribution.Manifest) (digest.Digest, error) {
  698. _, canonical, err := mfst.Payload()
  699. if err != nil {
  700. return "", err
  701. }
  702. // If pull by digest, then verify the manifest digest.
  703. if digested, isDigested := ref.(reference.Canonical); isDigested {
  704. verifier := digested.Digest().Verifier()
  705. if _, err := verifier.Write(canonical); err != nil {
  706. return "", err
  707. }
  708. if !verifier.Verified() {
  709. err := fmt.Errorf("manifest verification failed for digest %s", digested.Digest())
  710. logrus.Error(err)
  711. return "", err
  712. }
  713. return digested.Digest(), nil
  714. }
  715. return digest.FromBytes(canonical), nil
  716. }
  717. // allowV1Fallback checks if the error is a possible reason to fallback to v1
  718. // (even if confirmedV2 has been set already), and if so, wraps the error in
  719. // a fallbackError with confirmedV2 set to false. Otherwise, it returns the
  720. // error unmodified.
  721. func allowV1Fallback(err error) error {
  722. switch v := err.(type) {
  723. case errcode.Errors:
  724. if len(v) != 0 {
  725. if v0, ok := v[0].(errcode.Error); ok && shouldV2Fallback(v0) {
  726. return fallbackError{
  727. err: err,
  728. confirmedV2: false,
  729. transportOK: true,
  730. }
  731. }
  732. }
  733. case errcode.Error:
  734. if shouldV2Fallback(v) {
  735. return fallbackError{
  736. err: err,
  737. confirmedV2: false,
  738. transportOK: true,
  739. }
  740. }
  741. case *url.Error:
  742. if v.Err == auth.ErrNoBasicAuthCredentials {
  743. return fallbackError{err: err, confirmedV2: false}
  744. }
  745. }
  746. return err
  747. }
  748. func verifySchema1Manifest(signedManifest *schema1.SignedManifest, ref reference.Reference) (m *schema1.Manifest, err error) {
  749. // If pull by digest, then verify the manifest digest. NOTE: It is
  750. // important to do this first, before any other content validation. If the
  751. // digest cannot be verified, don't even bother with those other things.
  752. if digested, isCanonical := ref.(reference.Canonical); isCanonical {
  753. verifier := digested.Digest().Verifier()
  754. if _, err := verifier.Write(signedManifest.Canonical); err != nil {
  755. return nil, err
  756. }
  757. if !verifier.Verified() {
  758. err := fmt.Errorf("image verification failed for digest %s", digested.Digest())
  759. logrus.Error(err)
  760. return nil, err
  761. }
  762. }
  763. m = &signedManifest.Manifest
  764. if m.SchemaVersion != 1 {
  765. return nil, fmt.Errorf("unsupported schema version %d for %q", m.SchemaVersion, reference.FamiliarString(ref))
  766. }
  767. if len(m.FSLayers) != len(m.History) {
  768. return nil, fmt.Errorf("length of history not equal to number of layers for %q", reference.FamiliarString(ref))
  769. }
  770. if len(m.FSLayers) == 0 {
  771. return nil, fmt.Errorf("no FSLayers in manifest for %q", reference.FamiliarString(ref))
  772. }
  773. return m, nil
  774. }
  775. // fixManifestLayers removes repeated layers from the manifest and checks the
  776. // correctness of the parent chain.
  777. func fixManifestLayers(m *schema1.Manifest) error {
  778. imgs := make([]*image.V1Image, len(m.FSLayers))
  779. for i := range m.FSLayers {
  780. img := &image.V1Image{}
  781. if err := json.Unmarshal([]byte(m.History[i].V1Compatibility), img); err != nil {
  782. return err
  783. }
  784. imgs[i] = img
  785. if err := v1.ValidateID(img.ID); err != nil {
  786. return err
  787. }
  788. }
  789. if imgs[len(imgs)-1].Parent != "" && runtime.GOOS != "windows" {
  790. // Windows base layer can point to a base layer parent that is not in manifest.
  791. return errors.New("invalid parent ID in the base layer of the image")
  792. }
  793. // check general duplicates to error instead of a deadlock
  794. idmap := make(map[string]struct{})
  795. var lastID string
  796. for _, img := range imgs {
  797. // skip IDs that appear after each other, we handle those later
  798. if _, exists := idmap[img.ID]; img.ID != lastID && exists {
  799. return fmt.Errorf("ID %+v appears multiple times in manifest", img.ID)
  800. }
  801. lastID = img.ID
  802. idmap[lastID] = struct{}{}
  803. }
  804. // backwards loop so that we keep the remaining indexes after removing items
  805. for i := len(imgs) - 2; i >= 0; i-- {
  806. if imgs[i].ID == imgs[i+1].ID { // repeated ID. remove and continue
  807. m.FSLayers = append(m.FSLayers[:i], m.FSLayers[i+1:]...)
  808. m.History = append(m.History[:i], m.History[i+1:]...)
  809. } else if imgs[i].Parent != imgs[i+1].ID {
  810. return fmt.Errorf("invalid parent ID. Expected %v, got %v", imgs[i+1].ID, imgs[i].Parent)
  811. }
  812. }
  813. return nil
  814. }
  815. func createDownloadFile() (*os.File, error) {
  816. return ioutil.TempFile("", "GetImageBlob")
  817. }