pull_v2.go 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899
  1. package distribution
  2. import (
  3. "encoding/json"
  4. "errors"
  5. "fmt"
  6. "io"
  7. "io/ioutil"
  8. "net/url"
  9. "os"
  10. "runtime"
  11. "github.com/Sirupsen/logrus"
  12. "github.com/docker/distribution"
  13. "github.com/docker/distribution/manifest/manifestlist"
  14. "github.com/docker/distribution/manifest/schema1"
  15. "github.com/docker/distribution/manifest/schema2"
  16. "github.com/docker/distribution/reference"
  17. "github.com/docker/distribution/registry/api/errcode"
  18. "github.com/docker/distribution/registry/client/auth"
  19. "github.com/docker/distribution/registry/client/transport"
  20. "github.com/docker/docker/distribution/metadata"
  21. "github.com/docker/docker/distribution/xfer"
  22. "github.com/docker/docker/image"
  23. "github.com/docker/docker/image/v1"
  24. "github.com/docker/docker/layer"
  25. "github.com/docker/docker/pkg/ioutils"
  26. "github.com/docker/docker/pkg/progress"
  27. "github.com/docker/docker/pkg/stringid"
  28. refstore "github.com/docker/docker/reference"
  29. "github.com/docker/docker/registry"
  30. "github.com/opencontainers/go-digest"
  31. "golang.org/x/net/context"
  32. )
  33. var (
  34. errRootFSMismatch = errors.New("layers from manifest don't match image configuration")
  35. errRootFSInvalid = errors.New("invalid rootfs in image configuration")
  36. )
  37. // ImageConfigPullError is an error pulling the image config blob
  38. // (only applies to schema2).
  39. type ImageConfigPullError struct {
  40. Err error
  41. }
  42. // Error returns the error string for ImageConfigPullError.
  43. func (e ImageConfigPullError) Error() string {
  44. return "error pulling image configuration: " + e.Err.Error()
  45. }
  46. type v2Puller struct {
  47. V2MetadataService metadata.V2MetadataService
  48. endpoint registry.APIEndpoint
  49. config *ImagePullConfig
  50. repoInfo *registry.RepositoryInfo
  51. repo distribution.Repository
  52. // confirmedV2 is set to true if we confirm we're talking to a v2
  53. // registry. This is used to limit fallbacks to the v1 protocol.
  54. confirmedV2 bool
  55. }
  56. func (p *v2Puller) Pull(ctx context.Context, ref reference.Named) (err error) {
  57. // TODO(tiborvass): was ReceiveTimeout
  58. p.repo, p.confirmedV2, err = NewV2Repository(ctx, p.repoInfo, p.endpoint, p.config.MetaHeaders, p.config.AuthConfig, "pull")
  59. if err != nil {
  60. logrus.Warnf("Error getting v2 registry: %v", err)
  61. return err
  62. }
  63. if err = p.pullV2Repository(ctx, ref); err != nil {
  64. if _, ok := err.(fallbackError); ok {
  65. return err
  66. }
  67. if continueOnError(err) {
  68. return fallbackError{
  69. err: err,
  70. confirmedV2: p.confirmedV2,
  71. transportOK: true,
  72. }
  73. }
  74. }
  75. return err
  76. }
  77. func (p *v2Puller) pullV2Repository(ctx context.Context, ref reference.Named) (err error) {
  78. var layersDownloaded bool
  79. if !reference.IsNameOnly(ref) {
  80. layersDownloaded, err = p.pullV2Tag(ctx, ref)
  81. if err != nil {
  82. return err
  83. }
  84. } else {
  85. tags, err := p.repo.Tags(ctx).All(ctx)
  86. if err != nil {
  87. // If this repository doesn't exist on V2, we should
  88. // permit a fallback to V1.
  89. return allowV1Fallback(err)
  90. }
  91. // The v2 registry knows about this repository, so we will not
  92. // allow fallback to the v1 protocol even if we encounter an
  93. // error later on.
  94. p.confirmedV2 = true
  95. for _, tag := range tags {
  96. tagRef, err := reference.WithTag(ref, tag)
  97. if err != nil {
  98. return err
  99. }
  100. pulledNew, err := p.pullV2Tag(ctx, tagRef)
  101. if err != nil {
  102. // Since this is the pull-all-tags case, don't
  103. // allow an error pulling a particular tag to
  104. // make the whole pull fall back to v1.
  105. if fallbackErr, ok := err.(fallbackError); ok {
  106. return fallbackErr.err
  107. }
  108. return err
  109. }
  110. // pulledNew is true if either new layers were downloaded OR if existing images were newly tagged
  111. // TODO(tiborvass): should we change the name of `layersDownload`? What about message in WriteStatus?
  112. layersDownloaded = layersDownloaded || pulledNew
  113. }
  114. }
  115. writeStatus(reference.FamiliarString(ref), p.config.ProgressOutput, layersDownloaded)
  116. return nil
  117. }
  118. type v2LayerDescriptor struct {
  119. digest digest.Digest
  120. diffID layer.DiffID
  121. repoInfo *registry.RepositoryInfo
  122. repo distribution.Repository
  123. V2MetadataService metadata.V2MetadataService
  124. tmpFile *os.File
  125. verifier digest.Verifier
  126. src distribution.Descriptor
  127. }
  128. func (ld *v2LayerDescriptor) Key() string {
  129. return "v2:" + ld.digest.String()
  130. }
  131. func (ld *v2LayerDescriptor) ID() string {
  132. return stringid.TruncateID(ld.digest.String())
  133. }
  134. func (ld *v2LayerDescriptor) DiffID() (layer.DiffID, error) {
  135. if ld.diffID != "" {
  136. return ld.diffID, nil
  137. }
  138. return ld.V2MetadataService.GetDiffID(ld.digest)
  139. }
  140. func (ld *v2LayerDescriptor) Download(ctx context.Context, progressOutput progress.Output) (io.ReadCloser, int64, error) {
  141. logrus.Debugf("pulling blob %q", ld.digest)
  142. var (
  143. err error
  144. offset int64
  145. )
  146. if ld.tmpFile == nil {
  147. ld.tmpFile, err = createDownloadFile()
  148. if err != nil {
  149. return nil, 0, xfer.DoNotRetry{Err: err}
  150. }
  151. } else {
  152. offset, err = ld.tmpFile.Seek(0, os.SEEK_END)
  153. if err != nil {
  154. logrus.Debugf("error seeking to end of download file: %v", err)
  155. offset = 0
  156. ld.tmpFile.Close()
  157. if err := os.Remove(ld.tmpFile.Name()); err != nil {
  158. logrus.Errorf("Failed to remove temp file: %s", ld.tmpFile.Name())
  159. }
  160. ld.tmpFile, err = createDownloadFile()
  161. if err != nil {
  162. return nil, 0, xfer.DoNotRetry{Err: err}
  163. }
  164. } else if offset != 0 {
  165. logrus.Debugf("attempting to resume download of %q from %d bytes", ld.digest, offset)
  166. }
  167. }
  168. tmpFile := ld.tmpFile
  169. layerDownload, err := ld.open(ctx)
  170. if err != nil {
  171. logrus.Errorf("Error initiating layer download: %v", err)
  172. return nil, 0, retryOnError(err)
  173. }
  174. if offset != 0 {
  175. _, err := layerDownload.Seek(offset, os.SEEK_SET)
  176. if err != nil {
  177. if err := ld.truncateDownloadFile(); err != nil {
  178. return nil, 0, xfer.DoNotRetry{Err: err}
  179. }
  180. return nil, 0, err
  181. }
  182. }
  183. size, err := layerDownload.Seek(0, os.SEEK_END)
  184. if err != nil {
  185. // Seek failed, perhaps because there was no Content-Length
  186. // header. This shouldn't fail the download, because we can
  187. // still continue without a progress bar.
  188. size = 0
  189. } else {
  190. if size != 0 && offset > size {
  191. logrus.Debug("Partial download is larger than full blob. Starting over")
  192. offset = 0
  193. if err := ld.truncateDownloadFile(); err != nil {
  194. return nil, 0, xfer.DoNotRetry{Err: err}
  195. }
  196. }
  197. // Restore the seek offset either at the beginning of the
  198. // stream, or just after the last byte we have from previous
  199. // attempts.
  200. _, err = layerDownload.Seek(offset, os.SEEK_SET)
  201. if err != nil {
  202. return nil, 0, err
  203. }
  204. }
  205. reader := progress.NewProgressReader(ioutils.NewCancelReadCloser(ctx, layerDownload), progressOutput, size-offset, ld.ID(), "Downloading")
  206. defer reader.Close()
  207. if ld.verifier == nil {
  208. ld.verifier = ld.digest.Verifier()
  209. }
  210. _, err = io.Copy(tmpFile, io.TeeReader(reader, ld.verifier))
  211. if err != nil {
  212. if err == transport.ErrWrongCodeForByteRange {
  213. if err := ld.truncateDownloadFile(); err != nil {
  214. return nil, 0, xfer.DoNotRetry{Err: err}
  215. }
  216. return nil, 0, err
  217. }
  218. return nil, 0, retryOnError(err)
  219. }
  220. progress.Update(progressOutput, ld.ID(), "Verifying Checksum")
  221. if !ld.verifier.Verified() {
  222. err = fmt.Errorf("filesystem layer verification failed for digest %s", ld.digest)
  223. logrus.Error(err)
  224. // Allow a retry if this digest verification error happened
  225. // after a resumed download.
  226. if offset != 0 {
  227. if err := ld.truncateDownloadFile(); err != nil {
  228. return nil, 0, xfer.DoNotRetry{Err: err}
  229. }
  230. return nil, 0, err
  231. }
  232. return nil, 0, xfer.DoNotRetry{Err: err}
  233. }
  234. progress.Update(progressOutput, ld.ID(), "Download complete")
  235. logrus.Debugf("Downloaded %s to tempfile %s", ld.ID(), tmpFile.Name())
  236. _, err = tmpFile.Seek(0, os.SEEK_SET)
  237. if err != nil {
  238. tmpFile.Close()
  239. if err := os.Remove(tmpFile.Name()); err != nil {
  240. logrus.Errorf("Failed to remove temp file: %s", tmpFile.Name())
  241. }
  242. ld.tmpFile = nil
  243. ld.verifier = nil
  244. return nil, 0, xfer.DoNotRetry{Err: err}
  245. }
  246. // hand off the temporary file to the download manager, so it will only
  247. // be closed once
  248. ld.tmpFile = nil
  249. return ioutils.NewReadCloserWrapper(tmpFile, func() error {
  250. tmpFile.Close()
  251. err := os.RemoveAll(tmpFile.Name())
  252. if err != nil {
  253. logrus.Errorf("Failed to remove temp file: %s", tmpFile.Name())
  254. }
  255. return err
  256. }), size, nil
  257. }
  258. func (ld *v2LayerDescriptor) Close() {
  259. if ld.tmpFile != nil {
  260. ld.tmpFile.Close()
  261. if err := os.RemoveAll(ld.tmpFile.Name()); err != nil {
  262. logrus.Errorf("Failed to remove temp file: %s", ld.tmpFile.Name())
  263. }
  264. }
  265. }
  266. func (ld *v2LayerDescriptor) truncateDownloadFile() error {
  267. // Need a new hash context since we will be redoing the download
  268. ld.verifier = nil
  269. if _, err := ld.tmpFile.Seek(0, os.SEEK_SET); err != nil {
  270. logrus.Errorf("error seeking to beginning of download file: %v", err)
  271. return err
  272. }
  273. if err := ld.tmpFile.Truncate(0); err != nil {
  274. logrus.Errorf("error truncating download file: %v", err)
  275. return err
  276. }
  277. return nil
  278. }
  279. func (ld *v2LayerDescriptor) Registered(diffID layer.DiffID) {
  280. // Cache mapping from this layer's DiffID to the blobsum
  281. ld.V2MetadataService.Add(diffID, metadata.V2Metadata{Digest: ld.digest, SourceRepository: ld.repoInfo.Name.Name()})
  282. }
  283. func (p *v2Puller) pullV2Tag(ctx context.Context, ref reference.Named) (tagUpdated bool, err error) {
  284. manSvc, err := p.repo.Manifests(ctx)
  285. if err != nil {
  286. return false, err
  287. }
  288. var (
  289. manifest distribution.Manifest
  290. tagOrDigest string // Used for logging/progress only
  291. )
  292. if digested, isDigested := ref.(reference.Canonical); isDigested {
  293. manifest, err = manSvc.Get(ctx, digested.Digest())
  294. if err != nil {
  295. return false, err
  296. }
  297. tagOrDigest = digested.Digest().String()
  298. } else if tagged, isTagged := ref.(reference.NamedTagged); isTagged {
  299. manifest, err = manSvc.Get(ctx, "", distribution.WithTag(tagged.Tag()))
  300. if err != nil {
  301. return false, allowV1Fallback(err)
  302. }
  303. tagOrDigest = tagged.Tag()
  304. } else {
  305. return false, fmt.Errorf("internal error: reference has neither a tag nor a digest: %s", reference.FamiliarString(ref))
  306. }
  307. if manifest == nil {
  308. return false, fmt.Errorf("image manifest does not exist for tag or digest %q", tagOrDigest)
  309. }
  310. if m, ok := manifest.(*schema2.DeserializedManifest); ok {
  311. var allowedMediatype bool
  312. for _, t := range p.config.Schema2Types {
  313. if m.Manifest.Config.MediaType == t {
  314. allowedMediatype = true
  315. break
  316. }
  317. }
  318. if !allowedMediatype {
  319. configClass := mediaTypeClasses[m.Manifest.Config.MediaType]
  320. if configClass == "" {
  321. configClass = "unknown"
  322. }
  323. return false, fmt.Errorf("Encountered remote %q(%s) when fetching", m.Manifest.Config.MediaType, configClass)
  324. }
  325. }
  326. // If manSvc.Get succeeded, we can be confident that the registry on
  327. // the other side speaks the v2 protocol.
  328. p.confirmedV2 = true
  329. logrus.Debugf("Pulling ref from V2 registry: %s", reference.FamiliarString(ref))
  330. progress.Message(p.config.ProgressOutput, tagOrDigest, "Pulling from "+reference.FamiliarName(p.repo.Named()))
  331. var (
  332. id digest.Digest
  333. manifestDigest digest.Digest
  334. )
  335. switch v := manifest.(type) {
  336. case *schema1.SignedManifest:
  337. if p.config.RequireSchema2 {
  338. return false, fmt.Errorf("invalid manifest: not schema2")
  339. }
  340. id, manifestDigest, err = p.pullSchema1(ctx, ref, v)
  341. if err != nil {
  342. return false, err
  343. }
  344. case *schema2.DeserializedManifest:
  345. id, manifestDigest, err = p.pullSchema2(ctx, ref, v)
  346. if err != nil {
  347. return false, err
  348. }
  349. case *manifestlist.DeserializedManifestList:
  350. id, manifestDigest, err = p.pullManifestList(ctx, ref, v)
  351. if err != nil {
  352. return false, err
  353. }
  354. default:
  355. return false, errors.New("unsupported manifest format")
  356. }
  357. progress.Message(p.config.ProgressOutput, "", "Digest: "+manifestDigest.String())
  358. if p.config.ReferenceStore != nil {
  359. oldTagID, err := p.config.ReferenceStore.Get(ref)
  360. if err == nil {
  361. if oldTagID == id {
  362. return false, addDigestReference(p.config.ReferenceStore, ref, manifestDigest, id)
  363. }
  364. } else if err != refstore.ErrDoesNotExist {
  365. return false, err
  366. }
  367. if canonical, ok := ref.(reference.Canonical); ok {
  368. if err = p.config.ReferenceStore.AddDigest(canonical, id, true); err != nil {
  369. return false, err
  370. }
  371. } else {
  372. if err = addDigestReference(p.config.ReferenceStore, ref, manifestDigest, id); err != nil {
  373. return false, err
  374. }
  375. if err = p.config.ReferenceStore.AddTag(ref, id, true); err != nil {
  376. return false, err
  377. }
  378. }
  379. }
  380. return true, nil
  381. }
  382. func (p *v2Puller) pullSchema1(ctx context.Context, ref reference.Named, unverifiedManifest *schema1.SignedManifest) (id digest.Digest, manifestDigest digest.Digest, err error) {
  383. var verifiedManifest *schema1.Manifest
  384. verifiedManifest, err = verifySchema1Manifest(unverifiedManifest, ref)
  385. if err != nil {
  386. return "", "", err
  387. }
  388. rootFS := image.NewRootFS()
  389. // remove duplicate layers and check parent chain validity
  390. err = fixManifestLayers(verifiedManifest)
  391. if err != nil {
  392. return "", "", err
  393. }
  394. var descriptors []xfer.DownloadDescriptor
  395. // Image history converted to the new format
  396. var history []image.History
  397. // Note that the order of this loop is in the direction of bottom-most
  398. // to top-most, so that the downloads slice gets ordered correctly.
  399. for i := len(verifiedManifest.FSLayers) - 1; i >= 0; i-- {
  400. blobSum := verifiedManifest.FSLayers[i].BlobSum
  401. var throwAway struct {
  402. ThrowAway bool `json:"throwaway,omitempty"`
  403. }
  404. if err := json.Unmarshal([]byte(verifiedManifest.History[i].V1Compatibility), &throwAway); err != nil {
  405. return "", "", err
  406. }
  407. h, err := v1.HistoryFromConfig([]byte(verifiedManifest.History[i].V1Compatibility), throwAway.ThrowAway)
  408. if err != nil {
  409. return "", "", err
  410. }
  411. history = append(history, h)
  412. if throwAway.ThrowAway {
  413. continue
  414. }
  415. layerDescriptor := &v2LayerDescriptor{
  416. digest: blobSum,
  417. repoInfo: p.repoInfo,
  418. repo: p.repo,
  419. V2MetadataService: p.V2MetadataService,
  420. }
  421. descriptors = append(descriptors, layerDescriptor)
  422. }
  423. resultRootFS, release, err := p.config.DownloadManager.Download(ctx, *rootFS, descriptors, p.config.ProgressOutput)
  424. if err != nil {
  425. return "", "", err
  426. }
  427. defer release()
  428. config, err := v1.MakeConfigFromV1Config([]byte(verifiedManifest.History[0].V1Compatibility), &resultRootFS, history)
  429. if err != nil {
  430. return "", "", err
  431. }
  432. imageID, err := p.config.ImageStore.Put(config)
  433. if err != nil {
  434. return "", "", err
  435. }
  436. manifestDigest = digest.FromBytes(unverifiedManifest.Canonical)
  437. return imageID, manifestDigest, nil
  438. }
  439. func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *schema2.DeserializedManifest) (id digest.Digest, manifestDigest digest.Digest, err error) {
  440. manifestDigest, err = schema2ManifestDigest(ref, mfst)
  441. if err != nil {
  442. return "", "", err
  443. }
  444. target := mfst.Target()
  445. if _, err := p.config.ImageStore.Get(target.Digest); err == nil {
  446. // If the image already exists locally, no need to pull
  447. // anything.
  448. return target.Digest, manifestDigest, nil
  449. }
  450. var descriptors []xfer.DownloadDescriptor
  451. // Note that the order of this loop is in the direction of bottom-most
  452. // to top-most, so that the downloads slice gets ordered correctly.
  453. for _, d := range mfst.Layers {
  454. layerDescriptor := &v2LayerDescriptor{
  455. digest: d.Digest,
  456. repo: p.repo,
  457. repoInfo: p.repoInfo,
  458. V2MetadataService: p.V2MetadataService,
  459. src: d,
  460. }
  461. descriptors = append(descriptors, layerDescriptor)
  462. }
  463. configChan := make(chan []byte, 1)
  464. configErrChan := make(chan error, 1)
  465. layerErrChan := make(chan error, 1)
  466. downloadsDone := make(chan struct{})
  467. var cancel func()
  468. ctx, cancel = context.WithCancel(ctx)
  469. defer cancel()
  470. // Pull the image config
  471. go func() {
  472. configJSON, err := p.pullSchema2Config(ctx, target.Digest)
  473. if err != nil {
  474. configErrChan <- ImageConfigPullError{Err: err}
  475. cancel()
  476. return
  477. }
  478. configChan <- configJSON
  479. }()
  480. var (
  481. configJSON []byte // raw serialized image config
  482. downloadedRootFS *image.RootFS // rootFS from registered layers
  483. configRootFS *image.RootFS // rootFS from configuration
  484. release func() // release resources from rootFS download
  485. )
  486. // https://github.com/docker/docker/issues/24766 - Err on the side of caution,
  487. // explicitly blocking images intended for linux from the Windows daemon. On
  488. // Windows, we do this before the attempt to download, effectively serialising
  489. // the download slightly slowing it down. We have to do it this way, as
  490. // chances are the download of layers itself would fail due to file names
  491. // which aren't suitable for NTFS. At some point in the future, if a similar
  492. // check to block Windows images being pulled on Linux is implemented, it
  493. // may be necessary to perform the same type of serialisation.
  494. if runtime.GOOS == "windows" {
  495. configJSON, configRootFS, err = receiveConfig(p.config.ImageStore, configChan, configErrChan)
  496. if err != nil {
  497. return "", "", err
  498. }
  499. if configRootFS == nil {
  500. return "", "", errRootFSInvalid
  501. }
  502. if len(descriptors) != len(configRootFS.DiffIDs) {
  503. return "", "", errRootFSMismatch
  504. }
  505. // Populate diff ids in descriptors to avoid downloading foreign layers
  506. // which have been side loaded
  507. for i := range descriptors {
  508. descriptors[i].(*v2LayerDescriptor).diffID = configRootFS.DiffIDs[i]
  509. }
  510. }
  511. if p.config.DownloadManager != nil {
  512. go func() {
  513. var (
  514. err error
  515. rootFS image.RootFS
  516. )
  517. downloadRootFS := *image.NewRootFS()
  518. rootFS, release, err = p.config.DownloadManager.Download(ctx, downloadRootFS, descriptors, p.config.ProgressOutput)
  519. if err != nil {
  520. // Intentionally do not cancel the config download here
  521. // as the error from config download (if there is one)
  522. // is more interesting than the layer download error
  523. layerErrChan <- err
  524. return
  525. }
  526. downloadedRootFS = &rootFS
  527. close(downloadsDone)
  528. }()
  529. } else {
  530. // We have nothing to download
  531. close(downloadsDone)
  532. }
  533. if configJSON == nil {
  534. configJSON, configRootFS, err = receiveConfig(p.config.ImageStore, configChan, configErrChan)
  535. if err == nil && configRootFS == nil {
  536. err = errRootFSInvalid
  537. }
  538. if err != nil {
  539. cancel()
  540. select {
  541. case <-downloadsDone:
  542. case <-layerErrChan:
  543. }
  544. return "", "", err
  545. }
  546. }
  547. select {
  548. case <-downloadsDone:
  549. case err = <-layerErrChan:
  550. return "", "", err
  551. }
  552. if release != nil {
  553. defer release()
  554. }
  555. if downloadedRootFS != nil {
  556. // The DiffIDs returned in rootFS MUST match those in the config.
  557. // Otherwise the image config could be referencing layers that aren't
  558. // included in the manifest.
  559. if len(downloadedRootFS.DiffIDs) != len(configRootFS.DiffIDs) {
  560. return "", "", errRootFSMismatch
  561. }
  562. for i := range downloadedRootFS.DiffIDs {
  563. if downloadedRootFS.DiffIDs[i] != configRootFS.DiffIDs[i] {
  564. return "", "", errRootFSMismatch
  565. }
  566. }
  567. }
  568. imageID, err := p.config.ImageStore.Put(configJSON)
  569. if err != nil {
  570. return "", "", err
  571. }
  572. return imageID, manifestDigest, nil
  573. }
  574. func receiveConfig(s ImageConfigStore, configChan <-chan []byte, errChan <-chan error) ([]byte, *image.RootFS, error) {
  575. select {
  576. case configJSON := <-configChan:
  577. rootfs, err := s.RootFSFromConfig(configJSON)
  578. if err != nil {
  579. return nil, nil, err
  580. }
  581. return configJSON, rootfs, nil
  582. case err := <-errChan:
  583. return nil, nil, err
  584. // Don't need a case for ctx.Done in the select because cancellation
  585. // will trigger an error in p.pullSchema2ImageConfig.
  586. }
  587. }
  588. // pullManifestList handles "manifest lists" which point to various
  589. // platform-specific manifests.
  590. func (p *v2Puller) pullManifestList(ctx context.Context, ref reference.Named, mfstList *manifestlist.DeserializedManifestList) (id digest.Digest, manifestListDigest digest.Digest, err error) {
  591. manifestListDigest, err = schema2ManifestDigest(ref, mfstList)
  592. if err != nil {
  593. return "", "", err
  594. }
  595. logrus.Debugf("%s resolved to a manifestList object with %d entries; looking for a os/arch match", ref, len(mfstList.Manifests))
  596. var manifestDigest digest.Digest
  597. for _, manifestDescriptor := range mfstList.Manifests {
  598. // TODO(aaronl): The manifest list spec supports optional
  599. // "features" and "variant" fields. These are not yet used.
  600. // Once they are, their values should be interpreted here.
  601. if manifestDescriptor.Platform.Architecture == runtime.GOARCH && manifestDescriptor.Platform.OS == runtime.GOOS {
  602. manifestDigest = manifestDescriptor.Digest
  603. logrus.Debugf("found match for %s/%s with media type %s, digest %s", runtime.GOOS, runtime.GOARCH, manifestDescriptor.MediaType, manifestDigest.String())
  604. break
  605. }
  606. }
  607. if manifestDigest == "" {
  608. errMsg := fmt.Sprintf("no matching manifest for %s/%s in the manifest list entries", runtime.GOOS, runtime.GOARCH)
  609. logrus.Debugf(errMsg)
  610. return "", "", errors.New(errMsg)
  611. }
  612. manSvc, err := p.repo.Manifests(ctx)
  613. if err != nil {
  614. return "", "", err
  615. }
  616. manifest, err := manSvc.Get(ctx, manifestDigest)
  617. if err != nil {
  618. return "", "", err
  619. }
  620. manifestRef, err := reference.WithDigest(reference.TrimNamed(ref), manifestDigest)
  621. if err != nil {
  622. return "", "", err
  623. }
  624. switch v := manifest.(type) {
  625. case *schema1.SignedManifest:
  626. id, _, err = p.pullSchema1(ctx, manifestRef, v)
  627. if err != nil {
  628. return "", "", err
  629. }
  630. case *schema2.DeserializedManifest:
  631. id, _, err = p.pullSchema2(ctx, manifestRef, v)
  632. if err != nil {
  633. return "", "", err
  634. }
  635. default:
  636. return "", "", errors.New("unsupported manifest format")
  637. }
  638. return id, manifestListDigest, err
  639. }
  640. func (p *v2Puller) pullSchema2Config(ctx context.Context, dgst digest.Digest) (configJSON []byte, err error) {
  641. blobs := p.repo.Blobs(ctx)
  642. configJSON, err = blobs.Get(ctx, dgst)
  643. if err != nil {
  644. return nil, err
  645. }
  646. // Verify image config digest
  647. verifier := dgst.Verifier()
  648. if _, err := verifier.Write(configJSON); err != nil {
  649. return nil, err
  650. }
  651. if !verifier.Verified() {
  652. err := fmt.Errorf("image config verification failed for digest %s", dgst)
  653. logrus.Error(err)
  654. return nil, err
  655. }
  656. return configJSON, nil
  657. }
  658. // schema2ManifestDigest computes the manifest digest, and, if pulling by
  659. // digest, ensures that it matches the requested digest.
  660. func schema2ManifestDigest(ref reference.Named, mfst distribution.Manifest) (digest.Digest, error) {
  661. _, canonical, err := mfst.Payload()
  662. if err != nil {
  663. return "", err
  664. }
  665. // If pull by digest, then verify the manifest digest.
  666. if digested, isDigested := ref.(reference.Canonical); isDigested {
  667. verifier := digested.Digest().Verifier()
  668. if _, err := verifier.Write(canonical); err != nil {
  669. return "", err
  670. }
  671. if !verifier.Verified() {
  672. err := fmt.Errorf("manifest verification failed for digest %s", digested.Digest())
  673. logrus.Error(err)
  674. return "", err
  675. }
  676. return digested.Digest(), nil
  677. }
  678. return digest.FromBytes(canonical), nil
  679. }
  680. // allowV1Fallback checks if the error is a possible reason to fallback to v1
  681. // (even if confirmedV2 has been set already), and if so, wraps the error in
  682. // a fallbackError with confirmedV2 set to false. Otherwise, it returns the
  683. // error unmodified.
  684. func allowV1Fallback(err error) error {
  685. switch v := err.(type) {
  686. case errcode.Errors:
  687. if len(v) != 0 {
  688. if v0, ok := v[0].(errcode.Error); ok && shouldV2Fallback(v0) {
  689. return fallbackError{
  690. err: err,
  691. confirmedV2: false,
  692. transportOK: true,
  693. }
  694. }
  695. }
  696. case errcode.Error:
  697. if shouldV2Fallback(v) {
  698. return fallbackError{
  699. err: err,
  700. confirmedV2: false,
  701. transportOK: true,
  702. }
  703. }
  704. case *url.Error:
  705. if v.Err == auth.ErrNoBasicAuthCredentials {
  706. return fallbackError{err: err, confirmedV2: false}
  707. }
  708. }
  709. return err
  710. }
  711. func verifySchema1Manifest(signedManifest *schema1.SignedManifest, ref reference.Named) (m *schema1.Manifest, err error) {
  712. // If pull by digest, then verify the manifest digest. NOTE: It is
  713. // important to do this first, before any other content validation. If the
  714. // digest cannot be verified, don't even bother with those other things.
  715. if digested, isCanonical := ref.(reference.Canonical); isCanonical {
  716. verifier := digested.Digest().Verifier()
  717. if _, err := verifier.Write(signedManifest.Canonical); err != nil {
  718. return nil, err
  719. }
  720. if !verifier.Verified() {
  721. err := fmt.Errorf("image verification failed for digest %s", digested.Digest())
  722. logrus.Error(err)
  723. return nil, err
  724. }
  725. }
  726. m = &signedManifest.Manifest
  727. if m.SchemaVersion != 1 {
  728. return nil, fmt.Errorf("unsupported schema version %d for %q", m.SchemaVersion, reference.FamiliarString(ref))
  729. }
  730. if len(m.FSLayers) != len(m.History) {
  731. return nil, fmt.Errorf("length of history not equal to number of layers for %q", reference.FamiliarString(ref))
  732. }
  733. if len(m.FSLayers) == 0 {
  734. return nil, fmt.Errorf("no FSLayers in manifest for %q", reference.FamiliarString(ref))
  735. }
  736. return m, nil
  737. }
  738. // fixManifestLayers removes repeated layers from the manifest and checks the
  739. // correctness of the parent chain.
  740. func fixManifestLayers(m *schema1.Manifest) error {
  741. imgs := make([]*image.V1Image, len(m.FSLayers))
  742. for i := range m.FSLayers {
  743. img := &image.V1Image{}
  744. if err := json.Unmarshal([]byte(m.History[i].V1Compatibility), img); err != nil {
  745. return err
  746. }
  747. imgs[i] = img
  748. if err := v1.ValidateID(img.ID); err != nil {
  749. return err
  750. }
  751. }
  752. if imgs[len(imgs)-1].Parent != "" && runtime.GOOS != "windows" {
  753. // Windows base layer can point to a base layer parent that is not in manifest.
  754. return errors.New("invalid parent ID in the base layer of the image")
  755. }
  756. // check general duplicates to error instead of a deadlock
  757. idmap := make(map[string]struct{})
  758. var lastID string
  759. for _, img := range imgs {
  760. // skip IDs that appear after each other, we handle those later
  761. if _, exists := idmap[img.ID]; img.ID != lastID && exists {
  762. return fmt.Errorf("ID %+v appears multiple times in manifest", img.ID)
  763. }
  764. lastID = img.ID
  765. idmap[lastID] = struct{}{}
  766. }
  767. // backwards loop so that we keep the remaining indexes after removing items
  768. for i := len(imgs) - 2; i >= 0; i-- {
  769. if imgs[i].ID == imgs[i+1].ID { // repeated ID. remove and continue
  770. m.FSLayers = append(m.FSLayers[:i], m.FSLayers[i+1:]...)
  771. m.History = append(m.History[:i], m.History[i+1:]...)
  772. } else if imgs[i].Parent != imgs[i+1].ID {
  773. return fmt.Errorf("Invalid parent ID. Expected %v, got %v.", imgs[i+1].ID, imgs[i].Parent)
  774. }
  775. }
  776. return nil
  777. }
  778. func createDownloadFile() (*os.File, error) {
  779. return ioutil.TempFile("", "GetImageBlob")
  780. }