archive_test.go 40 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463
  1. package archive // import "github.com/docker/docker/pkg/archive"
  2. import (
  3. "archive/tar"
  4. "bytes"
  5. "compress/gzip"
  6. "errors"
  7. "fmt"
  8. "io"
  9. "io/fs"
  10. "os"
  11. "os/exec"
  12. "path/filepath"
  13. "reflect"
  14. "runtime"
  15. "strings"
  16. "testing"
  17. "time"
  18. "github.com/containerd/containerd/pkg/userns"
  19. "github.com/docker/docker/pkg/idtools"
  20. "github.com/docker/docker/pkg/ioutils"
  21. "gotest.tools/v3/assert"
  22. is "gotest.tools/v3/assert/cmp"
  23. "gotest.tools/v3/skip"
  24. )
  25. var tmp string
  26. func init() {
  27. tmp = "/tmp/"
  28. if runtime.GOOS == "windows" {
  29. tmp = os.Getenv("TEMP") + `\`
  30. }
  31. }
  32. var defaultArchiver = NewDefaultArchiver()
  33. func defaultTarUntar(src, dst string) error {
  34. return defaultArchiver.TarUntar(src, dst)
  35. }
  36. func defaultUntarPath(src, dst string) error {
  37. return defaultArchiver.UntarPath(src, dst)
  38. }
  39. func defaultCopyFileWithTar(src, dst string) (err error) {
  40. return defaultArchiver.CopyFileWithTar(src, dst)
  41. }
  42. func defaultCopyWithTar(src, dst string) error {
  43. return defaultArchiver.CopyWithTar(src, dst)
  44. }
  45. func TestIsArchivePathDir(t *testing.T) {
  46. cmd := exec.Command("sh", "-c", "mkdir -p /tmp/archivedir")
  47. output, err := cmd.CombinedOutput()
  48. if err != nil {
  49. t.Fatalf("Fail to create an archive file for test : %s.", output)
  50. }
  51. if IsArchivePath(tmp + "archivedir") {
  52. t.Fatalf("Incorrectly recognised directory as an archive")
  53. }
  54. }
  55. func TestIsArchivePathInvalidFile(t *testing.T) {
  56. cmd := exec.Command("sh", "-c", "dd if=/dev/zero bs=1024 count=1 of=/tmp/archive && gzip --stdout /tmp/archive > /tmp/archive.gz")
  57. output, err := cmd.CombinedOutput()
  58. if err != nil {
  59. t.Fatalf("Fail to create an archive file for test : %s.", output)
  60. }
  61. if IsArchivePath(tmp + "archive") {
  62. t.Fatalf("Incorrectly recognised invalid tar path as archive")
  63. }
  64. if IsArchivePath(tmp + "archive.gz") {
  65. t.Fatalf("Incorrectly recognised invalid compressed tar path as archive")
  66. }
  67. }
  68. func TestIsArchivePathTar(t *testing.T) {
  69. whichTar := "tar"
  70. cmdStr := fmt.Sprintf("touch /tmp/archivedata && %s -cf /tmp/archive /tmp/archivedata && gzip --stdout /tmp/archive > /tmp/archive.gz", whichTar)
  71. cmd := exec.Command("sh", "-c", cmdStr)
  72. output, err := cmd.CombinedOutput()
  73. if err != nil {
  74. t.Fatalf("Fail to create an archive file for test : %s.", output)
  75. }
  76. if !IsArchivePath(tmp + "/archive") {
  77. t.Fatalf("Did not recognise valid tar path as archive")
  78. }
  79. if !IsArchivePath(tmp + "archive.gz") {
  80. t.Fatalf("Did not recognise valid compressed tar path as archive")
  81. }
  82. }
  83. func testDecompressStream(t *testing.T, ext, compressCommand string) io.Reader {
  84. cmd := exec.Command("sh", "-c",
  85. fmt.Sprintf("touch /tmp/archive && %s /tmp/archive", compressCommand))
  86. output, err := cmd.CombinedOutput()
  87. if err != nil {
  88. t.Fatalf("Failed to create an archive file for test : %s.", output)
  89. }
  90. filename := "archive." + ext
  91. archive, err := os.Open(tmp + filename)
  92. if err != nil {
  93. t.Fatalf("Failed to open file %s: %v", filename, err)
  94. }
  95. defer archive.Close()
  96. r, err := DecompressStream(archive)
  97. if err != nil {
  98. t.Fatalf("Failed to decompress %s: %v", filename, err)
  99. }
  100. if _, err = io.ReadAll(r); err != nil {
  101. t.Fatalf("Failed to read the decompressed stream: %v ", err)
  102. }
  103. if err = r.Close(); err != nil {
  104. t.Fatalf("Failed to close the decompressed stream: %v ", err)
  105. }
  106. return r
  107. }
  108. func TestDecompressStreamGzip(t *testing.T) {
  109. testDecompressStream(t, "gz", "gzip -f")
  110. }
  111. func TestDecompressStreamBzip2(t *testing.T) {
  112. testDecompressStream(t, "bz2", "bzip2 -f")
  113. }
  114. func TestDecompressStreamXz(t *testing.T) {
  115. if runtime.GOOS == "windows" {
  116. t.Skip("Xz not present in msys2")
  117. }
  118. testDecompressStream(t, "xz", "xz -f")
  119. }
  120. func TestDecompressStreamZstd(t *testing.T) {
  121. if _, err := exec.LookPath("zstd"); err != nil {
  122. t.Skip("zstd not installed")
  123. }
  124. testDecompressStream(t, "zst", "zstd -f")
  125. }
  126. func TestCompressStreamXzUnsupported(t *testing.T) {
  127. dest, err := os.Create(tmp + "dest")
  128. if err != nil {
  129. t.Fatalf("Fail to create the destination file")
  130. }
  131. defer dest.Close()
  132. _, err = CompressStream(dest, Xz)
  133. if err == nil {
  134. t.Fatalf("Should fail as xz is unsupported for compression format.")
  135. }
  136. }
  137. func TestCompressStreamBzip2Unsupported(t *testing.T) {
  138. dest, err := os.Create(tmp + "dest")
  139. if err != nil {
  140. t.Fatalf("Fail to create the destination file")
  141. }
  142. defer dest.Close()
  143. _, err = CompressStream(dest, Bzip2)
  144. if err == nil {
  145. t.Fatalf("Should fail as bzip2 is unsupported for compression format.")
  146. }
  147. }
  148. func TestCompressStreamInvalid(t *testing.T) {
  149. dest, err := os.Create(tmp + "dest")
  150. if err != nil {
  151. t.Fatalf("Fail to create the destination file")
  152. }
  153. defer dest.Close()
  154. _, err = CompressStream(dest, -1)
  155. if err == nil {
  156. t.Fatalf("Should fail as xz is unsupported for compression format.")
  157. }
  158. }
  159. func TestExtensionInvalid(t *testing.T) {
  160. compression := Compression(-1)
  161. output := compression.Extension()
  162. if output != "" {
  163. t.Fatalf("The extension of an invalid compression should be an empty string.")
  164. }
  165. }
  166. func TestExtensionUncompressed(t *testing.T) {
  167. compression := Uncompressed
  168. output := compression.Extension()
  169. if output != "tar" {
  170. t.Fatalf("The extension of an uncompressed archive should be 'tar'.")
  171. }
  172. }
  173. func TestExtensionBzip2(t *testing.T) {
  174. compression := Bzip2
  175. output := compression.Extension()
  176. if output != "tar.bz2" {
  177. t.Fatalf("The extension of a bzip2 archive should be 'tar.bz2'")
  178. }
  179. }
  180. func TestExtensionGzip(t *testing.T) {
  181. compression := Gzip
  182. output := compression.Extension()
  183. if output != "tar.gz" {
  184. t.Fatalf("The extension of a gzip archive should be 'tar.gz'")
  185. }
  186. }
  187. func TestExtensionXz(t *testing.T) {
  188. compression := Xz
  189. output := compression.Extension()
  190. if output != "tar.xz" {
  191. t.Fatalf("The extension of a xz archive should be 'tar.xz'")
  192. }
  193. }
  194. func TestExtensionZstd(t *testing.T) {
  195. compression := Zstd
  196. output := compression.Extension()
  197. if output != "tar.zst" {
  198. t.Fatalf("The extension of a zstd archive should be 'tar.zst'")
  199. }
  200. }
  201. func TestCmdStreamLargeStderr(t *testing.T) {
  202. cmd := exec.Command("sh", "-c", "dd if=/dev/zero bs=1k count=1000 of=/dev/stderr; echo hello")
  203. out, err := cmdStream(cmd, nil)
  204. if err != nil {
  205. t.Fatalf("Failed to start command: %s", err)
  206. }
  207. errCh := make(chan error, 1)
  208. go func() {
  209. _, err := io.Copy(io.Discard, out)
  210. errCh <- err
  211. }()
  212. select {
  213. case err := <-errCh:
  214. if err != nil {
  215. t.Fatalf("Command should not have failed (err=%.100s...)", err)
  216. }
  217. case <-time.After(5 * time.Second):
  218. t.Fatalf("Command did not complete in 5 seconds; probable deadlock")
  219. }
  220. }
  221. func TestCmdStreamBad(t *testing.T) {
  222. // TODO Windows: Figure out why this is failing in CI but not locally
  223. if runtime.GOOS == "windows" {
  224. t.Skip("Failing on Windows CI machines")
  225. }
  226. badCmd := exec.Command("sh", "-c", "echo hello; echo >&2 error couldn\\'t reverse the phase pulser; exit 1")
  227. out, err := cmdStream(badCmd, nil)
  228. if err != nil {
  229. t.Fatalf("Failed to start command: %s", err)
  230. }
  231. if output, err := io.ReadAll(out); err == nil {
  232. t.Fatalf("Command should have failed")
  233. } else if err.Error() != "exit status 1: error couldn't reverse the phase pulser\n" {
  234. t.Fatalf("Wrong error value (%s)", err)
  235. } else if s := string(output); s != "hello\n" {
  236. t.Fatalf("Command output should be '%s', not '%s'", "hello\\n", output)
  237. }
  238. }
  239. func TestCmdStreamGood(t *testing.T) {
  240. cmd := exec.Command("sh", "-c", "echo hello; exit 0")
  241. out, err := cmdStream(cmd, nil)
  242. if err != nil {
  243. t.Fatal(err)
  244. }
  245. if output, err := io.ReadAll(out); err != nil {
  246. t.Fatalf("Command should not have failed (err=%s)", err)
  247. } else if s := string(output); s != "hello\n" {
  248. t.Fatalf("Command output should be '%s', not '%s'", "hello\\n", output)
  249. }
  250. }
  251. func TestUntarPathWithInvalidDest(t *testing.T) {
  252. tempFolder, err := os.MkdirTemp("", "docker-archive-test")
  253. assert.NilError(t, err)
  254. defer os.RemoveAll(tempFolder)
  255. invalidDestFolder := filepath.Join(tempFolder, "invalidDest")
  256. // Create a src file
  257. srcFile := filepath.Join(tempFolder, "src")
  258. tarFile := filepath.Join(tempFolder, "src.tar")
  259. os.Create(srcFile)
  260. os.Create(invalidDestFolder) // being a file (not dir) should cause an error
  261. // Translate back to Unix semantics as next exec.Command is run under sh
  262. srcFileU := srcFile
  263. tarFileU := tarFile
  264. if runtime.GOOS == "windows" {
  265. tarFileU = "/tmp/" + filepath.Base(filepath.Dir(tarFile)) + "/src.tar"
  266. srcFileU = "/tmp/" + filepath.Base(filepath.Dir(srcFile)) + "/src"
  267. }
  268. cmd := exec.Command("sh", "-c", "tar cf "+tarFileU+" "+srcFileU)
  269. _, err = cmd.CombinedOutput()
  270. assert.NilError(t, err)
  271. err = defaultUntarPath(tarFile, invalidDestFolder)
  272. if err == nil {
  273. t.Fatalf("UntarPath with invalid destination path should throw an error.")
  274. }
  275. }
  276. func TestUntarPathWithInvalidSrc(t *testing.T) {
  277. dest, err := os.MkdirTemp("", "docker-archive-test")
  278. if err != nil {
  279. t.Fatalf("Fail to create the destination file")
  280. }
  281. defer os.RemoveAll(dest)
  282. err = defaultUntarPath("/invalid/path", dest)
  283. if err == nil {
  284. t.Fatalf("UntarPath with invalid src path should throw an error.")
  285. }
  286. }
  287. func TestUntarPath(t *testing.T) {
  288. skip.If(t, runtime.GOOS != "windows" && os.Getuid() != 0, "skipping test that requires root")
  289. tmpFolder, err := os.MkdirTemp("", "docker-archive-test")
  290. assert.NilError(t, err)
  291. defer os.RemoveAll(tmpFolder)
  292. srcFile := filepath.Join(tmpFolder, "src")
  293. tarFile := filepath.Join(tmpFolder, "src.tar")
  294. os.Create(filepath.Join(tmpFolder, "src"))
  295. destFolder := filepath.Join(tmpFolder, "dest")
  296. err = os.MkdirAll(destFolder, 0o740)
  297. if err != nil {
  298. t.Fatalf("Fail to create the destination file")
  299. }
  300. // Translate back to Unix semantics as next exec.Command is run under sh
  301. srcFileU := srcFile
  302. tarFileU := tarFile
  303. if runtime.GOOS == "windows" {
  304. tarFileU = "/tmp/" + filepath.Base(filepath.Dir(tarFile)) + "/src.tar"
  305. srcFileU = "/tmp/" + filepath.Base(filepath.Dir(srcFile)) + "/src"
  306. }
  307. cmd := exec.Command("sh", "-c", "tar cf "+tarFileU+" "+srcFileU)
  308. _, err = cmd.CombinedOutput()
  309. assert.NilError(t, err)
  310. err = defaultUntarPath(tarFile, destFolder)
  311. if err != nil {
  312. t.Fatalf("UntarPath shouldn't throw an error, %s.", err)
  313. }
  314. expectedFile := filepath.Join(destFolder, srcFileU)
  315. _, err = os.Stat(expectedFile)
  316. if err != nil {
  317. t.Fatalf("Destination folder should contain the source file but did not.")
  318. }
  319. }
  320. // Do the same test as above but with the destination as file, it should fail
  321. func TestUntarPathWithDestinationFile(t *testing.T) {
  322. tmpFolder, err := os.MkdirTemp("", "docker-archive-test")
  323. if err != nil {
  324. t.Fatal(err)
  325. }
  326. defer os.RemoveAll(tmpFolder)
  327. srcFile := filepath.Join(tmpFolder, "src")
  328. tarFile := filepath.Join(tmpFolder, "src.tar")
  329. os.Create(filepath.Join(tmpFolder, "src"))
  330. // Translate back to Unix semantics as next exec.Command is run under sh
  331. srcFileU := srcFile
  332. tarFileU := tarFile
  333. if runtime.GOOS == "windows" {
  334. tarFileU = "/tmp/" + filepath.Base(filepath.Dir(tarFile)) + "/src.tar"
  335. srcFileU = "/tmp/" + filepath.Base(filepath.Dir(srcFile)) + "/src"
  336. }
  337. cmd := exec.Command("sh", "-c", "tar cf "+tarFileU+" "+srcFileU)
  338. _, err = cmd.CombinedOutput()
  339. if err != nil {
  340. t.Fatal(err)
  341. }
  342. destFile := filepath.Join(tmpFolder, "dest")
  343. _, err = os.Create(destFile)
  344. if err != nil {
  345. t.Fatalf("Fail to create the destination file")
  346. }
  347. err = defaultUntarPath(tarFile, destFile)
  348. if err == nil {
  349. t.Fatalf("UntarPath should throw an error if the destination if a file")
  350. }
  351. }
  352. // Do the same test as above but with the destination folder already exists
  353. // and the destination file is a directory
  354. // It's working, see https://github.com/docker/docker/issues/10040
  355. func TestUntarPathWithDestinationSrcFileAsFolder(t *testing.T) {
  356. tmpFolder, err := os.MkdirTemp("", "docker-archive-test")
  357. if err != nil {
  358. t.Fatal(err)
  359. }
  360. defer os.RemoveAll(tmpFolder)
  361. srcFile := filepath.Join(tmpFolder, "src")
  362. tarFile := filepath.Join(tmpFolder, "src.tar")
  363. os.Create(srcFile)
  364. // Translate back to Unix semantics as next exec.Command is run under sh
  365. srcFileU := srcFile
  366. tarFileU := tarFile
  367. if runtime.GOOS == "windows" {
  368. tarFileU = "/tmp/" + filepath.Base(filepath.Dir(tarFile)) + "/src.tar"
  369. srcFileU = "/tmp/" + filepath.Base(filepath.Dir(srcFile)) + "/src"
  370. }
  371. cmd := exec.Command("sh", "-c", "tar cf "+tarFileU+" "+srcFileU)
  372. _, err = cmd.CombinedOutput()
  373. if err != nil {
  374. t.Fatal(err)
  375. }
  376. destFolder := filepath.Join(tmpFolder, "dest")
  377. err = os.MkdirAll(destFolder, 0o740)
  378. if err != nil {
  379. t.Fatalf("Fail to create the destination folder")
  380. }
  381. // Let's create a folder that will has the same path as the extracted file (from tar)
  382. destSrcFileAsFolder := filepath.Join(destFolder, srcFileU)
  383. err = os.MkdirAll(destSrcFileAsFolder, 0o740)
  384. if err != nil {
  385. t.Fatal(err)
  386. }
  387. err = defaultUntarPath(tarFile, destFolder)
  388. if err != nil {
  389. t.Fatalf("UntarPath should throw not throw an error if the extracted file already exists and is a folder")
  390. }
  391. }
  392. func TestCopyWithTarInvalidSrc(t *testing.T) {
  393. tempFolder, err := os.MkdirTemp("", "docker-archive-test")
  394. if err != nil {
  395. t.Fatal(nil)
  396. }
  397. destFolder := filepath.Join(tempFolder, "dest")
  398. invalidSrc := filepath.Join(tempFolder, "doesnotexists")
  399. err = os.MkdirAll(destFolder, 0o740)
  400. if err != nil {
  401. t.Fatal(err)
  402. }
  403. err = defaultCopyWithTar(invalidSrc, destFolder)
  404. if err == nil {
  405. t.Fatalf("archiver.CopyWithTar with invalid src path should throw an error.")
  406. }
  407. }
  408. func TestCopyWithTarInexistentDestWillCreateIt(t *testing.T) {
  409. skip.If(t, runtime.GOOS != "windows" && os.Getuid() != 0, "skipping test that requires root")
  410. tempFolder, err := os.MkdirTemp("", "docker-archive-test")
  411. if err != nil {
  412. t.Fatal(nil)
  413. }
  414. srcFolder := filepath.Join(tempFolder, "src")
  415. inexistentDestFolder := filepath.Join(tempFolder, "doesnotexists")
  416. err = os.MkdirAll(srcFolder, 0o740)
  417. if err != nil {
  418. t.Fatal(err)
  419. }
  420. err = defaultCopyWithTar(srcFolder, inexistentDestFolder)
  421. if err != nil {
  422. t.Fatalf("CopyWithTar with an inexistent folder shouldn't fail.")
  423. }
  424. _, err = os.Stat(inexistentDestFolder)
  425. if err != nil {
  426. t.Fatalf("CopyWithTar with an inexistent folder should create it.")
  427. }
  428. }
  429. // Test CopyWithTar with a file as src
  430. func TestCopyWithTarSrcFile(t *testing.T) {
  431. folder, err := os.MkdirTemp("", "docker-archive-test")
  432. if err != nil {
  433. t.Fatal(err)
  434. }
  435. defer os.RemoveAll(folder)
  436. dest := filepath.Join(folder, "dest")
  437. srcFolder := filepath.Join(folder, "src")
  438. src := filepath.Join(folder, filepath.Join("src", "src"))
  439. err = os.MkdirAll(srcFolder, 0o740)
  440. if err != nil {
  441. t.Fatal(err)
  442. }
  443. err = os.MkdirAll(dest, 0o740)
  444. if err != nil {
  445. t.Fatal(err)
  446. }
  447. os.WriteFile(src, []byte("content"), 0o777)
  448. err = defaultCopyWithTar(src, dest)
  449. if err != nil {
  450. t.Fatalf("archiver.CopyWithTar shouldn't throw an error, %s.", err)
  451. }
  452. _, err = os.Stat(dest)
  453. // FIXME Check the content
  454. if err != nil {
  455. t.Fatalf("Destination file should be the same as the source.")
  456. }
  457. }
  458. // Test CopyWithTar with a folder as src
  459. func TestCopyWithTarSrcFolder(t *testing.T) {
  460. folder, err := os.MkdirTemp("", "docker-archive-test")
  461. if err != nil {
  462. t.Fatal(err)
  463. }
  464. defer os.RemoveAll(folder)
  465. dest := filepath.Join(folder, "dest")
  466. src := filepath.Join(folder, filepath.Join("src", "folder"))
  467. err = os.MkdirAll(src, 0o740)
  468. if err != nil {
  469. t.Fatal(err)
  470. }
  471. err = os.MkdirAll(dest, 0o740)
  472. if err != nil {
  473. t.Fatal(err)
  474. }
  475. os.WriteFile(filepath.Join(src, "file"), []byte("content"), 0o777)
  476. err = defaultCopyWithTar(src, dest)
  477. if err != nil {
  478. t.Fatalf("archiver.CopyWithTar shouldn't throw an error, %s.", err)
  479. }
  480. _, err = os.Stat(dest)
  481. // FIXME Check the content (the file inside)
  482. if err != nil {
  483. t.Fatalf("Destination folder should contain the source file but did not.")
  484. }
  485. }
  486. func TestCopyFileWithTarInvalidSrc(t *testing.T) {
  487. tempFolder, err := os.MkdirTemp("", "docker-archive-test")
  488. if err != nil {
  489. t.Fatal(err)
  490. }
  491. defer os.RemoveAll(tempFolder)
  492. destFolder := filepath.Join(tempFolder, "dest")
  493. err = os.MkdirAll(destFolder, 0o740)
  494. if err != nil {
  495. t.Fatal(err)
  496. }
  497. invalidFile := filepath.Join(tempFolder, "doesnotexists")
  498. err = defaultCopyFileWithTar(invalidFile, destFolder)
  499. if err == nil {
  500. t.Fatalf("archiver.CopyWithTar with invalid src path should throw an error.")
  501. }
  502. }
  503. func TestCopyFileWithTarInexistentDestWillCreateIt(t *testing.T) {
  504. tempFolder, err := os.MkdirTemp("", "docker-archive-test")
  505. if err != nil {
  506. t.Fatal(nil)
  507. }
  508. defer os.RemoveAll(tempFolder)
  509. srcFile := filepath.Join(tempFolder, "src")
  510. inexistentDestFolder := filepath.Join(tempFolder, "doesnotexists")
  511. _, err = os.Create(srcFile)
  512. if err != nil {
  513. t.Fatal(err)
  514. }
  515. err = defaultCopyFileWithTar(srcFile, inexistentDestFolder)
  516. if err != nil {
  517. t.Fatalf("CopyWithTar with an inexistent folder shouldn't fail.")
  518. }
  519. _, err = os.Stat(inexistentDestFolder)
  520. if err != nil {
  521. t.Fatalf("CopyWithTar with an inexistent folder should create it.")
  522. }
  523. // FIXME Test the src file and content
  524. }
  525. func TestCopyFileWithTarSrcFolder(t *testing.T) {
  526. folder, err := os.MkdirTemp("", "docker-archive-copyfilewithtar-test")
  527. if err != nil {
  528. t.Fatal(err)
  529. }
  530. defer os.RemoveAll(folder)
  531. dest := filepath.Join(folder, "dest")
  532. src := filepath.Join(folder, "srcfolder")
  533. err = os.MkdirAll(src, 0o740)
  534. if err != nil {
  535. t.Fatal(err)
  536. }
  537. err = os.MkdirAll(dest, 0o740)
  538. if err != nil {
  539. t.Fatal(err)
  540. }
  541. err = defaultCopyFileWithTar(src, dest)
  542. if err == nil {
  543. t.Fatalf("CopyFileWithTar should throw an error with a folder.")
  544. }
  545. }
  546. func TestCopyFileWithTarSrcFile(t *testing.T) {
  547. folder, err := os.MkdirTemp("", "docker-archive-test")
  548. if err != nil {
  549. t.Fatal(err)
  550. }
  551. defer os.RemoveAll(folder)
  552. dest := filepath.Join(folder, "dest")
  553. srcFolder := filepath.Join(folder, "src")
  554. src := filepath.Join(folder, filepath.Join("src", "src"))
  555. err = os.MkdirAll(srcFolder, 0o740)
  556. if err != nil {
  557. t.Fatal(err)
  558. }
  559. err = os.MkdirAll(dest, 0o740)
  560. if err != nil {
  561. t.Fatal(err)
  562. }
  563. os.WriteFile(src, []byte("content"), 0o777)
  564. err = defaultCopyWithTar(src, dest+"/")
  565. if err != nil {
  566. t.Fatalf("archiver.CopyFileWithTar shouldn't throw an error, %s.", err)
  567. }
  568. _, err = os.Stat(dest)
  569. if err != nil {
  570. t.Fatalf("Destination folder should contain the source file but did not.")
  571. }
  572. }
  573. func TestTarFiles(t *testing.T) {
  574. // try without hardlinks
  575. if err := checkNoChanges(1000, false); err != nil {
  576. t.Fatal(err)
  577. }
  578. // try with hardlinks
  579. if err := checkNoChanges(1000, true); err != nil {
  580. t.Fatal(err)
  581. }
  582. }
  583. func checkNoChanges(fileNum int, hardlinks bool) error {
  584. srcDir, err := os.MkdirTemp("", "docker-test-srcDir")
  585. if err != nil {
  586. return err
  587. }
  588. defer os.RemoveAll(srcDir)
  589. destDir, err := os.MkdirTemp("", "docker-test-destDir")
  590. if err != nil {
  591. return err
  592. }
  593. defer os.RemoveAll(destDir)
  594. _, err = prepareUntarSourceDirectory(fileNum, srcDir, hardlinks)
  595. if err != nil {
  596. return err
  597. }
  598. err = defaultTarUntar(srcDir, destDir)
  599. if err != nil {
  600. return err
  601. }
  602. changes, err := ChangesDirs(destDir, srcDir)
  603. if err != nil {
  604. return err
  605. }
  606. if len(changes) > 0 {
  607. return fmt.Errorf("with %d files and %v hardlinks: expected 0 changes, got %d", fileNum, hardlinks, len(changes))
  608. }
  609. return nil
  610. }
  611. func tarUntar(t *testing.T, origin string, options *TarOptions) ([]Change, error) {
  612. archive, err := TarWithOptions(origin, options)
  613. if err != nil {
  614. t.Fatal(err)
  615. }
  616. defer archive.Close()
  617. buf := make([]byte, 10)
  618. if _, err := archive.Read(buf); err != nil {
  619. return nil, err
  620. }
  621. wrap := io.MultiReader(bytes.NewReader(buf), archive)
  622. detectedCompression := DetectCompression(buf)
  623. compression := options.Compression
  624. if detectedCompression.Extension() != compression.Extension() {
  625. return nil, fmt.Errorf("Wrong compression detected. Actual compression: %s, found %s", compression.Extension(), detectedCompression.Extension())
  626. }
  627. tmp, err := os.MkdirTemp("", "docker-test-untar")
  628. if err != nil {
  629. return nil, err
  630. }
  631. defer os.RemoveAll(tmp)
  632. if err := Untar(wrap, tmp, nil); err != nil {
  633. return nil, err
  634. }
  635. if _, err := os.Stat(tmp); err != nil {
  636. return nil, err
  637. }
  638. return ChangesDirs(origin, tmp)
  639. }
  640. func TestDetectCompressionZstd(t *testing.T) {
  641. // test zstd compression without skippable frames.
  642. compressedData := []byte{
  643. 0x28, 0xb5, 0x2f, 0xfd, // magic number of Zstandard frame: 0xFD2FB528
  644. 0x04, 0x00, 0x31, 0x00, 0x00, // frame header
  645. 0x64, 0x6f, 0x63, 0x6b, 0x65, 0x72, // data block "docker"
  646. 0x16, 0x0e, 0x21, 0xc3, // content checksum
  647. }
  648. compression := DetectCompression(compressedData)
  649. if compression != Zstd {
  650. t.Fatal("Unexpected compression")
  651. }
  652. // test zstd compression with skippable frames.
  653. hex := []byte{
  654. 0x50, 0x2a, 0x4d, 0x18, // magic number of skippable frame: 0x184D2A50 to 0x184D2A5F
  655. 0x04, 0x00, 0x00, 0x00, // frame size
  656. 0x5d, 0x00, 0x00, 0x00, // user data
  657. 0x28, 0xb5, 0x2f, 0xfd, // magic number of Zstandard frame: 0xFD2FB528
  658. 0x04, 0x00, 0x31, 0x00, 0x00, // frame header
  659. 0x64, 0x6f, 0x63, 0x6b, 0x65, 0x72, // data block "docker"
  660. 0x16, 0x0e, 0x21, 0xc3, // content checksum
  661. }
  662. compression = DetectCompression(hex)
  663. if compression != Zstd {
  664. t.Fatal("Unexpected compression")
  665. }
  666. }
  667. func TestTarUntar(t *testing.T) {
  668. origin, err := os.MkdirTemp("", "docker-test-untar-origin")
  669. if err != nil {
  670. t.Fatal(err)
  671. }
  672. defer os.RemoveAll(origin)
  673. if err := os.WriteFile(filepath.Join(origin, "1"), []byte("hello world"), 0o700); err != nil {
  674. t.Fatal(err)
  675. }
  676. if err := os.WriteFile(filepath.Join(origin, "2"), []byte("welcome!"), 0o700); err != nil {
  677. t.Fatal(err)
  678. }
  679. if err := os.WriteFile(filepath.Join(origin, "3"), []byte("will be ignored"), 0o700); err != nil {
  680. t.Fatal(err)
  681. }
  682. for _, c := range []Compression{
  683. Uncompressed,
  684. Gzip,
  685. } {
  686. changes, err := tarUntar(t, origin, &TarOptions{
  687. Compression: c,
  688. ExcludePatterns: []string{"3"},
  689. })
  690. if err != nil {
  691. t.Fatalf("Error tar/untar for compression %s: %s", c.Extension(), err)
  692. }
  693. if len(changes) != 1 || changes[0].Path != string(filepath.Separator)+"3" {
  694. t.Fatalf("Unexpected differences after tarUntar: %v", changes)
  695. }
  696. }
  697. }
  698. func TestTarWithOptionsChownOptsAlwaysOverridesIdPair(t *testing.T) {
  699. origin, err := os.MkdirTemp("", "docker-test-tar-chown-opt")
  700. assert.NilError(t, err)
  701. defer os.RemoveAll(origin)
  702. filePath := filepath.Join(origin, "1")
  703. err = os.WriteFile(filePath, []byte("hello world"), 0o700)
  704. assert.NilError(t, err)
  705. idMaps := []idtools.IDMap{
  706. 0: {
  707. ContainerID: 0,
  708. HostID: 0,
  709. Size: 65536,
  710. },
  711. 1: {
  712. ContainerID: 0,
  713. HostID: 100000,
  714. Size: 65536,
  715. },
  716. }
  717. cases := []struct {
  718. opts *TarOptions
  719. expectedUID int
  720. expectedGID int
  721. }{
  722. {&TarOptions{ChownOpts: &idtools.Identity{UID: 1337, GID: 42}}, 1337, 42},
  723. {&TarOptions{ChownOpts: &idtools.Identity{UID: 100001, GID: 100001}, IDMap: idtools.IdentityMapping{UIDMaps: idMaps, GIDMaps: idMaps}}, 100001, 100001},
  724. {&TarOptions{ChownOpts: &idtools.Identity{UID: 0, GID: 0}, NoLchown: false}, 0, 0},
  725. {&TarOptions{ChownOpts: &idtools.Identity{UID: 1, GID: 1}, NoLchown: true}, 1, 1},
  726. {&TarOptions{ChownOpts: &idtools.Identity{UID: 1000, GID: 1000}, NoLchown: true}, 1000, 1000},
  727. }
  728. for _, testCase := range cases {
  729. reader, err := TarWithOptions(filePath, testCase.opts)
  730. assert.NilError(t, err)
  731. tr := tar.NewReader(reader)
  732. defer reader.Close()
  733. for {
  734. hdr, err := tr.Next()
  735. if err == io.EOF {
  736. // end of tar archive
  737. break
  738. }
  739. assert.NilError(t, err)
  740. assert.Check(t, is.Equal(hdr.Uid, testCase.expectedUID), "Uid equals expected value")
  741. assert.Check(t, is.Equal(hdr.Gid, testCase.expectedGID), "Gid equals expected value")
  742. }
  743. }
  744. }
  745. func TestTarWithOptions(t *testing.T) {
  746. origin, err := os.MkdirTemp("", "docker-test-untar-origin")
  747. if err != nil {
  748. t.Fatal(err)
  749. }
  750. if _, err := os.MkdirTemp(origin, "folder"); err != nil {
  751. t.Fatal(err)
  752. }
  753. defer os.RemoveAll(origin)
  754. if err := os.WriteFile(filepath.Join(origin, "1"), []byte("hello world"), 0o700); err != nil {
  755. t.Fatal(err)
  756. }
  757. if err := os.WriteFile(filepath.Join(origin, "2"), []byte("welcome!"), 0o700); err != nil {
  758. t.Fatal(err)
  759. }
  760. cases := []struct {
  761. opts *TarOptions
  762. numChanges int
  763. }{
  764. {&TarOptions{IncludeFiles: []string{"1"}}, 2},
  765. {&TarOptions{ExcludePatterns: []string{"2"}}, 1},
  766. {&TarOptions{ExcludePatterns: []string{"1", "folder*"}}, 2},
  767. {&TarOptions{IncludeFiles: []string{"1", "1"}}, 2},
  768. {&TarOptions{IncludeFiles: []string{"1"}, RebaseNames: map[string]string{"1": "test"}}, 4},
  769. }
  770. for _, testCase := range cases {
  771. changes, err := tarUntar(t, origin, testCase.opts)
  772. if err != nil {
  773. t.Fatalf("Error tar/untar when testing inclusion/exclusion: %s", err)
  774. }
  775. if len(changes) != testCase.numChanges {
  776. t.Errorf("Expected %d changes, got %d for %+v:",
  777. testCase.numChanges, len(changes), testCase.opts)
  778. }
  779. }
  780. }
  781. // Some tar archives such as http://haproxy.1wt.eu/download/1.5/src/devel/haproxy-1.5-dev21.tar.gz
  782. // use PAX Global Extended Headers.
  783. // Failing prevents the archives from being uncompressed during ADD
  784. func TestTypeXGlobalHeaderDoesNotFail(t *testing.T) {
  785. hdr := tar.Header{Typeflag: tar.TypeXGlobalHeader}
  786. tmpDir, err := os.MkdirTemp("", "docker-test-archive-pax-test")
  787. if err != nil {
  788. t.Fatal(err)
  789. }
  790. defer os.RemoveAll(tmpDir)
  791. err = createTarFile(filepath.Join(tmpDir, "pax_global_header"), tmpDir, &hdr, nil, nil)
  792. if err != nil {
  793. t.Fatal(err)
  794. }
  795. }
  796. // Some tar have both GNU specific (huge uid) and Ustar specific (long name) things.
  797. // Not supposed to happen (should use PAX instead of Ustar for long name) but it does and it should still work.
  798. func TestUntarUstarGnuConflict(t *testing.T) {
  799. f, err := os.Open("testdata/broken.tar")
  800. if err != nil {
  801. t.Fatal(err)
  802. }
  803. defer f.Close()
  804. found := false
  805. tr := tar.NewReader(f)
  806. // Iterate through the files in the archive.
  807. for {
  808. hdr, err := tr.Next()
  809. if err == io.EOF {
  810. // end of tar archive
  811. break
  812. }
  813. if err != nil {
  814. t.Fatal(err)
  815. }
  816. if hdr.Name == "root/.cpanm/work/1395823785.24209/Plack-1.0030/blib/man3/Plack::Middleware::LighttpdScriptNameFix.3pm" {
  817. found = true
  818. break
  819. }
  820. }
  821. if !found {
  822. t.Fatalf("%s not found in the archive", "root/.cpanm/work/1395823785.24209/Plack-1.0030/blib/man3/Plack::Middleware::LighttpdScriptNameFix.3pm")
  823. }
  824. }
  825. func prepareUntarSourceDirectory(numberOfFiles int, targetPath string, makeLinks bool) (int, error) {
  826. fileData := []byte("fooo")
  827. for n := 0; n < numberOfFiles; n++ {
  828. fileName := fmt.Sprintf("file-%d", n)
  829. if err := os.WriteFile(filepath.Join(targetPath, fileName), fileData, 0o700); err != nil {
  830. return 0, err
  831. }
  832. if makeLinks {
  833. if err := os.Link(filepath.Join(targetPath, fileName), filepath.Join(targetPath, fileName+"-link")); err != nil {
  834. return 0, err
  835. }
  836. }
  837. }
  838. totalSize := numberOfFiles * len(fileData)
  839. return totalSize, nil
  840. }
  841. func BenchmarkTarUntar(b *testing.B) {
  842. origin, err := os.MkdirTemp("", "docker-test-untar-origin")
  843. if err != nil {
  844. b.Fatal(err)
  845. }
  846. tempDir, err := os.MkdirTemp("", "docker-test-untar-destination")
  847. if err != nil {
  848. b.Fatal(err)
  849. }
  850. target := filepath.Join(tempDir, "dest")
  851. n, err := prepareUntarSourceDirectory(100, origin, false)
  852. if err != nil {
  853. b.Fatal(err)
  854. }
  855. defer os.RemoveAll(origin)
  856. defer os.RemoveAll(tempDir)
  857. b.ResetTimer()
  858. b.SetBytes(int64(n))
  859. for n := 0; n < b.N; n++ {
  860. err := defaultTarUntar(origin, target)
  861. if err != nil {
  862. b.Fatal(err)
  863. }
  864. os.RemoveAll(target)
  865. }
  866. }
  867. func BenchmarkTarUntarWithLinks(b *testing.B) {
  868. origin, err := os.MkdirTemp("", "docker-test-untar-origin")
  869. if err != nil {
  870. b.Fatal(err)
  871. }
  872. tempDir, err := os.MkdirTemp("", "docker-test-untar-destination")
  873. if err != nil {
  874. b.Fatal(err)
  875. }
  876. target := filepath.Join(tempDir, "dest")
  877. n, err := prepareUntarSourceDirectory(100, origin, true)
  878. if err != nil {
  879. b.Fatal(err)
  880. }
  881. defer os.RemoveAll(origin)
  882. defer os.RemoveAll(tempDir)
  883. b.ResetTimer()
  884. b.SetBytes(int64(n))
  885. for n := 0; n < b.N; n++ {
  886. err := defaultTarUntar(origin, target)
  887. if err != nil {
  888. b.Fatal(err)
  889. }
  890. os.RemoveAll(target)
  891. }
  892. }
  893. func TestUntarInvalidFilenames(t *testing.T) {
  894. for i, headers := range [][]*tar.Header{
  895. {
  896. {
  897. Name: "../victim/dotdot",
  898. Typeflag: tar.TypeReg,
  899. Mode: 0o644,
  900. },
  901. },
  902. {
  903. {
  904. // Note the leading slash
  905. Name: "/../victim/slash-dotdot",
  906. Typeflag: tar.TypeReg,
  907. Mode: 0o644,
  908. },
  909. },
  910. } {
  911. if err := testBreakout("untar", "docker-TestUntarInvalidFilenames", headers); err != nil {
  912. t.Fatalf("i=%d. %v", i, err)
  913. }
  914. }
  915. }
  916. func TestUntarHardlinkToSymlink(t *testing.T) {
  917. skip.If(t, runtime.GOOS != "windows" && os.Getuid() != 0, "skipping test that requires root")
  918. for i, headers := range [][]*tar.Header{
  919. {
  920. {
  921. Name: "symlink1",
  922. Typeflag: tar.TypeSymlink,
  923. Linkname: "regfile",
  924. Mode: 0o644,
  925. },
  926. {
  927. Name: "symlink2",
  928. Typeflag: tar.TypeLink,
  929. Linkname: "symlink1",
  930. Mode: 0o644,
  931. },
  932. {
  933. Name: "regfile",
  934. Typeflag: tar.TypeReg,
  935. Mode: 0o644,
  936. },
  937. },
  938. } {
  939. if err := testBreakout("untar", "docker-TestUntarHardlinkToSymlink", headers); err != nil {
  940. t.Fatalf("i=%d. %v", i, err)
  941. }
  942. }
  943. }
  944. func TestUntarInvalidHardlink(t *testing.T) {
  945. for i, headers := range [][]*tar.Header{
  946. { // try reading victim/hello (../)
  947. {
  948. Name: "dotdot",
  949. Typeflag: tar.TypeLink,
  950. Linkname: "../victim/hello",
  951. Mode: 0o644,
  952. },
  953. },
  954. { // try reading victim/hello (/../)
  955. {
  956. Name: "slash-dotdot",
  957. Typeflag: tar.TypeLink,
  958. // Note the leading slash
  959. Linkname: "/../victim/hello",
  960. Mode: 0o644,
  961. },
  962. },
  963. { // try writing victim/file
  964. {
  965. Name: "loophole-victim",
  966. Typeflag: tar.TypeLink,
  967. Linkname: "../victim",
  968. Mode: 0o755,
  969. },
  970. {
  971. Name: "loophole-victim/file",
  972. Typeflag: tar.TypeReg,
  973. Mode: 0o644,
  974. },
  975. },
  976. { // try reading victim/hello (hardlink, symlink)
  977. {
  978. Name: "loophole-victim",
  979. Typeflag: tar.TypeLink,
  980. Linkname: "../victim",
  981. Mode: 0o755,
  982. },
  983. {
  984. Name: "symlink",
  985. Typeflag: tar.TypeSymlink,
  986. Linkname: "loophole-victim/hello",
  987. Mode: 0o644,
  988. },
  989. },
  990. { // Try reading victim/hello (hardlink, hardlink)
  991. {
  992. Name: "loophole-victim",
  993. Typeflag: tar.TypeLink,
  994. Linkname: "../victim",
  995. Mode: 0o755,
  996. },
  997. {
  998. Name: "hardlink",
  999. Typeflag: tar.TypeLink,
  1000. Linkname: "loophole-victim/hello",
  1001. Mode: 0o644,
  1002. },
  1003. },
  1004. { // Try removing victim directory (hardlink)
  1005. {
  1006. Name: "loophole-victim",
  1007. Typeflag: tar.TypeLink,
  1008. Linkname: "../victim",
  1009. Mode: 0o755,
  1010. },
  1011. {
  1012. Name: "loophole-victim",
  1013. Typeflag: tar.TypeReg,
  1014. Mode: 0o644,
  1015. },
  1016. },
  1017. } {
  1018. if err := testBreakout("untar", "docker-TestUntarInvalidHardlink", headers); err != nil {
  1019. t.Fatalf("i=%d. %v", i, err)
  1020. }
  1021. }
  1022. }
  1023. func TestUntarInvalidSymlink(t *testing.T) {
  1024. for i, headers := range [][]*tar.Header{
  1025. { // try reading victim/hello (../)
  1026. {
  1027. Name: "dotdot",
  1028. Typeflag: tar.TypeSymlink,
  1029. Linkname: "../victim/hello",
  1030. Mode: 0o644,
  1031. },
  1032. },
  1033. { // try reading victim/hello (/../)
  1034. {
  1035. Name: "slash-dotdot",
  1036. Typeflag: tar.TypeSymlink,
  1037. // Note the leading slash
  1038. Linkname: "/../victim/hello",
  1039. Mode: 0o644,
  1040. },
  1041. },
  1042. { // try writing victim/file
  1043. {
  1044. Name: "loophole-victim",
  1045. Typeflag: tar.TypeSymlink,
  1046. Linkname: "../victim",
  1047. Mode: 0o755,
  1048. },
  1049. {
  1050. Name: "loophole-victim/file",
  1051. Typeflag: tar.TypeReg,
  1052. Mode: 0o644,
  1053. },
  1054. },
  1055. { // try reading victim/hello (symlink, symlink)
  1056. {
  1057. Name: "loophole-victim",
  1058. Typeflag: tar.TypeSymlink,
  1059. Linkname: "../victim",
  1060. Mode: 0o755,
  1061. },
  1062. {
  1063. Name: "symlink",
  1064. Typeflag: tar.TypeSymlink,
  1065. Linkname: "loophole-victim/hello",
  1066. Mode: 0o644,
  1067. },
  1068. },
  1069. { // try reading victim/hello (symlink, hardlink)
  1070. {
  1071. Name: "loophole-victim",
  1072. Typeflag: tar.TypeSymlink,
  1073. Linkname: "../victim",
  1074. Mode: 0o755,
  1075. },
  1076. {
  1077. Name: "hardlink",
  1078. Typeflag: tar.TypeLink,
  1079. Linkname: "loophole-victim/hello",
  1080. Mode: 0o644,
  1081. },
  1082. },
  1083. { // try removing victim directory (symlink)
  1084. {
  1085. Name: "loophole-victim",
  1086. Typeflag: tar.TypeSymlink,
  1087. Linkname: "../victim",
  1088. Mode: 0o755,
  1089. },
  1090. {
  1091. Name: "loophole-victim",
  1092. Typeflag: tar.TypeReg,
  1093. Mode: 0o644,
  1094. },
  1095. },
  1096. { // try writing to victim/newdir/newfile with a symlink in the path
  1097. {
  1098. // this header needs to be before the next one, or else there is an error
  1099. Name: "dir/loophole",
  1100. Typeflag: tar.TypeSymlink,
  1101. Linkname: "../../victim",
  1102. Mode: 0o755,
  1103. },
  1104. {
  1105. Name: "dir/loophole/newdir/newfile",
  1106. Typeflag: tar.TypeReg,
  1107. Mode: 0o644,
  1108. },
  1109. },
  1110. } {
  1111. if err := testBreakout("untar", "docker-TestUntarInvalidSymlink", headers); err != nil {
  1112. t.Fatalf("i=%d. %v", i, err)
  1113. }
  1114. }
  1115. }
  1116. func TestTempArchiveCloseMultipleTimes(t *testing.T) {
  1117. reader := io.NopCloser(strings.NewReader("hello"))
  1118. tempArchive, err := NewTempArchive(reader, "")
  1119. assert.NilError(t, err)
  1120. buf := make([]byte, 10)
  1121. n, err := tempArchive.Read(buf)
  1122. assert.NilError(t, err)
  1123. if n != 5 {
  1124. t.Fatalf("Expected to read 5 bytes. Read %d instead", n)
  1125. }
  1126. for i := 0; i < 3; i++ {
  1127. if err = tempArchive.Close(); err != nil {
  1128. t.Fatalf("i=%d. Unexpected error closing temp archive: %v", i, err)
  1129. }
  1130. }
  1131. }
  1132. // TestXGlobalNoParent is a regression test to check parent directories are not created for PAX headers
  1133. func TestXGlobalNoParent(t *testing.T) {
  1134. buf := &bytes.Buffer{}
  1135. w := tar.NewWriter(buf)
  1136. err := w.WriteHeader(&tar.Header{
  1137. Name: "foo/bar",
  1138. Typeflag: tar.TypeXGlobalHeader,
  1139. })
  1140. assert.NilError(t, err)
  1141. tmpDir, err := os.MkdirTemp("", "pax-test")
  1142. assert.NilError(t, err)
  1143. defer os.RemoveAll(tmpDir)
  1144. err = Untar(buf, tmpDir, nil)
  1145. assert.NilError(t, err)
  1146. _, err = os.Lstat(filepath.Join(tmpDir, "foo"))
  1147. assert.Check(t, err != nil)
  1148. assert.Check(t, errors.Is(err, os.ErrNotExist))
  1149. }
  1150. // TestImpliedDirectoryPermissions ensures that directories implied by paths in the tar file, but without their own
  1151. // header entries are created recursively with the default mode (permissions) stored in ImpliedDirectoryMode. This test
  1152. // also verifies that the permissions of explicit directories are respected.
  1153. func TestImpliedDirectoryPermissions(t *testing.T) {
  1154. skip.If(t, runtime.GOOS == "windows", "skipping test that requires Unix permissions")
  1155. buf := &bytes.Buffer{}
  1156. headers := []tar.Header{{
  1157. Name: "deeply/nested/and/implied",
  1158. }, {
  1159. Name: "explicit/",
  1160. Mode: 0o644,
  1161. }, {
  1162. Name: "explicit/permissions/",
  1163. Mode: 0o600,
  1164. }, {
  1165. Name: "explicit/permissions/specified",
  1166. Mode: 0o400,
  1167. }}
  1168. w := tar.NewWriter(buf)
  1169. for _, header := range headers {
  1170. err := w.WriteHeader(&header)
  1171. assert.NilError(t, err)
  1172. }
  1173. tmpDir := t.TempDir()
  1174. err := Untar(buf, tmpDir, nil)
  1175. assert.NilError(t, err)
  1176. assertMode := func(path string, expected uint32) {
  1177. t.Helper()
  1178. stat, err := os.Lstat(filepath.Join(tmpDir, path))
  1179. assert.Check(t, err)
  1180. assert.Check(t, is.Equal(stat.Mode().Perm(), fs.FileMode(expected)))
  1181. }
  1182. assertMode("deeply", ImpliedDirectoryMode)
  1183. assertMode("deeply/nested", ImpliedDirectoryMode)
  1184. assertMode("deeply/nested/and", ImpliedDirectoryMode)
  1185. assertMode("explicit", 0o644)
  1186. assertMode("explicit/permissions", 0o600)
  1187. assertMode("explicit/permissions/specified", 0o400)
  1188. }
  1189. func TestReplaceFileTarWrapper(t *testing.T) {
  1190. filesInArchive := 20
  1191. testcases := []struct {
  1192. doc string
  1193. filename string
  1194. modifier TarModifierFunc
  1195. expected string
  1196. fileCount int
  1197. }{
  1198. {
  1199. doc: "Modifier creates a new file",
  1200. filename: "newfile",
  1201. modifier: createModifier(t),
  1202. expected: "the new content",
  1203. fileCount: filesInArchive + 1,
  1204. },
  1205. {
  1206. doc: "Modifier replaces a file",
  1207. filename: "file-2",
  1208. modifier: createOrReplaceModifier,
  1209. expected: "the new content",
  1210. fileCount: filesInArchive,
  1211. },
  1212. {
  1213. doc: "Modifier replaces the last file",
  1214. filename: fmt.Sprintf("file-%d", filesInArchive-1),
  1215. modifier: createOrReplaceModifier,
  1216. expected: "the new content",
  1217. fileCount: filesInArchive,
  1218. },
  1219. {
  1220. doc: "Modifier appends to a file",
  1221. filename: "file-3",
  1222. modifier: appendModifier,
  1223. expected: "fooo\nnext line",
  1224. fileCount: filesInArchive,
  1225. },
  1226. }
  1227. for _, testcase := range testcases {
  1228. sourceArchive, cleanup := buildSourceArchive(t, filesInArchive)
  1229. defer cleanup()
  1230. resultArchive := ReplaceFileTarWrapper(
  1231. sourceArchive,
  1232. map[string]TarModifierFunc{testcase.filename: testcase.modifier})
  1233. actual := readFileFromArchive(t, resultArchive, testcase.filename, testcase.fileCount, testcase.doc)
  1234. assert.Check(t, is.Equal(testcase.expected, actual), testcase.doc)
  1235. }
  1236. }
  1237. // TestPrefixHeaderReadable tests that files that could be created with the
  1238. // version of this package that was built with <=go17 are still readable.
  1239. func TestPrefixHeaderReadable(t *testing.T) {
  1240. skip.If(t, runtime.GOOS != "windows" && os.Getuid() != 0, "skipping test that requires root")
  1241. skip.If(t, userns.RunningInUserNS(), "skipping test that requires more than 010000000 UIDs, which is unlikely to be satisfied when running in userns")
  1242. // https://gist.github.com/stevvooe/e2a790ad4e97425896206c0816e1a882#file-out-go
  1243. testFile := []byte("\x1f\x8b\x08\x08\x44\x21\x68\x59\x00\x03\x74\x2e\x74\x61\x72\x00\x4b\xcb\xcf\x67\xa0\x35\x30\x80\x00\x86\x06\x10\x47\x01\xc1\x37\x40\x00\x54\xb6\xb1\xa1\xa9\x99\x09\x48\x25\x1d\x40\x69\x71\x49\x62\x91\x02\xe5\x76\xa1\x79\x84\x21\x91\xd6\x80\x72\xaf\x8f\x82\x51\x30\x0a\x46\x36\x00\x00\xf0\x1c\x1e\x95\x00\x06\x00\x00")
  1244. tmpDir, err := os.MkdirTemp("", "prefix-test")
  1245. assert.NilError(t, err)
  1246. defer os.RemoveAll(tmpDir)
  1247. err = Untar(bytes.NewReader(testFile), tmpDir, nil)
  1248. assert.NilError(t, err)
  1249. baseName := "foo"
  1250. pth := strings.Repeat("a", 100-len(baseName)) + "/" + baseName
  1251. _, err = os.Lstat(filepath.Join(tmpDir, pth))
  1252. assert.NilError(t, err)
  1253. }
  1254. func buildSourceArchive(t *testing.T, numberOfFiles int) (io.ReadCloser, func()) {
  1255. srcDir, err := os.MkdirTemp("", "docker-test-srcDir")
  1256. assert.NilError(t, err)
  1257. _, err = prepareUntarSourceDirectory(numberOfFiles, srcDir, false)
  1258. assert.NilError(t, err)
  1259. sourceArchive, err := TarWithOptions(srcDir, &TarOptions{})
  1260. assert.NilError(t, err)
  1261. return sourceArchive, func() {
  1262. os.RemoveAll(srcDir)
  1263. sourceArchive.Close()
  1264. }
  1265. }
  1266. func createOrReplaceModifier(path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
  1267. return &tar.Header{
  1268. Mode: 0o600,
  1269. Typeflag: tar.TypeReg,
  1270. }, []byte("the new content"), nil
  1271. }
  1272. func createModifier(t *testing.T) TarModifierFunc {
  1273. return func(path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
  1274. assert.Check(t, is.Nil(content))
  1275. return createOrReplaceModifier(path, header, content)
  1276. }
  1277. }
  1278. func appendModifier(path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
  1279. buffer := bytes.Buffer{}
  1280. if content != nil {
  1281. if _, err := buffer.ReadFrom(content); err != nil {
  1282. return nil, nil, err
  1283. }
  1284. }
  1285. buffer.WriteString("\nnext line")
  1286. return &tar.Header{Mode: 0o600, Typeflag: tar.TypeReg}, buffer.Bytes(), nil
  1287. }
  1288. func readFileFromArchive(t *testing.T, archive io.ReadCloser, name string, expectedCount int, doc string) string {
  1289. skip.If(t, runtime.GOOS != "windows" && os.Getuid() != 0, "skipping test that requires root")
  1290. destDir, err := os.MkdirTemp("", "docker-test-destDir")
  1291. assert.NilError(t, err)
  1292. defer os.RemoveAll(destDir)
  1293. err = Untar(archive, destDir, nil)
  1294. assert.NilError(t, err)
  1295. files, _ := os.ReadDir(destDir)
  1296. assert.Check(t, is.Len(files, expectedCount), doc)
  1297. content, err := os.ReadFile(filepath.Join(destDir, name))
  1298. assert.Check(t, err)
  1299. return string(content)
  1300. }
  1301. func TestDisablePigz(t *testing.T) {
  1302. _, err := exec.LookPath("unpigz")
  1303. if err != nil {
  1304. t.Log("Test will not check full path when Pigz not installed")
  1305. }
  1306. t.Setenv("MOBY_DISABLE_PIGZ", "true")
  1307. r := testDecompressStream(t, "gz", "gzip -f")
  1308. // For the bufio pool
  1309. outsideReaderCloserWrapper := r.(*ioutils.ReadCloserWrapper)
  1310. // For the context canceller
  1311. contextReaderCloserWrapper := outsideReaderCloserWrapper.Reader.(*ioutils.ReadCloserWrapper)
  1312. assert.Equal(t, reflect.TypeOf(contextReaderCloserWrapper.Reader), reflect.TypeOf(&gzip.Reader{}))
  1313. }
  1314. func TestPigz(t *testing.T) {
  1315. r := testDecompressStream(t, "gz", "gzip -f")
  1316. // For the bufio pool
  1317. outsideReaderCloserWrapper := r.(*ioutils.ReadCloserWrapper)
  1318. // For the context canceller
  1319. contextReaderCloserWrapper := outsideReaderCloserWrapper.Reader.(*ioutils.ReadCloserWrapper)
  1320. _, err := exec.LookPath("unpigz")
  1321. if err == nil {
  1322. t.Log("Tested whether Pigz is used, as it installed")
  1323. // For the command wait wrapper
  1324. cmdWaitCloserWrapper := contextReaderCloserWrapper.Reader.(*ioutils.ReadCloserWrapper)
  1325. assert.Equal(t, reflect.TypeOf(cmdWaitCloserWrapper.Reader), reflect.TypeOf(&io.PipeReader{}))
  1326. } else {
  1327. t.Log("Tested whether Pigz is not used, as it not installed")
  1328. assert.Equal(t, reflect.TypeOf(contextReaderCloserWrapper.Reader), reflect.TypeOf(&gzip.Reader{}))
  1329. }
  1330. }
  1331. func TestNosysFileInfo(t *testing.T) {
  1332. st, err := os.Stat("archive_test.go")
  1333. assert.NilError(t, err)
  1334. h, err := tar.FileInfoHeader(nosysFileInfo{st}, "")
  1335. assert.NilError(t, err)
  1336. assert.Check(t, h.Uname == "")
  1337. assert.Check(t, h.Gname == "")
  1338. }