changes.go 9.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369
  1. package archive
  2. import (
  3. "bytes"
  4. "fmt"
  5. "io"
  6. "io/ioutil"
  7. "os"
  8. "path/filepath"
  9. "sort"
  10. "strings"
  11. "syscall"
  12. "time"
  13. "github.com/docker/docker/vendor/src/code.google.com/p/go/src/pkg/archive/tar"
  14. "github.com/Sirupsen/logrus"
  15. "github.com/docker/docker/pkg/pools"
  16. "github.com/docker/docker/pkg/system"
  17. )
  18. type ChangeType int
  19. const (
  20. ChangeModify = iota
  21. ChangeAdd
  22. ChangeDelete
  23. )
  24. type Change struct {
  25. Path string
  26. Kind ChangeType
  27. }
  28. func (change *Change) String() string {
  29. var kind string
  30. switch change.Kind {
  31. case ChangeModify:
  32. kind = "C"
  33. case ChangeAdd:
  34. kind = "A"
  35. case ChangeDelete:
  36. kind = "D"
  37. }
  38. return fmt.Sprintf("%s %s", kind, change.Path)
  39. }
  40. // for sort.Sort
  41. type changesByPath []Change
  42. func (c changesByPath) Less(i, j int) bool { return c[i].Path < c[j].Path }
  43. func (c changesByPath) Len() int { return len(c) }
  44. func (c changesByPath) Swap(i, j int) { c[j], c[i] = c[i], c[j] }
  45. // Gnu tar and the go tar writer don't have sub-second mtime
  46. // precision, which is problematic when we apply changes via tar
  47. // files, we handle this by comparing for exact times, *or* same
  48. // second count and either a or b having exactly 0 nanoseconds
  49. func sameFsTime(a, b time.Time) bool {
  50. return a == b ||
  51. (a.Unix() == b.Unix() &&
  52. (a.Nanosecond() == 0 || b.Nanosecond() == 0))
  53. }
  54. func sameFsTimeSpec(a, b syscall.Timespec) bool {
  55. return a.Sec == b.Sec &&
  56. (a.Nsec == b.Nsec || a.Nsec == 0 || b.Nsec == 0)
  57. }
  58. // Changes walks the path rw and determines changes for the files in the path,
  59. // with respect to the parent layers
  60. func Changes(layers []string, rw string) ([]Change, error) {
  61. var changes []Change
  62. err := filepath.Walk(rw, func(path string, f os.FileInfo, err error) error {
  63. if err != nil {
  64. return err
  65. }
  66. // Rebase path
  67. path, err = filepath.Rel(rw, path)
  68. if err != nil {
  69. return err
  70. }
  71. path = filepath.Join("/", path)
  72. // Skip root
  73. if path == "/" {
  74. return nil
  75. }
  76. // Skip AUFS metadata
  77. if matched, err := filepath.Match("/.wh..wh.*", path); err != nil || matched {
  78. return err
  79. }
  80. change := Change{
  81. Path: path,
  82. }
  83. // Find out what kind of modification happened
  84. file := filepath.Base(path)
  85. // If there is a whiteout, then the file was removed
  86. if strings.HasPrefix(file, ".wh.") {
  87. originalFile := file[len(".wh."):]
  88. change.Path = filepath.Join(filepath.Dir(path), originalFile)
  89. change.Kind = ChangeDelete
  90. } else {
  91. // Otherwise, the file was added
  92. change.Kind = ChangeAdd
  93. // ...Unless it already existed in a top layer, in which case, it's a modification
  94. for _, layer := range layers {
  95. stat, err := os.Stat(filepath.Join(layer, path))
  96. if err != nil && !os.IsNotExist(err) {
  97. return err
  98. }
  99. if err == nil {
  100. // The file existed in the top layer, so that's a modification
  101. // However, if it's a directory, maybe it wasn't actually modified.
  102. // If you modify /foo/bar/baz, then /foo will be part of the changed files only because it's the parent of bar
  103. if stat.IsDir() && f.IsDir() {
  104. if f.Size() == stat.Size() && f.Mode() == stat.Mode() && sameFsTime(f.ModTime(), stat.ModTime()) {
  105. // Both directories are the same, don't record the change
  106. return nil
  107. }
  108. }
  109. change.Kind = ChangeModify
  110. break
  111. }
  112. }
  113. }
  114. // Record change
  115. changes = append(changes, change)
  116. return nil
  117. })
  118. if err != nil && !os.IsNotExist(err) {
  119. return nil, err
  120. }
  121. return changes, nil
  122. }
  123. type FileInfo struct {
  124. parent *FileInfo
  125. name string
  126. stat *system.Stat_t
  127. children map[string]*FileInfo
  128. capability []byte
  129. added bool
  130. }
  131. func (root *FileInfo) LookUp(path string) *FileInfo {
  132. parent := root
  133. if path == "/" {
  134. return root
  135. }
  136. pathElements := strings.Split(path, "/")
  137. for _, elem := range pathElements {
  138. if elem != "" {
  139. child := parent.children[elem]
  140. if child == nil {
  141. return nil
  142. }
  143. parent = child
  144. }
  145. }
  146. return parent
  147. }
  148. func (info *FileInfo) path() string {
  149. if info.parent == nil {
  150. return "/"
  151. }
  152. return filepath.Join(info.parent.path(), info.name)
  153. }
  154. func (info *FileInfo) isDir() bool {
  155. return info.parent == nil || info.stat.Mode()&syscall.S_IFDIR != 0
  156. }
  157. func (info *FileInfo) addChanges(oldInfo *FileInfo, changes *[]Change) {
  158. sizeAtEntry := len(*changes)
  159. if oldInfo == nil {
  160. // add
  161. change := Change{
  162. Path: info.path(),
  163. Kind: ChangeAdd,
  164. }
  165. *changes = append(*changes, change)
  166. info.added = true
  167. }
  168. // We make a copy so we can modify it to detect additions
  169. // also, we only recurse on the old dir if the new info is a directory
  170. // otherwise any previous delete/change is considered recursive
  171. oldChildren := make(map[string]*FileInfo)
  172. if oldInfo != nil && info.isDir() {
  173. for k, v := range oldInfo.children {
  174. oldChildren[k] = v
  175. }
  176. }
  177. for name, newChild := range info.children {
  178. oldChild, _ := oldChildren[name]
  179. if oldChild != nil {
  180. // change?
  181. oldStat := oldChild.stat
  182. newStat := newChild.stat
  183. // Note: We can't compare inode or ctime or blocksize here, because these change
  184. // when copying a file into a container. However, that is not generally a problem
  185. // because any content change will change mtime, and any status change should
  186. // be visible when actually comparing the stat fields. The only time this
  187. // breaks down is if some code intentionally hides a change by setting
  188. // back mtime
  189. if oldStat.Mode() != newStat.Mode() ||
  190. oldStat.Uid() != newStat.Uid() ||
  191. oldStat.Gid() != newStat.Gid() ||
  192. oldStat.Rdev() != newStat.Rdev() ||
  193. // Don't look at size for dirs, its not a good measure of change
  194. (oldStat.Mode()&syscall.S_IFDIR != syscall.S_IFDIR &&
  195. (!sameFsTimeSpec(oldStat.Mtim(), newStat.Mtim()) || (oldStat.Size() != newStat.Size()))) ||
  196. bytes.Compare(oldChild.capability, newChild.capability) != 0 {
  197. change := Change{
  198. Path: newChild.path(),
  199. Kind: ChangeModify,
  200. }
  201. *changes = append(*changes, change)
  202. newChild.added = true
  203. }
  204. // Remove from copy so we can detect deletions
  205. delete(oldChildren, name)
  206. }
  207. newChild.addChanges(oldChild, changes)
  208. }
  209. for _, oldChild := range oldChildren {
  210. // delete
  211. change := Change{
  212. Path: oldChild.path(),
  213. Kind: ChangeDelete,
  214. }
  215. *changes = append(*changes, change)
  216. }
  217. // If there were changes inside this directory, we need to add it, even if the directory
  218. // itself wasn't changed. This is needed to properly save and restore filesystem permissions.
  219. if len(*changes) > sizeAtEntry && info.isDir() && !info.added && info.path() != "/" {
  220. change := Change{
  221. Path: info.path(),
  222. Kind: ChangeModify,
  223. }
  224. // Let's insert the directory entry before the recently added entries located inside this dir
  225. *changes = append(*changes, change) // just to resize the slice, will be overwritten
  226. copy((*changes)[sizeAtEntry+1:], (*changes)[sizeAtEntry:])
  227. (*changes)[sizeAtEntry] = change
  228. }
  229. }
  230. func (info *FileInfo) Changes(oldInfo *FileInfo) []Change {
  231. var changes []Change
  232. info.addChanges(oldInfo, &changes)
  233. return changes
  234. }
  235. func newRootFileInfo() *FileInfo {
  236. root := &FileInfo{
  237. name: "/",
  238. children: make(map[string]*FileInfo),
  239. }
  240. return root
  241. }
  242. // ChangesDirs compares two directories and generates an array of Change objects describing the changes.
  243. // If oldDir is "", then all files in newDir will be Add-Changes.
  244. func ChangesDirs(newDir, oldDir string) ([]Change, error) {
  245. var (
  246. oldRoot, newRoot *FileInfo
  247. )
  248. if oldDir == "" {
  249. emptyDir, err := ioutil.TempDir("", "empty")
  250. if err != nil {
  251. return nil, err
  252. }
  253. defer os.Remove(emptyDir)
  254. oldDir = emptyDir
  255. }
  256. oldRoot, newRoot, err := collectFileInfoForChanges(oldDir, newDir)
  257. if err != nil {
  258. return nil, err
  259. }
  260. return newRoot.Changes(oldRoot), nil
  261. }
  262. // ChangesSize calculates the size in bytes of the provided changes, based on newDir.
  263. func ChangesSize(newDir string, changes []Change) int64 {
  264. var size int64
  265. for _, change := range changes {
  266. if change.Kind == ChangeModify || change.Kind == ChangeAdd {
  267. file := filepath.Join(newDir, change.Path)
  268. fileInfo, _ := os.Lstat(file)
  269. if fileInfo != nil && !fileInfo.IsDir() {
  270. size += fileInfo.Size()
  271. }
  272. }
  273. }
  274. return size
  275. }
  276. // ExportChanges produces an Archive from the provided changes, relative to dir.
  277. func ExportChanges(dir string, changes []Change) (Archive, error) {
  278. reader, writer := io.Pipe()
  279. go func() {
  280. ta := &tarAppender{
  281. TarWriter: tar.NewWriter(writer),
  282. Buffer: pools.BufioWriter32KPool.Get(nil),
  283. SeenFiles: make(map[uint64]string),
  284. }
  285. // this buffer is needed for the duration of this piped stream
  286. defer pools.BufioWriter32KPool.Put(ta.Buffer)
  287. sort.Sort(changesByPath(changes))
  288. // In general we log errors here but ignore them because
  289. // during e.g. a diff operation the container can continue
  290. // mutating the filesystem and we can see transient errors
  291. // from this
  292. for _, change := range changes {
  293. if change.Kind == ChangeDelete {
  294. whiteOutDir := filepath.Dir(change.Path)
  295. whiteOutBase := filepath.Base(change.Path)
  296. whiteOut := filepath.Join(whiteOutDir, ".wh."+whiteOutBase)
  297. timestamp := time.Now()
  298. hdr := &tar.Header{
  299. Name: whiteOut[1:],
  300. Size: 0,
  301. ModTime: timestamp,
  302. AccessTime: timestamp,
  303. ChangeTime: timestamp,
  304. }
  305. if err := ta.TarWriter.WriteHeader(hdr); err != nil {
  306. logrus.Debugf("Can't write whiteout header: %s", err)
  307. }
  308. } else {
  309. path := filepath.Join(dir, change.Path)
  310. if err := ta.addTarFile(path, change.Path[1:]); err != nil {
  311. logrus.Debugf("Can't add file %s to tar: %s", path, err)
  312. }
  313. }
  314. }
  315. // Make sure to check the error on Close.
  316. if err := ta.TarWriter.Close(); err != nil {
  317. logrus.Debugf("Can't close layer: %s", err)
  318. }
  319. if err := writer.Close(); err != nil {
  320. logrus.Debugf("failed close Changes writer: %s", err)
  321. }
  322. }()
  323. return reader, nil
  324. }