deviceset.go 33 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306
  1. // +build linux
  2. package devmapper
  3. import (
  4. "encoding/json"
  5. "errors"
  6. "fmt"
  7. "io"
  8. "io/ioutil"
  9. "os"
  10. "os/exec"
  11. "path"
  12. "path/filepath"
  13. "strconv"
  14. "strings"
  15. "sync"
  16. "syscall"
  17. "time"
  18. log "github.com/Sirupsen/logrus"
  19. "github.com/docker/docker/daemon/graphdriver"
  20. "github.com/docker/docker/pkg/parsers"
  21. "github.com/docker/docker/pkg/units"
  22. "github.com/docker/libcontainer/label"
  23. )
  24. var (
  25. DefaultDataLoopbackSize int64 = 100 * 1024 * 1024 * 1024
  26. DefaultMetaDataLoopbackSize int64 = 2 * 1024 * 1024 * 1024
  27. DefaultBaseFsSize uint64 = 10 * 1024 * 1024 * 1024
  28. DefaultThinpBlockSize uint32 = 128 // 64K = 128 512b sectors
  29. )
  30. type DevInfo struct {
  31. Hash string `json:"-"`
  32. DeviceId int `json:"device_id"`
  33. Size uint64 `json:"size"`
  34. TransactionId uint64 `json:"transaction_id"`
  35. Initialized bool `json:"initialized"`
  36. devices *DeviceSet `json:"-"`
  37. mountCount int `json:"-"`
  38. mountPath string `json:"-"`
  39. // The global DeviceSet lock guarantees that we serialize all
  40. // the calls to libdevmapper (which is not threadsafe), but we
  41. // sometimes release that lock while sleeping. In that case
  42. // this per-device lock is still held, protecting against
  43. // other accesses to the device that we're doing the wait on.
  44. //
  45. // WARNING: In order to avoid AB-BA deadlocks when releasing
  46. // the global lock while holding the per-device locks all
  47. // device locks must be aquired *before* the device lock, and
  48. // multiple device locks should be aquired parent before child.
  49. lock sync.Mutex `json:"-"`
  50. }
  51. type MetaData struct {
  52. Devices map[string]*DevInfo `json:"Devices"`
  53. devicesLock sync.Mutex `json:"-"` // Protects all read/writes to Devices map
  54. }
  55. type DeviceSet struct {
  56. MetaData `json:"-"`
  57. sync.Mutex `json:"-"` // Protects Devices map and serializes calls into libdevmapper
  58. root string `json:"-"`
  59. devicePrefix string `json:"-"`
  60. TransactionId uint64 `json:"-"`
  61. NewTransactionId uint64 `json:"-"`
  62. NextDeviceId int `json:"next_device_id"`
  63. // Options
  64. dataLoopbackSize int64 `json:"-"`
  65. metaDataLoopbackSize int64 `json:"-"`
  66. baseFsSize uint64 `json:"-"`
  67. filesystem string `json:"-"`
  68. mountOptions string `json:"-"`
  69. mkfsArgs []string `json:"-"`
  70. dataDevice string `json:"-"`
  71. metadataDevice string `json:"-"`
  72. doBlkDiscard bool `json:"-"`
  73. thinpBlockSize uint32 `json:"-"`
  74. }
  75. type DiskUsage struct {
  76. Used uint64
  77. Total uint64
  78. }
  79. type Status struct {
  80. PoolName string
  81. DataLoopback string
  82. MetadataLoopback string
  83. Data DiskUsage
  84. Metadata DiskUsage
  85. SectorSize uint64
  86. }
  87. type DevStatus struct {
  88. DeviceId int
  89. Size uint64
  90. TransactionId uint64
  91. SizeInSectors uint64
  92. MappedSectors uint64
  93. HighestMappedSector uint64
  94. }
  95. func getDevName(name string) string {
  96. return "/dev/mapper/" + name
  97. }
  98. func (info *DevInfo) Name() string {
  99. hash := info.Hash
  100. if hash == "" {
  101. hash = "base"
  102. }
  103. return fmt.Sprintf("%s-%s", info.devices.devicePrefix, hash)
  104. }
  105. func (info *DevInfo) DevName() string {
  106. return getDevName(info.Name())
  107. }
  108. func (devices *DeviceSet) loopbackDir() string {
  109. return path.Join(devices.root, "devicemapper")
  110. }
  111. func (devices *DeviceSet) metadataDir() string {
  112. return path.Join(devices.root, "metadata")
  113. }
  114. func (devices *DeviceSet) metadataFile(info *DevInfo) string {
  115. file := info.Hash
  116. if file == "" {
  117. file = "base"
  118. }
  119. return path.Join(devices.metadataDir(), file)
  120. }
  121. func (devices *DeviceSet) deviceSetMetaFile() string {
  122. return path.Join(devices.metadataDir(), "deviceset-metadata")
  123. }
  124. func (devices *DeviceSet) oldMetadataFile() string {
  125. return path.Join(devices.loopbackDir(), "json")
  126. }
  127. func (devices *DeviceSet) getPoolName() string {
  128. return devices.devicePrefix + "-pool"
  129. }
  130. func (devices *DeviceSet) getPoolDevName() string {
  131. return getDevName(devices.getPoolName())
  132. }
  133. func (devices *DeviceSet) hasImage(name string) bool {
  134. dirname := devices.loopbackDir()
  135. filename := path.Join(dirname, name)
  136. _, err := os.Stat(filename)
  137. return err == nil
  138. }
  139. // ensureImage creates a sparse file of <size> bytes at the path
  140. // <root>/devicemapper/<name>.
  141. // If the file already exists, it does nothing.
  142. // Either way it returns the full path.
  143. func (devices *DeviceSet) ensureImage(name string, size int64) (string, error) {
  144. dirname := devices.loopbackDir()
  145. filename := path.Join(dirname, name)
  146. if err := os.MkdirAll(dirname, 0700); err != nil && !os.IsExist(err) {
  147. return "", err
  148. }
  149. if _, err := os.Stat(filename); err != nil {
  150. if !os.IsNotExist(err) {
  151. return "", err
  152. }
  153. log.Debugf("Creating loopback file %s for device-manage use", filename)
  154. file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0600)
  155. if err != nil {
  156. return "", err
  157. }
  158. defer file.Close()
  159. if err = file.Truncate(size); err != nil {
  160. return "", err
  161. }
  162. }
  163. return filename, nil
  164. }
  165. func (devices *DeviceSet) allocateTransactionId() uint64 {
  166. devices.NewTransactionId = devices.NewTransactionId + 1
  167. return devices.NewTransactionId
  168. }
  169. func (devices *DeviceSet) removeMetadata(info *DevInfo) error {
  170. if err := os.RemoveAll(devices.metadataFile(info)); err != nil {
  171. return fmt.Errorf("Error removing metadata file %s: %s", devices.metadataFile(info), err)
  172. }
  173. return nil
  174. }
  175. // Given json data and file path, write it to disk
  176. func (devices *DeviceSet) writeMetaFile(jsonData []byte, filePath string) error {
  177. tmpFile, err := ioutil.TempFile(devices.metadataDir(), ".tmp")
  178. if err != nil {
  179. return fmt.Errorf("Error creating metadata file: %s", err)
  180. }
  181. n, err := tmpFile.Write(jsonData)
  182. if err != nil {
  183. return fmt.Errorf("Error writing metadata to %s: %s", tmpFile.Name(), err)
  184. }
  185. if n < len(jsonData) {
  186. return io.ErrShortWrite
  187. }
  188. if err := tmpFile.Sync(); err != nil {
  189. return fmt.Errorf("Error syncing metadata file %s: %s", tmpFile.Name(), err)
  190. }
  191. if err := tmpFile.Close(); err != nil {
  192. return fmt.Errorf("Error closing metadata file %s: %s", tmpFile.Name(), err)
  193. }
  194. if err := os.Rename(tmpFile.Name(), filePath); err != nil {
  195. return fmt.Errorf("Error committing metadata file %s: %s", tmpFile.Name(), err)
  196. }
  197. return nil
  198. }
  199. func (devices *DeviceSet) saveMetadata(info *DevInfo) error {
  200. jsonData, err := json.Marshal(info)
  201. if err != nil {
  202. return fmt.Errorf("Error encoding metadata to json: %s", err)
  203. }
  204. err = devices.writeMetaFile(jsonData, devices.metadataFile(info))
  205. if err != nil {
  206. return err
  207. }
  208. if devices.NewTransactionId != devices.TransactionId {
  209. if err = setTransactionId(devices.getPoolDevName(), devices.TransactionId, devices.NewTransactionId); err != nil {
  210. return fmt.Errorf("Error setting devmapper transition ID: %s", err)
  211. }
  212. devices.TransactionId = devices.NewTransactionId
  213. }
  214. return nil
  215. }
  216. func (devices *DeviceSet) lookupDevice(hash string) (*DevInfo, error) {
  217. devices.devicesLock.Lock()
  218. defer devices.devicesLock.Unlock()
  219. info := devices.Devices[hash]
  220. if info == nil {
  221. info = devices.loadMetadata(hash)
  222. if info == nil {
  223. return nil, fmt.Errorf("Unknown device %s", hash)
  224. }
  225. devices.Devices[hash] = info
  226. }
  227. return info, nil
  228. }
  229. func (devices *DeviceSet) registerDevice(id int, hash string, size uint64) (*DevInfo, error) {
  230. log.Debugf("registerDevice(%v, %v)", id, hash)
  231. info := &DevInfo{
  232. Hash: hash,
  233. DeviceId: id,
  234. Size: size,
  235. TransactionId: devices.allocateTransactionId(),
  236. Initialized: false,
  237. devices: devices,
  238. }
  239. devices.devicesLock.Lock()
  240. devices.Devices[hash] = info
  241. devices.devicesLock.Unlock()
  242. if err := devices.saveMetadata(info); err != nil {
  243. // Try to remove unused device
  244. devices.devicesLock.Lock()
  245. delete(devices.Devices, hash)
  246. devices.devicesLock.Unlock()
  247. return nil, err
  248. }
  249. return info, nil
  250. }
  251. func (devices *DeviceSet) activateDeviceIfNeeded(info *DevInfo) error {
  252. log.Debugf("activateDeviceIfNeeded(%v)", info.Hash)
  253. if devinfo, _ := getInfo(info.Name()); devinfo != nil && devinfo.Exists != 0 {
  254. return nil
  255. }
  256. return activateDevice(devices.getPoolDevName(), info.Name(), info.DeviceId, info.Size)
  257. }
  258. func (devices *DeviceSet) createFilesystem(info *DevInfo) error {
  259. devname := info.DevName()
  260. args := []string{}
  261. for _, arg := range devices.mkfsArgs {
  262. args = append(args, arg)
  263. }
  264. args = append(args, devname)
  265. var err error
  266. switch devices.filesystem {
  267. case "xfs":
  268. err = exec.Command("mkfs.xfs", args...).Run()
  269. case "ext4":
  270. err = exec.Command("mkfs.ext4", append([]string{"-E", "nodiscard,lazy_itable_init=0,lazy_journal_init=0"}, args...)...).Run()
  271. if err != nil {
  272. err = exec.Command("mkfs.ext4", append([]string{"-E", "nodiscard,lazy_itable_init=0"}, args...)...).Run()
  273. }
  274. if err != nil {
  275. return err
  276. }
  277. err = exec.Command("tune2fs", append([]string{"-c", "-1", "-i", "0"}, devname)...).Run()
  278. default:
  279. err = fmt.Errorf("Unsupported filesystem type %s", devices.filesystem)
  280. }
  281. if err != nil {
  282. return err
  283. }
  284. return nil
  285. }
  286. func (devices *DeviceSet) initMetaData() error {
  287. _, _, _, params, err := getStatus(devices.getPoolName())
  288. if err != nil {
  289. return err
  290. }
  291. if _, err := fmt.Sscanf(params, "%d", &devices.TransactionId); err != nil {
  292. return err
  293. }
  294. devices.NewTransactionId = devices.TransactionId
  295. // Migrate old metadatafile
  296. jsonData, err := ioutil.ReadFile(devices.oldMetadataFile())
  297. if err != nil && !os.IsNotExist(err) {
  298. return err
  299. }
  300. if jsonData != nil {
  301. m := MetaData{Devices: make(map[string]*DevInfo)}
  302. if err := json.Unmarshal(jsonData, &m); err != nil {
  303. return err
  304. }
  305. for hash, info := range m.Devices {
  306. info.Hash = hash
  307. // If the transaction id is larger than the actual one we lost the device due to some crash
  308. if info.TransactionId <= devices.TransactionId {
  309. devices.saveMetadata(info)
  310. }
  311. }
  312. if err := os.Rename(devices.oldMetadataFile(), devices.oldMetadataFile()+".migrated"); err != nil {
  313. return err
  314. }
  315. }
  316. return nil
  317. }
  318. func (devices *DeviceSet) loadMetadata(hash string) *DevInfo {
  319. info := &DevInfo{Hash: hash, devices: devices}
  320. jsonData, err := ioutil.ReadFile(devices.metadataFile(info))
  321. if err != nil {
  322. return nil
  323. }
  324. if err := json.Unmarshal(jsonData, &info); err != nil {
  325. return nil
  326. }
  327. // If the transaction id is larger than the actual one we lost the device due to some crash
  328. if info.TransactionId > devices.TransactionId {
  329. return nil
  330. }
  331. return info
  332. }
  333. func (devices *DeviceSet) setupBaseImage() error {
  334. oldInfo, _ := devices.lookupDevice("")
  335. if oldInfo != nil && oldInfo.Initialized {
  336. return nil
  337. }
  338. if oldInfo != nil && !oldInfo.Initialized {
  339. log.Debugf("Removing uninitialized base image")
  340. if err := devices.deleteDevice(oldInfo); err != nil {
  341. return err
  342. }
  343. }
  344. log.Debugf("Initializing base device-manager snapshot")
  345. id := devices.NextDeviceId
  346. // Create initial device
  347. if err := createDevice(devices.getPoolDevName(), &id); err != nil {
  348. return err
  349. }
  350. // Ids are 24bit, so wrap around
  351. devices.NextDeviceId = (id + 1) & 0xffffff
  352. log.Debugf("Registering base device (id %v) with FS size %v", id, devices.baseFsSize)
  353. info, err := devices.registerDevice(id, "", devices.baseFsSize)
  354. if err != nil {
  355. _ = deleteDevice(devices.getPoolDevName(), id)
  356. return err
  357. }
  358. log.Debugf("Creating filesystem on base device-manager snapshot")
  359. if err = devices.activateDeviceIfNeeded(info); err != nil {
  360. return err
  361. }
  362. if err := devices.createFilesystem(info); err != nil {
  363. return err
  364. }
  365. info.Initialized = true
  366. if err = devices.saveMetadata(info); err != nil {
  367. info.Initialized = false
  368. return err
  369. }
  370. return nil
  371. }
  372. func setCloseOnExec(name string) {
  373. if fileInfos, _ := ioutil.ReadDir("/proc/self/fd"); fileInfos != nil {
  374. for _, i := range fileInfos {
  375. link, _ := os.Readlink(filepath.Join("/proc/self/fd", i.Name()))
  376. if link == name {
  377. fd, err := strconv.Atoi(i.Name())
  378. if err == nil {
  379. syscall.CloseOnExec(fd)
  380. }
  381. }
  382. }
  383. }
  384. }
  385. func (devices *DeviceSet) log(level int, file string, line int, dmError int, message string) {
  386. if level >= 7 {
  387. return // Ignore _LOG_DEBUG
  388. }
  389. log.Debugf("libdevmapper(%d): %s:%d (%d) %s", level, file, line, dmError, message)
  390. }
  391. func major(device uint64) uint64 {
  392. return (device >> 8) & 0xfff
  393. }
  394. func minor(device uint64) uint64 {
  395. return (device & 0xff) | ((device >> 12) & 0xfff00)
  396. }
  397. func (devices *DeviceSet) ResizePool(size int64) error {
  398. dirname := devices.loopbackDir()
  399. datafilename := path.Join(dirname, "data")
  400. if len(devices.dataDevice) > 0 {
  401. datafilename = devices.dataDevice
  402. }
  403. metadatafilename := path.Join(dirname, "metadata")
  404. if len(devices.metadataDevice) > 0 {
  405. metadatafilename = devices.metadataDevice
  406. }
  407. datafile, err := os.OpenFile(datafilename, os.O_RDWR, 0)
  408. if datafile == nil {
  409. return err
  410. }
  411. defer datafile.Close()
  412. fi, err := datafile.Stat()
  413. if fi == nil {
  414. return err
  415. }
  416. if fi.Size() > size {
  417. return fmt.Errorf("Can't shrink file")
  418. }
  419. dataloopback := FindLoopDeviceFor(datafile)
  420. if dataloopback == nil {
  421. return fmt.Errorf("Unable to find loopback mount for: %s", datafilename)
  422. }
  423. defer dataloopback.Close()
  424. metadatafile, err := os.OpenFile(metadatafilename, os.O_RDWR, 0)
  425. if metadatafile == nil {
  426. return err
  427. }
  428. defer metadatafile.Close()
  429. metadataloopback := FindLoopDeviceFor(metadatafile)
  430. if metadataloopback == nil {
  431. return fmt.Errorf("Unable to find loopback mount for: %s", metadatafilename)
  432. }
  433. defer metadataloopback.Close()
  434. // Grow loopback file
  435. if err := datafile.Truncate(size); err != nil {
  436. return fmt.Errorf("Unable to grow loopback file: %s", err)
  437. }
  438. // Reload size for loopback device
  439. if err := LoopbackSetCapacity(dataloopback); err != nil {
  440. return fmt.Errorf("Unable to update loopback capacity: %s", err)
  441. }
  442. // Suspend the pool
  443. if err := suspendDevice(devices.getPoolName()); err != nil {
  444. return fmt.Errorf("Unable to suspend pool: %s", err)
  445. }
  446. // Reload with the new block sizes
  447. if err := reloadPool(devices.getPoolName(), dataloopback, metadataloopback, devices.thinpBlockSize); err != nil {
  448. return fmt.Errorf("Unable to reload pool: %s", err)
  449. }
  450. // Resume the pool
  451. if err := resumeDevice(devices.getPoolName()); err != nil {
  452. return fmt.Errorf("Unable to resume pool: %s", err)
  453. }
  454. return nil
  455. }
  456. func (devices *DeviceSet) loadDeviceSetMetaData() error {
  457. jsonData, err := ioutil.ReadFile(devices.deviceSetMetaFile())
  458. if err != nil {
  459. return nil
  460. }
  461. if err := json.Unmarshal(jsonData, devices); err != nil {
  462. return nil
  463. }
  464. return nil
  465. }
  466. func (devices *DeviceSet) saveDeviceSetMetaData() error {
  467. jsonData, err := json.Marshal(devices)
  468. if err != nil {
  469. return fmt.Errorf("Error encoding metadata to json: %s", err)
  470. }
  471. err = devices.writeMetaFile(jsonData, devices.deviceSetMetaFile())
  472. if err != nil {
  473. return err
  474. }
  475. return nil
  476. }
  477. func (devices *DeviceSet) initDevmapper(doInit bool) error {
  478. logInit(devices)
  479. _, err := getDriverVersion()
  480. if err != nil {
  481. // Can't even get driver version, assume not supported
  482. return graphdriver.ErrNotSupported
  483. }
  484. if err := os.MkdirAll(devices.metadataDir(), 0700); err != nil && !os.IsExist(err) {
  485. return err
  486. }
  487. // Set the device prefix from the device id and inode of the docker root dir
  488. st, err := os.Stat(devices.root)
  489. if err != nil {
  490. return fmt.Errorf("Error looking up dir %s: %s", devices.root, err)
  491. }
  492. sysSt := st.Sys().(*syscall.Stat_t)
  493. // "reg-" stands for "regular file".
  494. // In the future we might use "dev-" for "device file", etc.
  495. // docker-maj,min[-inode] stands for:
  496. // - Managed by docker
  497. // - The target of this device is at major <maj> and minor <min>
  498. // - If <inode> is defined, use that file inside the device as a loopback image. Otherwise use the device itself.
  499. devices.devicePrefix = fmt.Sprintf("docker-%d:%d-%d", major(sysSt.Dev), minor(sysSt.Dev), sysSt.Ino)
  500. log.Debugf("Generated prefix: %s", devices.devicePrefix)
  501. // Check for the existence of the device <prefix>-pool
  502. log.Debugf("Checking for existence of the pool '%s'", devices.getPoolName())
  503. info, err := getInfo(devices.getPoolName())
  504. if info == nil {
  505. log.Debugf("Error device getInfo: %s", err)
  506. return err
  507. }
  508. // It seems libdevmapper opens this without O_CLOEXEC, and go exec will not close files
  509. // that are not Close-on-exec, and lxc-start will die if it inherits any unexpected files,
  510. // so we add this badhack to make sure it closes itself
  511. setCloseOnExec("/dev/mapper/control")
  512. // Make sure the sparse images exist in <root>/devicemapper/data and
  513. // <root>/devicemapper/metadata
  514. createdLoopback := false
  515. // If the pool doesn't exist, create it
  516. if info.Exists == 0 {
  517. log.Debugf("Pool doesn't exist. Creating it.")
  518. var (
  519. dataFile *os.File
  520. metadataFile *os.File
  521. )
  522. if devices.dataDevice == "" {
  523. // Make sure the sparse images exist in <root>/devicemapper/data
  524. hasData := devices.hasImage("data")
  525. if !doInit && !hasData {
  526. return errors.New("Loopback data file not found")
  527. }
  528. if !hasData {
  529. createdLoopback = true
  530. }
  531. data, err := devices.ensureImage("data", devices.dataLoopbackSize)
  532. if err != nil {
  533. log.Debugf("Error device ensureImage (data): %s", err)
  534. return err
  535. }
  536. dataFile, err = attachLoopDevice(data)
  537. if err != nil {
  538. return err
  539. }
  540. } else {
  541. dataFile, err = os.OpenFile(devices.dataDevice, os.O_RDWR, 0600)
  542. if err != nil {
  543. return err
  544. }
  545. }
  546. defer dataFile.Close()
  547. if devices.metadataDevice == "" {
  548. // Make sure the sparse images exist in <root>/devicemapper/metadata
  549. hasMetadata := devices.hasImage("metadata")
  550. if !doInit && !hasMetadata {
  551. return errors.New("Loopback metadata file not found")
  552. }
  553. if !hasMetadata {
  554. createdLoopback = true
  555. }
  556. metadata, err := devices.ensureImage("metadata", devices.metaDataLoopbackSize)
  557. if err != nil {
  558. log.Debugf("Error device ensureImage (metadata): %s", err)
  559. return err
  560. }
  561. metadataFile, err = attachLoopDevice(metadata)
  562. if err != nil {
  563. return err
  564. }
  565. } else {
  566. metadataFile, err = os.OpenFile(devices.metadataDevice, os.O_RDWR, 0600)
  567. if err != nil {
  568. return err
  569. }
  570. }
  571. defer metadataFile.Close()
  572. if err := createPool(devices.getPoolName(), dataFile, metadataFile, devices.thinpBlockSize); err != nil {
  573. return err
  574. }
  575. }
  576. // If we didn't just create the data or metadata image, we need to
  577. // load the transaction id and migrate old metadata
  578. if !createdLoopback {
  579. if err = devices.initMetaData(); err != nil {
  580. return err
  581. }
  582. }
  583. // Right now this loads only NextDeviceId. If there is more metatadata
  584. // down the line, we might have to move it earlier.
  585. devices.loadDeviceSetMetaData()
  586. // Setup the base image
  587. if doInit {
  588. if err := devices.setupBaseImage(); err != nil {
  589. log.Debugf("Error device setupBaseImage: %s", err)
  590. return err
  591. }
  592. }
  593. return nil
  594. }
  595. func (devices *DeviceSet) AddDevice(hash, baseHash string) error {
  596. baseInfo, err := devices.lookupDevice(baseHash)
  597. if err != nil {
  598. return err
  599. }
  600. baseInfo.lock.Lock()
  601. defer baseInfo.lock.Unlock()
  602. devices.Lock()
  603. defer devices.Unlock()
  604. if info, _ := devices.lookupDevice(hash); info != nil {
  605. return fmt.Errorf("device %s already exists", hash)
  606. }
  607. deviceId := devices.NextDeviceId
  608. if err := createSnapDevice(devices.getPoolDevName(), &deviceId, baseInfo.Name(), baseInfo.DeviceId); err != nil {
  609. log.Debugf("Error creating snap device: %s", err)
  610. return err
  611. }
  612. // Ids are 24bit, so wrap around
  613. devices.NextDeviceId = (deviceId + 1) & 0xffffff
  614. if _, err := devices.registerDevice(deviceId, hash, baseInfo.Size); err != nil {
  615. deleteDevice(devices.getPoolDevName(), deviceId)
  616. log.Debugf("Error registering device: %s", err)
  617. return err
  618. }
  619. return nil
  620. }
  621. func (devices *DeviceSet) deleteDevice(info *DevInfo) error {
  622. if devices.doBlkDiscard {
  623. // This is a workaround for the kernel not discarding block so
  624. // on the thin pool when we remove a thinp device, so we do it
  625. // manually
  626. if err := devices.activateDeviceIfNeeded(info); err == nil {
  627. if err := BlockDeviceDiscard(info.DevName()); err != nil {
  628. log.Debugf("Error discarding block on device: %s (ignoring)", err)
  629. }
  630. }
  631. }
  632. devinfo, _ := getInfo(info.Name())
  633. if devinfo != nil && devinfo.Exists != 0 {
  634. if err := devices.removeDeviceAndWait(info.Name()); err != nil {
  635. log.Debugf("Error removing device: %s", err)
  636. return err
  637. }
  638. }
  639. if err := deleteDevice(devices.getPoolDevName(), info.DeviceId); err != nil {
  640. log.Debugf("Error deleting device: %s", err)
  641. return err
  642. }
  643. devices.allocateTransactionId()
  644. devices.devicesLock.Lock()
  645. delete(devices.Devices, info.Hash)
  646. devices.devicesLock.Unlock()
  647. if err := devices.removeMetadata(info); err != nil {
  648. devices.devicesLock.Lock()
  649. devices.Devices[info.Hash] = info
  650. devices.devicesLock.Unlock()
  651. log.Debugf("Error removing meta data: %s", err)
  652. return err
  653. }
  654. return nil
  655. }
  656. func (devices *DeviceSet) DeleteDevice(hash string) error {
  657. info, err := devices.lookupDevice(hash)
  658. if err != nil {
  659. return err
  660. }
  661. info.lock.Lock()
  662. defer info.lock.Unlock()
  663. devices.Lock()
  664. defer devices.Unlock()
  665. return devices.deleteDevice(info)
  666. }
  667. func (devices *DeviceSet) deactivatePool() error {
  668. log.Debugf("[devmapper] deactivatePool()")
  669. defer log.Debugf("[devmapper] deactivatePool END")
  670. devname := devices.getPoolDevName()
  671. devinfo, err := getInfo(devname)
  672. if err != nil {
  673. return err
  674. }
  675. if d, err := getDeps(devname); err == nil {
  676. // Access to more Debug output
  677. log.Debugf("[devmapper] getDeps() %s: %#v", devname, d)
  678. }
  679. if devinfo.Exists != 0 {
  680. return removeDevice(devname)
  681. }
  682. return nil
  683. }
  684. func (devices *DeviceSet) deactivateDevice(info *DevInfo) error {
  685. log.Debugf("[devmapper] deactivateDevice(%s)", info.Hash)
  686. defer log.Debugf("[devmapper] deactivateDevice END")
  687. // Wait for the unmount to be effective,
  688. // by watching the value of Info.OpenCount for the device
  689. if err := devices.waitClose(info); err != nil {
  690. log.Errorf("Warning: error waiting for device %s to close: %s", info.Hash, err)
  691. }
  692. devinfo, err := getInfo(info.Name())
  693. if err != nil {
  694. return err
  695. }
  696. if devinfo.Exists != 0 {
  697. if err := devices.removeDeviceAndWait(info.Name()); err != nil {
  698. return err
  699. }
  700. }
  701. return nil
  702. }
  703. // Issues the underlying dm remove operation and then waits
  704. // for it to finish.
  705. func (devices *DeviceSet) removeDeviceAndWait(devname string) error {
  706. var err error
  707. for i := 0; i < 1000; i++ {
  708. err = removeDevice(devname)
  709. if err == nil {
  710. break
  711. }
  712. if err != ErrBusy {
  713. return err
  714. }
  715. // If we see EBUSY it may be a transient error,
  716. // sleep a bit a retry a few times.
  717. devices.Unlock()
  718. time.Sleep(10 * time.Millisecond)
  719. devices.Lock()
  720. }
  721. if err != nil {
  722. return err
  723. }
  724. if err := devices.waitRemove(devname); err != nil {
  725. return err
  726. }
  727. return nil
  728. }
  729. // waitRemove blocks until either:
  730. // a) the device registered at <device_set_prefix>-<hash> is removed,
  731. // or b) the 10 second timeout expires.
  732. func (devices *DeviceSet) waitRemove(devname string) error {
  733. log.Debugf("[deviceset %s] waitRemove(%s)", devices.devicePrefix, devname)
  734. defer log.Debugf("[deviceset %s] waitRemove(%s) END", devices.devicePrefix, devname)
  735. i := 0
  736. for ; i < 1000; i++ {
  737. devinfo, err := getInfo(devname)
  738. if err != nil {
  739. // If there is an error we assume the device doesn't exist.
  740. // The error might actually be something else, but we can't differentiate.
  741. return nil
  742. }
  743. if i%100 == 0 {
  744. log.Debugf("Waiting for removal of %s: exists=%d", devname, devinfo.Exists)
  745. }
  746. if devinfo.Exists == 0 {
  747. break
  748. }
  749. devices.Unlock()
  750. time.Sleep(10 * time.Millisecond)
  751. devices.Lock()
  752. }
  753. if i == 1000 {
  754. return fmt.Errorf("Timeout while waiting for device %s to be removed", devname)
  755. }
  756. return nil
  757. }
  758. // waitClose blocks until either:
  759. // a) the device registered at <device_set_prefix>-<hash> is closed,
  760. // or b) the 10 second timeout expires.
  761. func (devices *DeviceSet) waitClose(info *DevInfo) error {
  762. i := 0
  763. for ; i < 1000; i++ {
  764. devinfo, err := getInfo(info.Name())
  765. if err != nil {
  766. return err
  767. }
  768. if i%100 == 0 {
  769. log.Debugf("Waiting for unmount of %s: opencount=%d", info.Hash, devinfo.OpenCount)
  770. }
  771. if devinfo.OpenCount == 0 {
  772. break
  773. }
  774. devices.Unlock()
  775. time.Sleep(10 * time.Millisecond)
  776. devices.Lock()
  777. }
  778. if i == 1000 {
  779. return fmt.Errorf("Timeout while waiting for device %s to close", info.Hash)
  780. }
  781. return nil
  782. }
  783. func (devices *DeviceSet) Shutdown() error {
  784. log.Debugf("[deviceset %s] shutdown()", devices.devicePrefix)
  785. log.Debugf("[devmapper] Shutting down DeviceSet: %s", devices.root)
  786. defer log.Debugf("[deviceset %s] shutdown END", devices.devicePrefix)
  787. var devs []*DevInfo
  788. devices.devicesLock.Lock()
  789. for _, info := range devices.Devices {
  790. devs = append(devs, info)
  791. }
  792. devices.devicesLock.Unlock()
  793. for _, info := range devs {
  794. info.lock.Lock()
  795. if info.mountCount > 0 {
  796. // We use MNT_DETACH here in case it is still busy in some running
  797. // container. This means it'll go away from the global scope directly,
  798. // and the device will be released when that container dies.
  799. if err := syscall.Unmount(info.mountPath, syscall.MNT_DETACH); err != nil {
  800. log.Debugf("Shutdown unmounting %s, error: %s", info.mountPath, err)
  801. }
  802. devices.Lock()
  803. if err := devices.deactivateDevice(info); err != nil {
  804. log.Debugf("Shutdown deactivate %s , error: %s", info.Hash, err)
  805. }
  806. devices.Unlock()
  807. }
  808. info.lock.Unlock()
  809. }
  810. info, _ := devices.lookupDevice("")
  811. if info != nil {
  812. info.lock.Lock()
  813. devices.Lock()
  814. if err := devices.deactivateDevice(info); err != nil {
  815. log.Debugf("Shutdown deactivate base , error: %s", err)
  816. }
  817. devices.Unlock()
  818. info.lock.Unlock()
  819. }
  820. devices.Lock()
  821. if err := devices.deactivatePool(); err != nil {
  822. log.Debugf("Shutdown deactivate pool , error: %s", err)
  823. }
  824. devices.saveDeviceSetMetaData()
  825. devices.Unlock()
  826. return nil
  827. }
  828. func (devices *DeviceSet) MountDevice(hash, path, mountLabel string) error {
  829. info, err := devices.lookupDevice(hash)
  830. if err != nil {
  831. return err
  832. }
  833. info.lock.Lock()
  834. defer info.lock.Unlock()
  835. devices.Lock()
  836. defer devices.Unlock()
  837. if info.mountCount > 0 {
  838. if path != info.mountPath {
  839. return fmt.Errorf("Trying to mount devmapper device in multple places (%s, %s)", info.mountPath, path)
  840. }
  841. info.mountCount++
  842. return nil
  843. }
  844. if err := devices.activateDeviceIfNeeded(info); err != nil {
  845. return fmt.Errorf("Error activating devmapper device for '%s': %s", hash, err)
  846. }
  847. var flags uintptr = syscall.MS_MGC_VAL
  848. fstype, err := ProbeFsType(info.DevName())
  849. if err != nil {
  850. return err
  851. }
  852. options := ""
  853. if fstype == "xfs" {
  854. // XFS needs nouuid or it can't mount filesystems with the same fs
  855. options = joinMountOptions(options, "nouuid")
  856. }
  857. options = joinMountOptions(options, devices.mountOptions)
  858. options = joinMountOptions(options, label.FormatMountLabel("", mountLabel))
  859. err = syscall.Mount(info.DevName(), path, fstype, flags, joinMountOptions("discard", options))
  860. if err != nil && err == syscall.EINVAL {
  861. err = syscall.Mount(info.DevName(), path, fstype, flags, options)
  862. }
  863. if err != nil {
  864. return fmt.Errorf("Error mounting '%s' on '%s': %s", info.DevName(), path, err)
  865. }
  866. info.mountCount = 1
  867. info.mountPath = path
  868. return nil
  869. }
  870. func (devices *DeviceSet) UnmountDevice(hash string) error {
  871. log.Debugf("[devmapper] UnmountDevice(hash=%s)", hash)
  872. defer log.Debugf("[devmapper] UnmountDevice END")
  873. info, err := devices.lookupDevice(hash)
  874. if err != nil {
  875. return err
  876. }
  877. info.lock.Lock()
  878. defer info.lock.Unlock()
  879. devices.Lock()
  880. defer devices.Unlock()
  881. if info.mountCount == 0 {
  882. return fmt.Errorf("UnmountDevice: device not-mounted id %s\n", hash)
  883. }
  884. info.mountCount--
  885. if info.mountCount > 0 {
  886. return nil
  887. }
  888. log.Debugf("[devmapper] Unmount(%s)", info.mountPath)
  889. if err := syscall.Unmount(info.mountPath, 0); err != nil {
  890. return err
  891. }
  892. log.Debugf("[devmapper] Unmount done")
  893. if err := devices.deactivateDevice(info); err != nil {
  894. return err
  895. }
  896. info.mountPath = ""
  897. return nil
  898. }
  899. func (devices *DeviceSet) HasDevice(hash string) bool {
  900. devices.Lock()
  901. defer devices.Unlock()
  902. info, _ := devices.lookupDevice(hash)
  903. return info != nil
  904. }
  905. func (devices *DeviceSet) HasActivatedDevice(hash string) bool {
  906. info, _ := devices.lookupDevice(hash)
  907. if info == nil {
  908. return false
  909. }
  910. info.lock.Lock()
  911. defer info.lock.Unlock()
  912. devices.Lock()
  913. defer devices.Unlock()
  914. devinfo, _ := getInfo(info.Name())
  915. return devinfo != nil && devinfo.Exists != 0
  916. }
  917. func (devices *DeviceSet) List() []string {
  918. devices.Lock()
  919. defer devices.Unlock()
  920. devices.devicesLock.Lock()
  921. ids := make([]string, len(devices.Devices))
  922. i := 0
  923. for k := range devices.Devices {
  924. ids[i] = k
  925. i++
  926. }
  927. devices.devicesLock.Unlock()
  928. return ids
  929. }
  930. func (devices *DeviceSet) deviceStatus(devName string) (sizeInSectors, mappedSectors, highestMappedSector uint64, err error) {
  931. var params string
  932. _, sizeInSectors, _, params, err = getStatus(devName)
  933. if err != nil {
  934. return
  935. }
  936. if _, err = fmt.Sscanf(params, "%d %d", &mappedSectors, &highestMappedSector); err == nil {
  937. return
  938. }
  939. return
  940. }
  941. func (devices *DeviceSet) GetDeviceStatus(hash string) (*DevStatus, error) {
  942. info, err := devices.lookupDevice(hash)
  943. if err != nil {
  944. return nil, err
  945. }
  946. info.lock.Lock()
  947. defer info.lock.Unlock()
  948. devices.Lock()
  949. defer devices.Unlock()
  950. status := &DevStatus{
  951. DeviceId: info.DeviceId,
  952. Size: info.Size,
  953. TransactionId: info.TransactionId,
  954. }
  955. if err := devices.activateDeviceIfNeeded(info); err != nil {
  956. return nil, fmt.Errorf("Error activating devmapper device for '%s': %s", hash, err)
  957. }
  958. if sizeInSectors, mappedSectors, highestMappedSector, err := devices.deviceStatus(info.DevName()); err != nil {
  959. return nil, err
  960. } else {
  961. status.SizeInSectors = sizeInSectors
  962. status.MappedSectors = mappedSectors
  963. status.HighestMappedSector = highestMappedSector
  964. }
  965. return status, nil
  966. }
  967. func (devices *DeviceSet) poolStatus() (totalSizeInSectors, transactionId, dataUsed, dataTotal, metadataUsed, metadataTotal uint64, err error) {
  968. var params string
  969. if _, totalSizeInSectors, _, params, err = getStatus(devices.getPoolName()); err == nil {
  970. _, err = fmt.Sscanf(params, "%d %d/%d %d/%d", &transactionId, &metadataUsed, &metadataTotal, &dataUsed, &dataTotal)
  971. }
  972. return
  973. }
  974. func (devices *DeviceSet) Status() *Status {
  975. devices.Lock()
  976. defer devices.Unlock()
  977. status := &Status{}
  978. status.PoolName = devices.getPoolName()
  979. if len(devices.dataDevice) > 0 {
  980. status.DataLoopback = devices.dataDevice
  981. } else {
  982. status.DataLoopback = path.Join(devices.loopbackDir(), "data")
  983. }
  984. if len(devices.metadataDevice) > 0 {
  985. status.MetadataLoopback = devices.metadataDevice
  986. } else {
  987. status.MetadataLoopback = path.Join(devices.loopbackDir(), "metadata")
  988. }
  989. totalSizeInSectors, _, dataUsed, dataTotal, metadataUsed, metadataTotal, err := devices.poolStatus()
  990. if err == nil {
  991. // Convert from blocks to bytes
  992. blockSizeInSectors := totalSizeInSectors / dataTotal
  993. status.Data.Used = dataUsed * blockSizeInSectors * 512
  994. status.Data.Total = dataTotal * blockSizeInSectors * 512
  995. // metadata blocks are always 4k
  996. status.Metadata.Used = metadataUsed * 4096
  997. status.Metadata.Total = metadataTotal * 4096
  998. status.SectorSize = blockSizeInSectors * 512
  999. }
  1000. return status
  1001. }
  1002. func NewDeviceSet(root string, doInit bool, options []string) (*DeviceSet, error) {
  1003. SetDevDir("/dev")
  1004. devices := &DeviceSet{
  1005. root: root,
  1006. MetaData: MetaData{Devices: make(map[string]*DevInfo)},
  1007. dataLoopbackSize: DefaultDataLoopbackSize,
  1008. metaDataLoopbackSize: DefaultMetaDataLoopbackSize,
  1009. baseFsSize: DefaultBaseFsSize,
  1010. filesystem: "ext4",
  1011. doBlkDiscard: true,
  1012. thinpBlockSize: DefaultThinpBlockSize,
  1013. }
  1014. foundBlkDiscard := false
  1015. for _, option := range options {
  1016. key, val, err := parsers.ParseKeyValueOpt(option)
  1017. if err != nil {
  1018. return nil, err
  1019. }
  1020. key = strings.ToLower(key)
  1021. switch key {
  1022. case "dm.basesize":
  1023. size, err := units.RAMInBytes(val)
  1024. if err != nil {
  1025. return nil, err
  1026. }
  1027. devices.baseFsSize = uint64(size)
  1028. case "dm.loopdatasize":
  1029. size, err := units.RAMInBytes(val)
  1030. if err != nil {
  1031. return nil, err
  1032. }
  1033. devices.dataLoopbackSize = size
  1034. case "dm.loopmetadatasize":
  1035. size, err := units.RAMInBytes(val)
  1036. if err != nil {
  1037. return nil, err
  1038. }
  1039. devices.metaDataLoopbackSize = size
  1040. case "dm.fs":
  1041. if val != "ext4" && val != "xfs" {
  1042. return nil, fmt.Errorf("Unsupported filesystem %s\n", val)
  1043. }
  1044. devices.filesystem = val
  1045. case "dm.mkfsarg":
  1046. devices.mkfsArgs = append(devices.mkfsArgs, val)
  1047. case "dm.mountopt":
  1048. devices.mountOptions = joinMountOptions(devices.mountOptions, val)
  1049. case "dm.metadatadev":
  1050. devices.metadataDevice = val
  1051. case "dm.datadev":
  1052. devices.dataDevice = val
  1053. case "dm.blkdiscard":
  1054. foundBlkDiscard = true
  1055. devices.doBlkDiscard, err = strconv.ParseBool(val)
  1056. if err != nil {
  1057. return nil, err
  1058. }
  1059. case "dm.blocksize":
  1060. size, err := units.RAMInBytes(val)
  1061. if err != nil {
  1062. return nil, err
  1063. }
  1064. // convert to 512b sectors
  1065. devices.thinpBlockSize = uint32(size) >> 9
  1066. default:
  1067. return nil, fmt.Errorf("Unknown option %s\n", key)
  1068. }
  1069. }
  1070. // By default, don't do blk discard hack on raw devices, its rarely useful and is expensive
  1071. if !foundBlkDiscard && devices.dataDevice != "" {
  1072. devices.doBlkDiscard = false
  1073. }
  1074. if err := devices.initDevmapper(doInit); err != nil {
  1075. return nil, err
  1076. }
  1077. return devices, nil
  1078. }