deviceset.go 28 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091
  1. // +build linux,amd64
  2. package devmapper
  3. import (
  4. "encoding/json"
  5. "errors"
  6. "fmt"
  7. "github.com/dotcloud/docker/pkg/label"
  8. "github.com/dotcloud/docker/utils"
  9. "io"
  10. "io/ioutil"
  11. "path"
  12. "path/filepath"
  13. "strconv"
  14. "strings"
  15. "sync"
  16. "syscall"
  17. "time"
  18. )
  19. var (
  20. DefaultDataLoopbackSize int64 = 100 * 1024 * 1024 * 1024
  21. DefaultMetaDataLoopbackSize int64 = 2 * 1024 * 1024 * 1024
  22. DefaultBaseFsSize uint64 = 10 * 1024 * 1024 * 1024
  23. )
  24. type DevInfo struct {
  25. Hash string `json:"-"`
  26. DeviceId int `json:"device_id"`
  27. Size uint64 `json:"size"`
  28. TransactionId uint64 `json:"transaction_id"`
  29. Initialized bool `json:"initialized"`
  30. devices *DeviceSet `json:"-"`
  31. mountCount int `json:"-"`
  32. mountPath string `json:"-"`
  33. // A floating mount means one reference is not owned and
  34. // will be stolen by the next mount. This allows us to
  35. // avoid unmounting directly after creation before the
  36. // first get (since we need to mount to set up the device
  37. // a bit first).
  38. floating bool `json:"-"`
  39. // The global DeviceSet lock guarantees that we serialize all
  40. // the calls to libdevmapper (which is not threadsafe), but we
  41. // sometimes release that lock while sleeping. In that case
  42. // this per-device lock is still held, protecting against
  43. // other accesses to the device that we're doing the wait on.
  44. lock sync.Mutex `json:"-"`
  45. }
  46. type MetaData struct {
  47. Devices map[string]*DevInfo `json:devices`
  48. }
  49. type DeviceSet struct {
  50. MetaData
  51. sync.Mutex // Protects Devices map and serializes calls into libdevmapper
  52. root string
  53. devicePrefix string
  54. TransactionId uint64
  55. NewTransactionId uint64
  56. nextFreeDevice int
  57. sawBusy bool
  58. }
  59. type DiskUsage struct {
  60. Used uint64
  61. Total uint64
  62. }
  63. type Status struct {
  64. PoolName string
  65. DataLoopback string
  66. MetadataLoopback string
  67. Data DiskUsage
  68. Metadata DiskUsage
  69. SectorSize uint64
  70. }
  71. type DevStatus struct {
  72. DeviceId int
  73. Size uint64
  74. TransactionId uint64
  75. SizeInSectors uint64
  76. MappedSectors uint64
  77. HighestMappedSector uint64
  78. }
  79. type UnmountMode int
  80. const (
  81. UnmountRegular UnmountMode = iota
  82. UnmountFloat
  83. UnmountSink
  84. )
  85. func getDevName(name string) string {
  86. return "/dev/mapper/" + name
  87. }
  88. func (info *DevInfo) Name() string {
  89. hash := info.Hash
  90. if hash == "" {
  91. hash = "base"
  92. }
  93. return fmt.Sprintf("%s-%s", info.devices.devicePrefix, hash)
  94. }
  95. func (info *DevInfo) DevName() string {
  96. return getDevName(info.Name())
  97. }
  98. func (devices *DeviceSet) loopbackDir() string {
  99. return path.Join(devices.root, "devicemapper")
  100. }
  101. func (devices *DeviceSet) jsonFile() string {
  102. return path.Join(devices.loopbackDir(), "json")
  103. }
  104. func (devices *DeviceSet) getPoolName() string {
  105. return devices.devicePrefix + "-pool"
  106. }
  107. func (devices *DeviceSet) getPoolDevName() string {
  108. return getDevName(devices.getPoolName())
  109. }
  110. func (devices *DeviceSet) hasImage(name string) bool {
  111. dirname := devices.loopbackDir()
  112. filename := path.Join(dirname, name)
  113. _, err := osStat(filename)
  114. return err == nil
  115. }
  116. // ensureImage creates a sparse file of <size> bytes at the path
  117. // <root>/devicemapper/<name>.
  118. // If the file already exists, it does nothing.
  119. // Either way it returns the full path.
  120. func (devices *DeviceSet) ensureImage(name string, size int64) (string, error) {
  121. dirname := devices.loopbackDir()
  122. filename := path.Join(dirname, name)
  123. if err := osMkdirAll(dirname, 0700); err != nil && !osIsExist(err) {
  124. return "", err
  125. }
  126. if _, err := osStat(filename); err != nil {
  127. if !osIsNotExist(err) {
  128. return "", err
  129. }
  130. utils.Debugf("Creating loopback file %s for device-manage use", filename)
  131. file, err := osOpenFile(filename, osORdWr|osOCreate, 0600)
  132. if err != nil {
  133. return "", err
  134. }
  135. defer file.Close()
  136. if err = file.Truncate(size); err != nil {
  137. return "", err
  138. }
  139. }
  140. return filename, nil
  141. }
  142. func (devices *DeviceSet) allocateDeviceId() int {
  143. // TODO: Add smarter reuse of deleted devices
  144. id := devices.nextFreeDevice
  145. devices.nextFreeDevice = devices.nextFreeDevice + 1
  146. return id
  147. }
  148. func (devices *DeviceSet) allocateTransactionId() uint64 {
  149. devices.NewTransactionId = devices.NewTransactionId + 1
  150. return devices.NewTransactionId
  151. }
  152. func (devices *DeviceSet) saveMetadata() error {
  153. jsonData, err := json.Marshal(devices.MetaData)
  154. if err != nil {
  155. return fmt.Errorf("Error encoding metadata to json: %s", err)
  156. }
  157. tmpFile, err := ioutil.TempFile(filepath.Dir(devices.jsonFile()), ".json")
  158. if err != nil {
  159. return fmt.Errorf("Error creating metadata file: %s", err)
  160. }
  161. n, err := tmpFile.Write(jsonData)
  162. if err != nil {
  163. return fmt.Errorf("Error writing metadata to %s: %s", tmpFile.Name(), err)
  164. }
  165. if n < len(jsonData) {
  166. return io.ErrShortWrite
  167. }
  168. if err := tmpFile.Sync(); err != nil {
  169. return fmt.Errorf("Error syncing metadata file %s: %s", tmpFile.Name(), err)
  170. }
  171. if err := tmpFile.Close(); err != nil {
  172. return fmt.Errorf("Error closing metadata file %s: %s", tmpFile.Name(), err)
  173. }
  174. if err := osRename(tmpFile.Name(), devices.jsonFile()); err != nil {
  175. return fmt.Errorf("Error committing metadata file %s: %s", tmpFile.Name(), err)
  176. }
  177. if devices.NewTransactionId != devices.TransactionId {
  178. if err = setTransactionId(devices.getPoolDevName(), devices.TransactionId, devices.NewTransactionId); err != nil {
  179. return fmt.Errorf("Error setting devmapper transition ID: %s", err)
  180. }
  181. devices.TransactionId = devices.NewTransactionId
  182. }
  183. return nil
  184. }
  185. func (devices *DeviceSet) registerDevice(id int, hash string, size uint64) (*DevInfo, error) {
  186. utils.Debugf("registerDevice(%v, %v)", id, hash)
  187. info := &DevInfo{
  188. Hash: hash,
  189. DeviceId: id,
  190. Size: size,
  191. TransactionId: devices.allocateTransactionId(),
  192. Initialized: false,
  193. devices: devices,
  194. }
  195. devices.Devices[hash] = info
  196. if err := devices.saveMetadata(); err != nil {
  197. // Try to remove unused device
  198. delete(devices.Devices, hash)
  199. return nil, err
  200. }
  201. return info, nil
  202. }
  203. func (devices *DeviceSet) activateDeviceIfNeeded(hash string) error {
  204. utils.Debugf("activateDeviceIfNeeded(%v)", hash)
  205. info := devices.Devices[hash]
  206. if info == nil {
  207. return fmt.Errorf("Unknown device %s", hash)
  208. }
  209. if devinfo, _ := getInfo(info.Name()); devinfo != nil && devinfo.Exists != 0 {
  210. return nil
  211. }
  212. return activateDevice(devices.getPoolDevName(), info.Name(), info.DeviceId, info.Size)
  213. }
  214. func (devices *DeviceSet) createFilesystem(info *DevInfo) error {
  215. devname := info.DevName()
  216. err := execRun("mkfs.ext4", "-E", "discard,lazy_itable_init=0,lazy_journal_init=0", devname)
  217. if err != nil {
  218. err = execRun("mkfs.ext4", "-E", "discard,lazy_itable_init=0", devname)
  219. }
  220. if err != nil {
  221. utils.Debugf("\n--->Err: %s\n", err)
  222. return err
  223. }
  224. return nil
  225. }
  226. func (devices *DeviceSet) loadMetaData() error {
  227. utils.Debugf("loadMetadata()")
  228. defer utils.Debugf("loadMetadata END")
  229. _, _, _, params, err := getStatus(devices.getPoolName())
  230. if err != nil {
  231. utils.Debugf("\n--->Err: %s\n", err)
  232. return err
  233. }
  234. if _, err := fmt.Sscanf(params, "%d", &devices.TransactionId); err != nil {
  235. utils.Debugf("\n--->Err: %s\n", err)
  236. return err
  237. }
  238. devices.NewTransactionId = devices.TransactionId
  239. jsonData, err := ioutil.ReadFile(devices.jsonFile())
  240. if err != nil && !osIsNotExist(err) {
  241. utils.Debugf("\n--->Err: %s\n", err)
  242. return err
  243. }
  244. devices.MetaData.Devices = make(map[string]*DevInfo)
  245. if jsonData != nil {
  246. if err := json.Unmarshal(jsonData, &devices.MetaData); err != nil {
  247. utils.Debugf("\n--->Err: %s\n", err)
  248. return err
  249. }
  250. }
  251. for hash, d := range devices.Devices {
  252. d.Hash = hash
  253. d.devices = devices
  254. if d.DeviceId >= devices.nextFreeDevice {
  255. devices.nextFreeDevice = d.DeviceId + 1
  256. }
  257. // If the transaction id is larger than the actual one we lost the device due to some crash
  258. if d.TransactionId > devices.TransactionId {
  259. utils.Debugf("Removing lost device %s with id %d", hash, d.TransactionId)
  260. delete(devices.Devices, hash)
  261. }
  262. }
  263. return nil
  264. }
  265. func (devices *DeviceSet) setupBaseImage() error {
  266. oldInfo := devices.Devices[""]
  267. if oldInfo != nil && oldInfo.Initialized {
  268. return nil
  269. }
  270. if oldInfo != nil && !oldInfo.Initialized {
  271. utils.Debugf("Removing uninitialized base image")
  272. if err := devices.deleteDevice(""); err != nil {
  273. utils.Debugf("\n--->Err: %s\n", err)
  274. return err
  275. }
  276. }
  277. utils.Debugf("Initializing base device-manager snapshot")
  278. id := devices.allocateDeviceId()
  279. // Create initial device
  280. if err := createDevice(devices.getPoolDevName(), id); err != nil {
  281. utils.Debugf("\n--->Err: %s\n", err)
  282. return err
  283. }
  284. utils.Debugf("Registering base device (id %v) with FS size %v", id, DefaultBaseFsSize)
  285. info, err := devices.registerDevice(id, "", DefaultBaseFsSize)
  286. if err != nil {
  287. _ = deleteDevice(devices.getPoolDevName(), id)
  288. utils.Debugf("\n--->Err: %s\n", err)
  289. return err
  290. }
  291. utils.Debugf("Creating filesystem on base device-manager snapshot")
  292. if err = devices.activateDeviceIfNeeded(""); err != nil {
  293. utils.Debugf("\n--->Err: %s\n", err)
  294. return err
  295. }
  296. if err := devices.createFilesystem(info); err != nil {
  297. utils.Debugf("\n--->Err: %s\n", err)
  298. return err
  299. }
  300. info.Initialized = true
  301. if err = devices.saveMetadata(); err != nil {
  302. info.Initialized = false
  303. utils.Debugf("\n--->Err: %s\n", err)
  304. return err
  305. }
  306. return nil
  307. }
  308. func setCloseOnExec(name string) {
  309. if fileInfos, _ := ioutil.ReadDir("/proc/self/fd"); fileInfos != nil {
  310. for _, i := range fileInfos {
  311. link, _ := osReadlink(filepath.Join("/proc/self/fd", i.Name()))
  312. if link == name {
  313. fd, err := strconv.Atoi(i.Name())
  314. if err == nil {
  315. sysCloseOnExec(fd)
  316. }
  317. }
  318. }
  319. }
  320. }
  321. func (devices *DeviceSet) log(level int, file string, line int, dmError int, message string) {
  322. if level >= 7 {
  323. return // Ignore _LOG_DEBUG
  324. }
  325. if strings.Contains(message, "busy") {
  326. devices.sawBusy = true
  327. }
  328. utils.Debugf("libdevmapper(%d): %s:%d (%d) %s", level, file, line, dmError, message)
  329. }
  330. func major(device uint64) uint64 {
  331. return (device >> 8) & 0xfff
  332. }
  333. func minor(device uint64) uint64 {
  334. return (device & 0xff) | ((device >> 12) & 0xfff00)
  335. }
  336. func (devices *DeviceSet) ResizePool(size int64) error {
  337. dirname := devices.loopbackDir()
  338. datafilename := path.Join(dirname, "data")
  339. metadatafilename := path.Join(dirname, "metadata")
  340. datafile, err := osOpenFile(datafilename, osORdWr, 0)
  341. if datafile == nil {
  342. return err
  343. }
  344. defer datafile.Close()
  345. fi, err := datafile.Stat()
  346. if fi == nil {
  347. return err
  348. }
  349. if fi.Size() > size {
  350. return fmt.Errorf("Can't shrink file")
  351. }
  352. dataloopback := FindLoopDeviceFor(datafile)
  353. if dataloopback == nil {
  354. return fmt.Errorf("Unable to find loopback mount for: %s", datafilename)
  355. }
  356. defer dataloopback.Close()
  357. metadatafile, err := osOpenFile(metadatafilename, osORdWr, 0)
  358. if metadatafile == nil {
  359. return err
  360. }
  361. defer metadatafile.Close()
  362. metadataloopback := FindLoopDeviceFor(metadatafile)
  363. if metadataloopback == nil {
  364. return fmt.Errorf("Unable to find loopback mount for: %s", metadatafilename)
  365. }
  366. defer metadataloopback.Close()
  367. // Grow loopback file
  368. if err := datafile.Truncate(size); err != nil {
  369. return fmt.Errorf("Unable to grow loopback file: %s", err)
  370. }
  371. // Reload size for loopback device
  372. if err := LoopbackSetCapacity(dataloopback); err != nil {
  373. return fmt.Errorf("Unable to update loopback capacity: %s", err)
  374. }
  375. // Suspend the pool
  376. if err := suspendDevice(devices.getPoolName()); err != nil {
  377. return fmt.Errorf("Unable to suspend pool: %s", err)
  378. }
  379. // Reload with the new block sizes
  380. if err := reloadPool(devices.getPoolName(), dataloopback, metadataloopback); err != nil {
  381. return fmt.Errorf("Unable to reload pool: %s", err)
  382. }
  383. // Resume the pool
  384. if err := resumeDevice(devices.getPoolName()); err != nil {
  385. return fmt.Errorf("Unable to resume pool: %s", err)
  386. }
  387. return nil
  388. }
  389. func (devices *DeviceSet) initDevmapper(doInit bool) error {
  390. logInit(devices)
  391. // Make sure the sparse images exist in <root>/devicemapper/data and
  392. // <root>/devicemapper/metadata
  393. hasData := devices.hasImage("data")
  394. hasMetadata := devices.hasImage("metadata")
  395. if !doInit && !hasData {
  396. return errors.New("Loopback data file not found")
  397. }
  398. if !doInit && !hasMetadata {
  399. return errors.New("Loopback metadata file not found")
  400. }
  401. createdLoopback := !hasData || !hasMetadata
  402. data, err := devices.ensureImage("data", DefaultDataLoopbackSize)
  403. if err != nil {
  404. utils.Debugf("Error device ensureImage (data): %s\n", err)
  405. return err
  406. }
  407. metadata, err := devices.ensureImage("metadata", DefaultMetaDataLoopbackSize)
  408. if err != nil {
  409. utils.Debugf("Error device ensureImage (metadata): %s\n", err)
  410. return err
  411. }
  412. // Set the device prefix from the device id and inode of the docker root dir
  413. st, err := osStat(devices.root)
  414. if err != nil {
  415. return fmt.Errorf("Error looking up dir %s: %s", devices.root, err)
  416. }
  417. sysSt := toSysStatT(st.Sys())
  418. // "reg-" stands for "regular file".
  419. // In the future we might use "dev-" for "device file", etc.
  420. // docker-maj,min[-inode] stands for:
  421. // - Managed by docker
  422. // - The target of this device is at major <maj> and minor <min>
  423. // - If <inode> is defined, use that file inside the device as a loopback image. Otherwise use the device itself.
  424. devices.devicePrefix = fmt.Sprintf("docker-%d:%d-%d", major(sysSt.Dev), minor(sysSt.Dev), sysSt.Ino)
  425. utils.Debugf("Generated prefix: %s", devices.devicePrefix)
  426. // Check for the existence of the device <prefix>-pool
  427. utils.Debugf("Checking for existence of the pool '%s'", devices.getPoolName())
  428. info, err := getInfo(devices.getPoolName())
  429. if info == nil {
  430. utils.Debugf("Error device getInfo: %s", err)
  431. return err
  432. }
  433. // It seems libdevmapper opens this without O_CLOEXEC, and go exec will not close files
  434. // that are not Close-on-exec, and lxc-start will die if it inherits any unexpected files,
  435. // so we add this badhack to make sure it closes itself
  436. setCloseOnExec("/dev/mapper/control")
  437. // If the pool doesn't exist, create it
  438. if info.Exists == 0 {
  439. utils.Debugf("Pool doesn't exist. Creating it.")
  440. dataFile, err := attachLoopDevice(data)
  441. if err != nil {
  442. utils.Debugf("\n--->Err: %s\n", err)
  443. return err
  444. }
  445. defer dataFile.Close()
  446. metadataFile, err := attachLoopDevice(metadata)
  447. if err != nil {
  448. utils.Debugf("\n--->Err: %s\n", err)
  449. return err
  450. }
  451. defer metadataFile.Close()
  452. if err := createPool(devices.getPoolName(), dataFile, metadataFile); err != nil {
  453. utils.Debugf("\n--->Err: %s\n", err)
  454. return err
  455. }
  456. }
  457. // If we didn't just create the data or metadata image, we need to
  458. // load the metadata from the existing file.
  459. if !createdLoopback {
  460. if err = devices.loadMetaData(); err != nil {
  461. utils.Debugf("\n--->Err: %s\n", err)
  462. return err
  463. }
  464. }
  465. // Setup the base image
  466. if doInit {
  467. if err := devices.setupBaseImage(); err != nil {
  468. utils.Debugf("Error device setupBaseImage: %s\n", err)
  469. return err
  470. }
  471. }
  472. return nil
  473. }
  474. func (devices *DeviceSet) AddDevice(hash, baseHash string) error {
  475. devices.Lock()
  476. defer devices.Unlock()
  477. if devices.Devices[hash] != nil {
  478. return fmt.Errorf("hash %s already exists", hash)
  479. }
  480. baseInfo := devices.Devices[baseHash]
  481. if baseInfo == nil {
  482. return fmt.Errorf("Error adding device for '%s': can't find device for parent '%s'", hash, baseHash)
  483. }
  484. baseInfo.lock.Lock()
  485. defer baseInfo.lock.Unlock()
  486. deviceId := devices.allocateDeviceId()
  487. if err := devices.createSnapDevice(devices.getPoolDevName(), deviceId, baseInfo.Name(), baseInfo.DeviceId); err != nil {
  488. utils.Debugf("Error creating snap device: %s\n", err)
  489. return err
  490. }
  491. if _, err := devices.registerDevice(deviceId, hash, baseInfo.Size); err != nil {
  492. deleteDevice(devices.getPoolDevName(), deviceId)
  493. utils.Debugf("Error registering device: %s\n", err)
  494. return err
  495. }
  496. return nil
  497. }
  498. func (devices *DeviceSet) deleteDevice(hash string) error {
  499. info := devices.Devices[hash]
  500. if info == nil {
  501. return fmt.Errorf("hash %s doesn't exists", hash)
  502. }
  503. // This is a workaround for the kernel not discarding block so
  504. // on the thin pool when we remove a thinp device, so we do it
  505. // manually
  506. if err := devices.activateDeviceIfNeeded(hash); err == nil {
  507. if err := BlockDeviceDiscard(info.DevName()); err != nil {
  508. utils.Debugf("Error discarding block on device: %s (ignoring)\n", err)
  509. }
  510. }
  511. devinfo, _ := getInfo(info.Name())
  512. if devinfo != nil && devinfo.Exists != 0 {
  513. if err := devices.removeDeviceAndWait(info.Name()); err != nil {
  514. utils.Debugf("Error removing device: %s\n", err)
  515. return err
  516. }
  517. }
  518. if info.Initialized {
  519. info.Initialized = false
  520. if err := devices.saveMetadata(); err != nil {
  521. utils.Debugf("Error saving meta data: %s\n", err)
  522. return err
  523. }
  524. }
  525. if err := deleteDevice(devices.getPoolDevName(), info.DeviceId); err != nil {
  526. utils.Debugf("Error deleting device: %s\n", err)
  527. return err
  528. }
  529. devices.allocateTransactionId()
  530. delete(devices.Devices, info.Hash)
  531. if err := devices.saveMetadata(); err != nil {
  532. devices.Devices[info.Hash] = info
  533. utils.Debugf("Error saving meta data: %s\n", err)
  534. return err
  535. }
  536. return nil
  537. }
  538. func (devices *DeviceSet) DeleteDevice(hash string) error {
  539. devices.Lock()
  540. defer devices.Unlock()
  541. info := devices.Devices[hash]
  542. if info == nil {
  543. return fmt.Errorf("Unknown device %s", hash)
  544. }
  545. info.lock.Lock()
  546. defer info.lock.Unlock()
  547. return devices.deleteDevice(hash)
  548. }
  549. func (devices *DeviceSet) deactivatePool() error {
  550. utils.Debugf("[devmapper] deactivatePool()")
  551. defer utils.Debugf("[devmapper] deactivatePool END")
  552. devname := devices.getPoolDevName()
  553. devinfo, err := getInfo(devname)
  554. if err != nil {
  555. utils.Debugf("\n--->Err: %s\n", err)
  556. return err
  557. }
  558. if devinfo.Exists != 0 {
  559. return removeDevice(devname)
  560. }
  561. return nil
  562. }
  563. func (devices *DeviceSet) deactivateDevice(hash string) error {
  564. utils.Debugf("[devmapper] deactivateDevice(%s)", hash)
  565. defer utils.Debugf("[devmapper] deactivateDevice END")
  566. // Wait for the unmount to be effective,
  567. // by watching the value of Info.OpenCount for the device
  568. if err := devices.waitClose(hash); err != nil {
  569. utils.Errorf("Warning: error waiting for device %s to close: %s\n", hash, err)
  570. }
  571. info := devices.Devices[hash]
  572. if info == nil {
  573. return fmt.Errorf("Unknown device %s", hash)
  574. }
  575. devinfo, err := getInfo(info.Name())
  576. if err != nil {
  577. utils.Debugf("\n--->Err: %s\n", err)
  578. return err
  579. }
  580. if devinfo.Exists != 0 {
  581. if err := devices.removeDeviceAndWait(info.Name()); err != nil {
  582. utils.Debugf("\n--->Err: %s\n", err)
  583. return err
  584. }
  585. }
  586. return nil
  587. }
  588. // Issues the underlying dm remove operation and then waits
  589. // for it to finish.
  590. func (devices *DeviceSet) removeDeviceAndWait(devname string) error {
  591. var err error
  592. for i := 0; i < 1000; i++ {
  593. devices.sawBusy = false
  594. err = removeDevice(devname)
  595. if err == nil {
  596. break
  597. }
  598. if !devices.sawBusy {
  599. return err
  600. }
  601. // If we see EBUSY it may be a transient error,
  602. // sleep a bit a retry a few times.
  603. devices.Unlock()
  604. time.Sleep(10 * time.Millisecond)
  605. devices.Lock()
  606. }
  607. if err != nil {
  608. return err
  609. }
  610. if err := devices.waitRemove(devname); err != nil {
  611. return err
  612. }
  613. return nil
  614. }
  615. // waitRemove blocks until either:
  616. // a) the device registered at <device_set_prefix>-<hash> is removed,
  617. // or b) the 10 second timeout expires.
  618. func (devices *DeviceSet) waitRemove(devname string) error {
  619. utils.Debugf("[deviceset %s] waitRemove(%s)", devices.devicePrefix, devname)
  620. defer utils.Debugf("[deviceset %s] waitRemove(%s) END", devices.devicePrefix, devname)
  621. i := 0
  622. for ; i < 1000; i += 1 {
  623. devinfo, err := getInfo(devname)
  624. if err != nil {
  625. // If there is an error we assume the device doesn't exist.
  626. // The error might actually be something else, but we can't differentiate.
  627. return nil
  628. }
  629. if i%100 == 0 {
  630. utils.Debugf("Waiting for removal of %s: exists=%d", devname, devinfo.Exists)
  631. }
  632. if devinfo.Exists == 0 {
  633. break
  634. }
  635. devices.Unlock()
  636. time.Sleep(10 * time.Millisecond)
  637. devices.Lock()
  638. }
  639. if i == 1000 {
  640. return fmt.Errorf("Timeout while waiting for device %s to be removed", devname)
  641. }
  642. return nil
  643. }
  644. // waitClose blocks until either:
  645. // a) the device registered at <device_set_prefix>-<hash> is closed,
  646. // or b) the 10 second timeout expires.
  647. func (devices *DeviceSet) waitClose(hash string) error {
  648. info := devices.Devices[hash]
  649. if info == nil {
  650. return fmt.Errorf("Unknown device %s", hash)
  651. }
  652. i := 0
  653. for ; i < 1000; i += 1 {
  654. devinfo, err := getInfo(info.Name())
  655. if err != nil {
  656. return err
  657. }
  658. if i%100 == 0 {
  659. utils.Debugf("Waiting for unmount of %s: opencount=%d", hash, devinfo.OpenCount)
  660. }
  661. if devinfo.OpenCount == 0 {
  662. break
  663. }
  664. devices.Unlock()
  665. time.Sleep(10 * time.Millisecond)
  666. devices.Lock()
  667. }
  668. if i == 1000 {
  669. return fmt.Errorf("Timeout while waiting for device %s to close", hash)
  670. }
  671. return nil
  672. }
  673. func (devices *DeviceSet) Shutdown() error {
  674. devices.Lock()
  675. defer devices.Unlock()
  676. utils.Debugf("[deviceset %s] shutdown()", devices.devicePrefix)
  677. utils.Debugf("[devmapper] Shutting down DeviceSet: %s", devices.root)
  678. defer utils.Debugf("[deviceset %s] shutdown END", devices.devicePrefix)
  679. for _, info := range devices.Devices {
  680. info.lock.Lock()
  681. if info.mountCount > 0 {
  682. // We use MNT_DETACH here in case it is still busy in some running
  683. // container. This means it'll go away from the global scope directly,
  684. // and the device will be released when that container dies.
  685. if err := sysUnmount(info.mountPath, syscall.MNT_DETACH); err != nil {
  686. utils.Debugf("Shutdown unmounting %s, error: %s\n", info.mountPath, err)
  687. }
  688. if err := devices.deactivateDevice(info.Hash); err != nil {
  689. utils.Debugf("Shutdown deactivate %s , error: %s\n", info.Hash, err)
  690. }
  691. }
  692. info.lock.Unlock()
  693. }
  694. if err := devices.deactivatePool(); err != nil {
  695. utils.Debugf("Shutdown deactivate pool , error: %s\n", err)
  696. }
  697. return nil
  698. }
  699. func (devices *DeviceSet) MountDevice(hash, path string, mountLabel string) error {
  700. devices.Lock()
  701. defer devices.Unlock()
  702. info := devices.Devices[hash]
  703. if info == nil {
  704. return fmt.Errorf("Unknown device %s", hash)
  705. }
  706. info.lock.Lock()
  707. defer info.lock.Unlock()
  708. if info.mountCount > 0 {
  709. if path != info.mountPath {
  710. return fmt.Errorf("Trying to mount devmapper device in multple places (%s, %s)", info.mountPath, path)
  711. }
  712. if info.floating {
  713. // Steal floating ref
  714. info.floating = false
  715. } else {
  716. info.mountCount++
  717. }
  718. return nil
  719. }
  720. if err := devices.activateDeviceIfNeeded(hash); err != nil {
  721. return fmt.Errorf("Error activating devmapper device for '%s': %s", hash, err)
  722. }
  723. var flags uintptr = sysMsMgcVal
  724. mountOptions := label.FormatMountLabel("discard", mountLabel)
  725. err := sysMount(info.DevName(), path, "ext4", flags, mountOptions)
  726. if err != nil && err == sysEInval {
  727. mountOptions = label.FormatMountLabel(mountLabel, "")
  728. err = sysMount(info.DevName(), path, "ext4", flags, mountOptions)
  729. }
  730. if err != nil {
  731. return fmt.Errorf("Error mounting '%s' on '%s': %s", info.DevName(), path, err)
  732. }
  733. info.mountCount = 1
  734. info.mountPath = path
  735. info.floating = false
  736. return devices.setInitialized(hash)
  737. }
  738. func (devices *DeviceSet) UnmountDevice(hash string, mode UnmountMode) error {
  739. utils.Debugf("[devmapper] UnmountDevice(hash=%s, mode=%d)", hash, mode)
  740. defer utils.Debugf("[devmapper] UnmountDevice END")
  741. devices.Lock()
  742. defer devices.Unlock()
  743. info := devices.Devices[hash]
  744. if info == nil {
  745. return fmt.Errorf("UnmountDevice: no such device %s\n", hash)
  746. }
  747. info.lock.Lock()
  748. defer info.lock.Unlock()
  749. if mode == UnmountFloat {
  750. if info.floating {
  751. return fmt.Errorf("UnmountDevice: can't float floating reference %s\n", hash)
  752. }
  753. // Leave this reference floating
  754. info.floating = true
  755. return nil
  756. }
  757. if mode == UnmountSink {
  758. if !info.floating {
  759. // Someone already sunk this
  760. return nil
  761. }
  762. // Otherwise, treat this as a regular unmount
  763. }
  764. if info.mountCount == 0 {
  765. return fmt.Errorf("UnmountDevice: device not-mounted id %s\n", hash)
  766. }
  767. info.mountCount--
  768. if info.mountCount > 0 {
  769. return nil
  770. }
  771. utils.Debugf("[devmapper] Unmount(%s)", info.mountPath)
  772. if err := sysUnmount(info.mountPath, 0); err != nil {
  773. utils.Debugf("\n--->Err: %s\n", err)
  774. return err
  775. }
  776. utils.Debugf("[devmapper] Unmount done")
  777. if err := devices.deactivateDevice(hash); err != nil {
  778. return err
  779. }
  780. info.mountPath = ""
  781. return nil
  782. }
  783. func (devices *DeviceSet) HasDevice(hash string) bool {
  784. devices.Lock()
  785. defer devices.Unlock()
  786. return devices.Devices[hash] != nil
  787. }
  788. func (devices *DeviceSet) HasInitializedDevice(hash string) bool {
  789. devices.Lock()
  790. defer devices.Unlock()
  791. info := devices.Devices[hash]
  792. return info != nil && info.Initialized
  793. }
  794. func (devices *DeviceSet) HasActivatedDevice(hash string) bool {
  795. devices.Lock()
  796. defer devices.Unlock()
  797. info := devices.Devices[hash]
  798. if info == nil {
  799. return false
  800. }
  801. info.lock.Lock()
  802. defer info.lock.Unlock()
  803. devinfo, _ := getInfo(info.Name())
  804. return devinfo != nil && devinfo.Exists != 0
  805. }
  806. func (devices *DeviceSet) setInitialized(hash string) error {
  807. info := devices.Devices[hash]
  808. if info == nil {
  809. return fmt.Errorf("Unknown device %s", hash)
  810. }
  811. info.Initialized = true
  812. if err := devices.saveMetadata(); err != nil {
  813. info.Initialized = false
  814. utils.Debugf("\n--->Err: %s\n", err)
  815. return err
  816. }
  817. return nil
  818. }
  819. func (devices *DeviceSet) List() []string {
  820. devices.Lock()
  821. defer devices.Unlock()
  822. ids := make([]string, len(devices.Devices))
  823. i := 0
  824. for k := range devices.Devices {
  825. ids[i] = k
  826. i++
  827. }
  828. return ids
  829. }
  830. func (devices *DeviceSet) deviceStatus(devName string) (sizeInSectors, mappedSectors, highestMappedSector uint64, err error) {
  831. var params string
  832. _, sizeInSectors, _, params, err = getStatus(devName)
  833. if err != nil {
  834. return
  835. }
  836. if _, err = fmt.Sscanf(params, "%d %d", &mappedSectors, &highestMappedSector); err == nil {
  837. return
  838. }
  839. return
  840. }
  841. func (devices *DeviceSet) GetDeviceStatus(hash string) (*DevStatus, error) {
  842. devices.Lock()
  843. defer devices.Unlock()
  844. info := devices.Devices[hash]
  845. if info == nil {
  846. return nil, fmt.Errorf("No device %s", hash)
  847. }
  848. info.lock.Lock()
  849. defer info.lock.Unlock()
  850. status := &DevStatus{
  851. DeviceId: info.DeviceId,
  852. Size: info.Size,
  853. TransactionId: info.TransactionId,
  854. }
  855. if err := devices.activateDeviceIfNeeded(hash); err != nil {
  856. return nil, fmt.Errorf("Error activating devmapper device for '%s': %s", hash, err)
  857. }
  858. if sizeInSectors, mappedSectors, highestMappedSector, err := devices.deviceStatus(info.DevName()); err != nil {
  859. return nil, err
  860. } else {
  861. status.SizeInSectors = sizeInSectors
  862. status.MappedSectors = mappedSectors
  863. status.HighestMappedSector = highestMappedSector
  864. }
  865. return status, nil
  866. }
  867. func (devices *DeviceSet) poolStatus() (totalSizeInSectors, transactionId, dataUsed, dataTotal, metadataUsed, metadataTotal uint64, err error) {
  868. var params string
  869. if _, totalSizeInSectors, _, params, err = getStatus(devices.getPoolName()); err == nil {
  870. _, err = fmt.Sscanf(params, "%d %d/%d %d/%d", &transactionId, &metadataUsed, &metadataTotal, &dataUsed, &dataTotal)
  871. }
  872. return
  873. }
  874. func (devices *DeviceSet) Status() *Status {
  875. devices.Lock()
  876. defer devices.Unlock()
  877. status := &Status{}
  878. status.PoolName = devices.getPoolName()
  879. status.DataLoopback = path.Join(devices.loopbackDir(), "data")
  880. status.MetadataLoopback = path.Join(devices.loopbackDir(), "metadata")
  881. totalSizeInSectors, _, dataUsed, dataTotal, metadataUsed, metadataTotal, err := devices.poolStatus()
  882. if err == nil {
  883. // Convert from blocks to bytes
  884. blockSizeInSectors := totalSizeInSectors / dataTotal
  885. status.Data.Used = dataUsed * blockSizeInSectors * 512
  886. status.Data.Total = dataTotal * blockSizeInSectors * 512
  887. // metadata blocks are always 4k
  888. status.Metadata.Used = metadataUsed * 4096
  889. status.Metadata.Total = metadataTotal * 4096
  890. status.SectorSize = blockSizeInSectors * 512
  891. }
  892. return status
  893. }
  894. func NewDeviceSet(root string, doInit bool) (*DeviceSet, error) {
  895. SetDevDir("/dev")
  896. devices := &DeviceSet{
  897. root: root,
  898. MetaData: MetaData{Devices: make(map[string]*DevInfo)},
  899. }
  900. if err := devices.initDevmapper(doInit); err != nil {
  901. return nil, err
  902. }
  903. return devices, nil
  904. }