deviceset.go 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968
  1. // +build linux,amd64
  2. package devmapper
  3. import (
  4. "encoding/json"
  5. "errors"
  6. "fmt"
  7. "github.com/dotcloud/docker/utils"
  8. "io"
  9. "io/ioutil"
  10. "path"
  11. "path/filepath"
  12. "strconv"
  13. "sync"
  14. "time"
  15. )
  16. var (
  17. DefaultDataLoopbackSize int64 = 100 * 1024 * 1024 * 1024
  18. DefaultMetaDataLoopbackSize int64 = 2 * 1024 * 1024 * 1024
  19. DefaultBaseFsSize uint64 = 10 * 1024 * 1024 * 1024
  20. )
  21. type DevInfo struct {
  22. Hash string `json:"-"`
  23. DeviceId int `json:"device_id"`
  24. Size uint64 `json:"size"`
  25. TransactionId uint64 `json:"transaction_id"`
  26. Initialized bool `json:"initialized"`
  27. devices *DeviceSet `json:"-"`
  28. }
  29. type MetaData struct {
  30. Devices map[string]*DevInfo `json:devices`
  31. }
  32. type DeviceSet struct {
  33. MetaData
  34. sync.Mutex
  35. root string
  36. devicePrefix string
  37. TransactionId uint64
  38. NewTransactionId uint64
  39. nextFreeDevice int
  40. activeMounts map[string]int
  41. }
  42. type DiskUsage struct {
  43. Used uint64
  44. Total uint64
  45. }
  46. type Status struct {
  47. PoolName string
  48. DataLoopback string
  49. MetadataLoopback string
  50. Data DiskUsage
  51. Metadata DiskUsage
  52. SectorSize uint64
  53. }
  54. type DevStatus struct {
  55. DeviceId int
  56. Size uint64
  57. TransactionId uint64
  58. SizeInSectors uint64
  59. MappedSectors uint64
  60. HighestMappedSector uint64
  61. }
  62. func getDevName(name string) string {
  63. return "/dev/mapper/" + name
  64. }
  65. func (info *DevInfo) Name() string {
  66. hash := info.Hash
  67. if hash == "" {
  68. hash = "base"
  69. }
  70. return fmt.Sprintf("%s-%s", info.devices.devicePrefix, hash)
  71. }
  72. func (info *DevInfo) DevName() string {
  73. return getDevName(info.Name())
  74. }
  75. func (devices *DeviceSet) loopbackDir() string {
  76. return path.Join(devices.root, "devicemapper")
  77. }
  78. func (devices *DeviceSet) jsonFile() string {
  79. return path.Join(devices.loopbackDir(), "json")
  80. }
  81. func (devices *DeviceSet) getPoolName() string {
  82. return devices.devicePrefix + "-pool"
  83. }
  84. func (devices *DeviceSet) getPoolDevName() string {
  85. return getDevName(devices.getPoolName())
  86. }
  87. func (devices *DeviceSet) hasImage(name string) bool {
  88. dirname := devices.loopbackDir()
  89. filename := path.Join(dirname, name)
  90. _, err := osStat(filename)
  91. return err == nil
  92. }
  93. // ensureImage creates a sparse file of <size> bytes at the path
  94. // <root>/devicemapper/<name>.
  95. // If the file already exists, it does nothing.
  96. // Either way it returns the full path.
  97. func (devices *DeviceSet) ensureImage(name string, size int64) (string, error) {
  98. dirname := devices.loopbackDir()
  99. filename := path.Join(dirname, name)
  100. if err := osMkdirAll(dirname, 0700); err != nil && !osIsExist(err) {
  101. return "", err
  102. }
  103. if _, err := osStat(filename); err != nil {
  104. if !osIsNotExist(err) {
  105. return "", err
  106. }
  107. utils.Debugf("Creating loopback file %s for device-manage use", filename)
  108. file, err := osOpenFile(filename, osORdWr|osOCreate, 0600)
  109. if err != nil {
  110. return "", err
  111. }
  112. defer file.Close()
  113. if err = file.Truncate(size); err != nil {
  114. return "", err
  115. }
  116. }
  117. return filename, nil
  118. }
  119. func (devices *DeviceSet) allocateDeviceId() int {
  120. // TODO: Add smarter reuse of deleted devices
  121. id := devices.nextFreeDevice
  122. devices.nextFreeDevice = devices.nextFreeDevice + 1
  123. return id
  124. }
  125. func (devices *DeviceSet) allocateTransactionId() uint64 {
  126. devices.NewTransactionId = devices.NewTransactionId + 1
  127. return devices.NewTransactionId
  128. }
  129. func (devices *DeviceSet) saveMetadata() error {
  130. jsonData, err := json.Marshal(devices.MetaData)
  131. if err != nil {
  132. return fmt.Errorf("Error encoding metadata to json: %s", err)
  133. }
  134. tmpFile, err := ioutil.TempFile(filepath.Dir(devices.jsonFile()), ".json")
  135. if err != nil {
  136. return fmt.Errorf("Error creating metadata file: %s", err)
  137. }
  138. n, err := tmpFile.Write(jsonData)
  139. if err != nil {
  140. return fmt.Errorf("Error writing metadata to %s: %s", tmpFile.Name(), err)
  141. }
  142. if n < len(jsonData) {
  143. return io.ErrShortWrite
  144. }
  145. if err := tmpFile.Sync(); err != nil {
  146. return fmt.Errorf("Error syncing metadata file %s: %s", tmpFile.Name(), err)
  147. }
  148. if err := tmpFile.Close(); err != nil {
  149. return fmt.Errorf("Error closing metadata file %s: %s", tmpFile.Name(), err)
  150. }
  151. if err := osRename(tmpFile.Name(), devices.jsonFile()); err != nil {
  152. return fmt.Errorf("Error committing metadata file %s: %s", tmpFile.Name(), err)
  153. }
  154. if devices.NewTransactionId != devices.TransactionId {
  155. if err = setTransactionId(devices.getPoolDevName(), devices.TransactionId, devices.NewTransactionId); err != nil {
  156. return fmt.Errorf("Error setting devmapper transition ID: %s", err)
  157. }
  158. devices.TransactionId = devices.NewTransactionId
  159. }
  160. return nil
  161. }
  162. func (devices *DeviceSet) registerDevice(id int, hash string, size uint64) (*DevInfo, error) {
  163. utils.Debugf("registerDevice(%v, %v)", id, hash)
  164. info := &DevInfo{
  165. Hash: hash,
  166. DeviceId: id,
  167. Size: size,
  168. TransactionId: devices.allocateTransactionId(),
  169. Initialized: false,
  170. devices: devices,
  171. }
  172. devices.Devices[hash] = info
  173. if err := devices.saveMetadata(); err != nil {
  174. // Try to remove unused device
  175. delete(devices.Devices, hash)
  176. return nil, err
  177. }
  178. return info, nil
  179. }
  180. func (devices *DeviceSet) activateDeviceIfNeeded(hash string) error {
  181. utils.Debugf("activateDeviceIfNeeded(%v)", hash)
  182. info := devices.Devices[hash]
  183. if info == nil {
  184. return fmt.Errorf("Unknown device %s", hash)
  185. }
  186. if devinfo, _ := getInfo(info.Name()); devinfo != nil && devinfo.Exists != 0 {
  187. return nil
  188. }
  189. return activateDevice(devices.getPoolDevName(), info.Name(), info.DeviceId, info.Size)
  190. }
  191. func (devices *DeviceSet) createFilesystem(info *DevInfo) error {
  192. devname := info.DevName()
  193. err := execRun("mkfs.ext4", "-E", "discard,lazy_itable_init=0,lazy_journal_init=0", devname)
  194. if err != nil {
  195. err = execRun("mkfs.ext4", "-E", "discard,lazy_itable_init=0", devname)
  196. }
  197. if err != nil {
  198. utils.Debugf("\n--->Err: %s\n", err)
  199. return err
  200. }
  201. return nil
  202. }
  203. func (devices *DeviceSet) loadMetaData() error {
  204. utils.Debugf("loadMetadata()")
  205. defer utils.Debugf("loadMetadata END")
  206. _, _, _, params, err := getStatus(devices.getPoolName())
  207. if err != nil {
  208. utils.Debugf("\n--->Err: %s\n", err)
  209. return err
  210. }
  211. if _, err := fmt.Sscanf(params, "%d", &devices.TransactionId); err != nil {
  212. utils.Debugf("\n--->Err: %s\n", err)
  213. return err
  214. }
  215. devices.NewTransactionId = devices.TransactionId
  216. jsonData, err := ioutil.ReadFile(devices.jsonFile())
  217. if err != nil && !osIsNotExist(err) {
  218. utils.Debugf("\n--->Err: %s\n", err)
  219. return err
  220. }
  221. devices.MetaData.Devices = make(map[string]*DevInfo)
  222. if jsonData != nil {
  223. if err := json.Unmarshal(jsonData, &devices.MetaData); err != nil {
  224. utils.Debugf("\n--->Err: %s\n", err)
  225. return err
  226. }
  227. }
  228. for hash, d := range devices.Devices {
  229. d.Hash = hash
  230. d.devices = devices
  231. if d.DeviceId >= devices.nextFreeDevice {
  232. devices.nextFreeDevice = d.DeviceId + 1
  233. }
  234. // If the transaction id is larger than the actual one we lost the device due to some crash
  235. if d.TransactionId > devices.TransactionId {
  236. utils.Debugf("Removing lost device %s with id %d", hash, d.TransactionId)
  237. delete(devices.Devices, hash)
  238. }
  239. }
  240. return nil
  241. }
  242. func (devices *DeviceSet) setupBaseImage() error {
  243. oldInfo := devices.Devices[""]
  244. if oldInfo != nil && oldInfo.Initialized {
  245. return nil
  246. }
  247. if oldInfo != nil && !oldInfo.Initialized {
  248. utils.Debugf("Removing uninitialized base image")
  249. if err := devices.removeDevice(""); err != nil {
  250. utils.Debugf("\n--->Err: %s\n", err)
  251. return err
  252. }
  253. }
  254. utils.Debugf("Initializing base device-manager snapshot")
  255. id := devices.allocateDeviceId()
  256. // Create initial device
  257. if err := createDevice(devices.getPoolDevName(), id); err != nil {
  258. utils.Debugf("\n--->Err: %s\n", err)
  259. return err
  260. }
  261. utils.Debugf("Registering base device (id %v) with FS size %v", id, DefaultBaseFsSize)
  262. info, err := devices.registerDevice(id, "", DefaultBaseFsSize)
  263. if err != nil {
  264. _ = deleteDevice(devices.getPoolDevName(), id)
  265. utils.Debugf("\n--->Err: %s\n", err)
  266. return err
  267. }
  268. utils.Debugf("Creating filesystem on base device-manager snapshot")
  269. if err = devices.activateDeviceIfNeeded(""); err != nil {
  270. utils.Debugf("\n--->Err: %s\n", err)
  271. return err
  272. }
  273. if err := devices.createFilesystem(info); err != nil {
  274. utils.Debugf("\n--->Err: %s\n", err)
  275. return err
  276. }
  277. info.Initialized = true
  278. if err = devices.saveMetadata(); err != nil {
  279. info.Initialized = false
  280. utils.Debugf("\n--->Err: %s\n", err)
  281. return err
  282. }
  283. return nil
  284. }
  285. func setCloseOnExec(name string) {
  286. if fileInfos, _ := ioutil.ReadDir("/proc/self/fd"); fileInfos != nil {
  287. for _, i := range fileInfos {
  288. link, _ := osReadlink(filepath.Join("/proc/self/fd", i.Name()))
  289. if link == name {
  290. fd, err := strconv.Atoi(i.Name())
  291. if err == nil {
  292. sysCloseOnExec(fd)
  293. }
  294. }
  295. }
  296. }
  297. }
  298. func (devices *DeviceSet) log(level int, file string, line int, dmError int, message string) {
  299. if level >= 7 {
  300. return // Ignore _LOG_DEBUG
  301. }
  302. utils.Debugf("libdevmapper(%d): %s:%d (%d) %s", level, file, line, dmError, message)
  303. }
  304. func major(device uint64) uint64 {
  305. return (device >> 8) & 0xfff
  306. }
  307. func minor(device uint64) uint64 {
  308. return (device & 0xff) | ((device >> 12) & 0xfff00)
  309. }
  310. func (devices *DeviceSet) ResizePool(size int64) error {
  311. dirname := devices.loopbackDir()
  312. datafilename := path.Join(dirname, "data")
  313. metadatafilename := path.Join(dirname, "metadata")
  314. datafile, err := osOpenFile(datafilename, osORdWr, 0)
  315. if datafile == nil {
  316. return err
  317. }
  318. defer datafile.Close()
  319. fi, err := datafile.Stat()
  320. if fi == nil {
  321. return err
  322. }
  323. if fi.Size() > size {
  324. return fmt.Errorf("Can't shrink file")
  325. }
  326. dataloopback := FindLoopDeviceFor(datafile)
  327. if dataloopback == nil {
  328. return fmt.Errorf("Unable to find loopback mount for: %s", datafilename)
  329. }
  330. defer dataloopback.Close()
  331. metadatafile, err := osOpenFile(metadatafilename, osORdWr, 0)
  332. if metadatafile == nil {
  333. return err
  334. }
  335. defer metadatafile.Close()
  336. metadataloopback := FindLoopDeviceFor(metadatafile)
  337. if metadataloopback == nil {
  338. return fmt.Errorf("Unable to find loopback mount for: %s", metadatafilename)
  339. }
  340. defer metadataloopback.Close()
  341. // Grow loopback file
  342. if err := datafile.Truncate(size); err != nil {
  343. return fmt.Errorf("Unable to grow loopback file: %s", err)
  344. }
  345. // Reload size for loopback device
  346. if err := LoopbackSetCapacity(dataloopback); err != nil {
  347. return fmt.Errorf("Unable to update loopback capacity: %s", err)
  348. }
  349. // Suspend the pool
  350. if err := suspendDevice(devices.getPoolName()); err != nil {
  351. return fmt.Errorf("Unable to suspend pool: %s", err)
  352. }
  353. // Reload with the new block sizes
  354. if err := reloadPool(devices.getPoolName(), dataloopback, metadataloopback); err != nil {
  355. return fmt.Errorf("Unable to reload pool: %s", err)
  356. }
  357. // Resume the pool
  358. if err := resumeDevice(devices.getPoolName()); err != nil {
  359. return fmt.Errorf("Unable to resume pool: %s", err)
  360. }
  361. return nil
  362. }
  363. func (devices *DeviceSet) initDevmapper(doInit bool) error {
  364. logInit(devices)
  365. // Make sure the sparse images exist in <root>/devicemapper/data and
  366. // <root>/devicemapper/metadata
  367. hasData := devices.hasImage("data")
  368. hasMetadata := devices.hasImage("metadata")
  369. if !doInit && !hasData {
  370. return errors.New("Loopback data file not found")
  371. }
  372. if !doInit && !hasMetadata {
  373. return errors.New("Loopback metadata file not found")
  374. }
  375. createdLoopback := !hasData || !hasMetadata
  376. data, err := devices.ensureImage("data", DefaultDataLoopbackSize)
  377. if err != nil {
  378. utils.Debugf("Error device ensureImage (data): %s\n", err)
  379. return err
  380. }
  381. metadata, err := devices.ensureImage("metadata", DefaultMetaDataLoopbackSize)
  382. if err != nil {
  383. utils.Debugf("Error device ensureImage (metadata): %s\n", err)
  384. return err
  385. }
  386. // Set the device prefix from the device id and inode of the docker root dir
  387. st, err := osStat(devices.root)
  388. if err != nil {
  389. return fmt.Errorf("Error looking up dir %s: %s", devices.root, err)
  390. }
  391. sysSt := toSysStatT(st.Sys())
  392. // "reg-" stands for "regular file".
  393. // In the future we might use "dev-" for "device file", etc.
  394. // docker-maj,min[-inode] stands for:
  395. // - Managed by docker
  396. // - The target of this device is at major <maj> and minor <min>
  397. // - If <inode> is defined, use that file inside the device as a loopback image. Otherwise use the device itself.
  398. devices.devicePrefix = fmt.Sprintf("docker-%d:%d-%d", major(sysSt.Dev), minor(sysSt.Dev), sysSt.Ino)
  399. utils.Debugf("Generated prefix: %s", devices.devicePrefix)
  400. // Check for the existence of the device <prefix>-pool
  401. utils.Debugf("Checking for existence of the pool '%s'", devices.getPoolName())
  402. info, err := getInfo(devices.getPoolName())
  403. if info == nil {
  404. utils.Debugf("Error device getInfo: %s", err)
  405. return err
  406. }
  407. // It seems libdevmapper opens this without O_CLOEXEC, and go exec will not close files
  408. // that are not Close-on-exec, and lxc-start will die if it inherits any unexpected files,
  409. // so we add this badhack to make sure it closes itself
  410. setCloseOnExec("/dev/mapper/control")
  411. // If the pool doesn't exist, create it
  412. if info.Exists == 0 {
  413. utils.Debugf("Pool doesn't exist. Creating it.")
  414. dataFile, err := attachLoopDevice(data)
  415. if err != nil {
  416. utils.Debugf("\n--->Err: %s\n", err)
  417. return err
  418. }
  419. defer dataFile.Close()
  420. metadataFile, err := attachLoopDevice(metadata)
  421. if err != nil {
  422. utils.Debugf("\n--->Err: %s\n", err)
  423. return err
  424. }
  425. defer metadataFile.Close()
  426. if err := createPool(devices.getPoolName(), dataFile, metadataFile); err != nil {
  427. utils.Debugf("\n--->Err: %s\n", err)
  428. return err
  429. }
  430. }
  431. // If we didn't just create the data or metadata image, we need to
  432. // load the metadata from the existing file.
  433. if !createdLoopback {
  434. if err = devices.loadMetaData(); err != nil {
  435. utils.Debugf("\n--->Err: %s\n", err)
  436. return err
  437. }
  438. }
  439. // Setup the base image
  440. if doInit {
  441. if err := devices.setupBaseImage(); err != nil {
  442. utils.Debugf("Error device setupBaseImage: %s\n", err)
  443. return err
  444. }
  445. }
  446. return nil
  447. }
  448. func (devices *DeviceSet) AddDevice(hash, baseHash string) error {
  449. devices.Lock()
  450. defer devices.Unlock()
  451. if devices.Devices[hash] != nil {
  452. return fmt.Errorf("hash %s already exists", hash)
  453. }
  454. baseInfo := devices.Devices[baseHash]
  455. if baseInfo == nil {
  456. return fmt.Errorf("Error adding device for '%s': can't find device for parent '%s'", hash, baseHash)
  457. }
  458. deviceId := devices.allocateDeviceId()
  459. if err := devices.createSnapDevice(devices.getPoolDevName(), deviceId, baseInfo.Name(), baseInfo.DeviceId); err != nil {
  460. utils.Debugf("Error creating snap device: %s\n", err)
  461. return err
  462. }
  463. if _, err := devices.registerDevice(deviceId, hash, baseInfo.Size); err != nil {
  464. deleteDevice(devices.getPoolDevName(), deviceId)
  465. utils.Debugf("Error registering device: %s\n", err)
  466. return err
  467. }
  468. return nil
  469. }
  470. func (devices *DeviceSet) removeDevice(hash string) error {
  471. info := devices.Devices[hash]
  472. if info == nil {
  473. return fmt.Errorf("hash %s doesn't exists", hash)
  474. }
  475. // This is a workaround for the kernel not discarding block so
  476. // on the thin pool when we remove a thinp device, so we do it
  477. // manually
  478. if err := devices.activateDeviceIfNeeded(hash); err == nil {
  479. if err := BlockDeviceDiscard(info.DevName()); err != nil {
  480. utils.Debugf("Error discarding block on device: %s (ignoring)\n", err)
  481. }
  482. }
  483. devinfo, _ := getInfo(info.Name())
  484. if devinfo != nil && devinfo.Exists != 0 {
  485. if err := removeDevice(info.Name()); err != nil {
  486. utils.Debugf("Error removing device: %s\n", err)
  487. return err
  488. }
  489. }
  490. if info.Initialized {
  491. info.Initialized = false
  492. if err := devices.saveMetadata(); err != nil {
  493. utils.Debugf("Error saving meta data: %s\n", err)
  494. return err
  495. }
  496. }
  497. if err := deleteDevice(devices.getPoolDevName(), info.DeviceId); err != nil {
  498. utils.Debugf("Error deleting device: %s\n", err)
  499. return err
  500. }
  501. devices.allocateTransactionId()
  502. delete(devices.Devices, info.Hash)
  503. if err := devices.saveMetadata(); err != nil {
  504. devices.Devices[info.Hash] = info
  505. utils.Debugf("Error saving meta data: %s\n", err)
  506. return err
  507. }
  508. return nil
  509. }
  510. func (devices *DeviceSet) RemoveDevice(hash string) error {
  511. devices.Lock()
  512. defer devices.Unlock()
  513. return devices.removeDevice(hash)
  514. }
  515. func (devices *DeviceSet) deactivateDevice(hash string) error {
  516. utils.Debugf("[devmapper] deactivateDevice(%s)", hash)
  517. defer utils.Debugf("[devmapper] deactivateDevice END")
  518. var devname string
  519. // FIXME: shouldn't we just register the pool into devices?
  520. devname, err := devices.byHash(hash)
  521. if err != nil {
  522. return err
  523. }
  524. devinfo, err := getInfo(devname)
  525. if err != nil {
  526. utils.Debugf("\n--->Err: %s\n", err)
  527. return err
  528. }
  529. if devinfo.Exists != 0 {
  530. if err := removeDevice(devname); err != nil {
  531. utils.Debugf("\n--->Err: %s\n", err)
  532. return err
  533. }
  534. if err := devices.waitRemove(hash); err != nil {
  535. return err
  536. }
  537. }
  538. return nil
  539. }
  540. // waitRemove blocks until either:
  541. // a) the device registered at <device_set_prefix>-<hash> is removed,
  542. // or b) the 1 second timeout expires.
  543. func (devices *DeviceSet) waitRemove(hash string) error {
  544. utils.Debugf("[deviceset %s] waitRemove(%s)", devices.devicePrefix, hash)
  545. defer utils.Debugf("[deviceset %s] waitRemove(%) END", devices.devicePrefix, hash)
  546. devname, err := devices.byHash(hash)
  547. if err != nil {
  548. return err
  549. }
  550. i := 0
  551. for ; i < 1000; i += 1 {
  552. devinfo, err := getInfo(devname)
  553. if err != nil {
  554. // If there is an error we assume the device doesn't exist.
  555. // The error might actually be something else, but we can't differentiate.
  556. return nil
  557. }
  558. if i%100 == 0 {
  559. utils.Debugf("Waiting for removal of %s: exists=%d", devname, devinfo.Exists)
  560. }
  561. if devinfo.Exists == 0 {
  562. break
  563. }
  564. time.Sleep(1 * time.Millisecond)
  565. }
  566. if i == 1000 {
  567. return fmt.Errorf("Timeout while waiting for device %s to be removed", devname)
  568. }
  569. return nil
  570. }
  571. // waitClose blocks until either:
  572. // a) the device registered at <device_set_prefix>-<hash> is closed,
  573. // or b) the 1 second timeout expires.
  574. func (devices *DeviceSet) waitClose(hash string) error {
  575. devname, err := devices.byHash(hash)
  576. if err != nil {
  577. return err
  578. }
  579. i := 0
  580. for ; i < 1000; i += 1 {
  581. devinfo, err := getInfo(devname)
  582. if err != nil {
  583. return err
  584. }
  585. if i%100 == 0 {
  586. utils.Debugf("Waiting for unmount of %s: opencount=%d", devname, devinfo.OpenCount)
  587. }
  588. if devinfo.OpenCount == 0 {
  589. break
  590. }
  591. time.Sleep(1 * time.Millisecond)
  592. }
  593. if i == 1000 {
  594. return fmt.Errorf("Timeout while waiting for device %s to close", devname)
  595. }
  596. return nil
  597. }
  598. // byHash is a hack to allow looking up the deviceset's pool by the hash "pool".
  599. // FIXME: it seems probably cleaner to register the pool in devices.Devices,
  600. // but I am afraid of arcane implications deep in the devicemapper code,
  601. // so this will do.
  602. func (devices *DeviceSet) byHash(hash string) (devname string, err error) {
  603. if hash == "pool" {
  604. return devices.getPoolDevName(), nil
  605. }
  606. info := devices.Devices[hash]
  607. if info == nil {
  608. return "", fmt.Errorf("hash %s doesn't exists", hash)
  609. }
  610. return info.Name(), nil
  611. }
  612. func (devices *DeviceSet) Shutdown() error {
  613. devices.Lock()
  614. defer devices.Unlock()
  615. utils.Debugf("[deviceset %s] shutdown()", devices.devicePrefix)
  616. utils.Debugf("[devmapper] Shutting down DeviceSet: %s", devices.root)
  617. defer utils.Debugf("[deviceset %s] shutdown END", devices.devicePrefix)
  618. for path, count := range devices.activeMounts {
  619. for i := count; i > 0; i-- {
  620. if err := sysUnmount(path, 0); err != nil {
  621. utils.Debugf("Shutdown unmounting %s, error: %s\n", path, err)
  622. }
  623. }
  624. delete(devices.activeMounts, path)
  625. }
  626. for _, d := range devices.Devices {
  627. if err := devices.waitClose(d.Hash); err != nil {
  628. utils.Errorf("Warning: error waiting for device %s to unmount: %s\n", d.Hash, err)
  629. }
  630. if err := devices.deactivateDevice(d.Hash); err != nil {
  631. utils.Debugf("Shutdown deactivate %s , error: %s\n", d.Hash, err)
  632. }
  633. }
  634. pool := devices.getPoolDevName()
  635. if devinfo, err := getInfo(pool); err == nil && devinfo.Exists != 0 {
  636. if err := devices.deactivateDevice("pool"); err != nil {
  637. utils.Debugf("Shutdown deactivate %s , error: %s\n", pool, err)
  638. }
  639. }
  640. return nil
  641. }
  642. func (devices *DeviceSet) MountDevice(hash, path string, readOnly bool) error {
  643. devices.Lock()
  644. defer devices.Unlock()
  645. if err := devices.activateDeviceIfNeeded(hash); err != nil {
  646. return fmt.Errorf("Error activating devmapper device for '%s': %s", hash, err)
  647. }
  648. info := devices.Devices[hash]
  649. var flags uintptr = sysMsMgcVal
  650. if readOnly {
  651. flags = flags | sysMsRdOnly
  652. }
  653. err := sysMount(info.DevName(), path, "ext4", flags, "discard")
  654. if err != nil && err == sysEInval {
  655. err = sysMount(info.DevName(), path, "ext4", flags, "")
  656. }
  657. if err != nil {
  658. return fmt.Errorf("Error mounting '%s' on '%s': %s", info.DevName(), path, err)
  659. }
  660. count := devices.activeMounts[path]
  661. devices.activeMounts[path] = count + 1
  662. return devices.setInitialized(hash)
  663. }
  664. func (devices *DeviceSet) UnmountDevice(hash, path string, deactivate bool) error {
  665. utils.Debugf("[devmapper] UnmountDevice(hash=%s path=%s)", hash, path)
  666. defer utils.Debugf("[devmapper] UnmountDevice END")
  667. devices.Lock()
  668. defer devices.Unlock()
  669. utils.Debugf("[devmapper] Unmount(%s)", path)
  670. if err := sysUnmount(path, 0); err != nil {
  671. utils.Debugf("\n--->Err: %s\n", err)
  672. return err
  673. }
  674. utils.Debugf("[devmapper] Unmount done")
  675. // Wait for the unmount to be effective,
  676. // by watching the value of Info.OpenCount for the device
  677. if err := devices.waitClose(hash); err != nil {
  678. return err
  679. }
  680. if count := devices.activeMounts[path]; count > 1 {
  681. devices.activeMounts[path] = count - 1
  682. } else {
  683. delete(devices.activeMounts, path)
  684. }
  685. if deactivate {
  686. devices.deactivateDevice(hash)
  687. }
  688. return nil
  689. }
  690. func (devices *DeviceSet) HasDevice(hash string) bool {
  691. devices.Lock()
  692. defer devices.Unlock()
  693. return devices.Devices[hash] != nil
  694. }
  695. func (devices *DeviceSet) HasInitializedDevice(hash string) bool {
  696. devices.Lock()
  697. defer devices.Unlock()
  698. info := devices.Devices[hash]
  699. return info != nil && info.Initialized
  700. }
  701. func (devices *DeviceSet) HasActivatedDevice(hash string) bool {
  702. devices.Lock()
  703. defer devices.Unlock()
  704. info := devices.Devices[hash]
  705. if info == nil {
  706. return false
  707. }
  708. devinfo, _ := getInfo(info.Name())
  709. return devinfo != nil && devinfo.Exists != 0
  710. }
  711. func (devices *DeviceSet) setInitialized(hash string) error {
  712. info := devices.Devices[hash]
  713. if info == nil {
  714. return fmt.Errorf("Unknown device %s", hash)
  715. }
  716. info.Initialized = true
  717. if err := devices.saveMetadata(); err != nil {
  718. info.Initialized = false
  719. utils.Debugf("\n--->Err: %s\n", err)
  720. return err
  721. }
  722. return nil
  723. }
  724. func (devices *DeviceSet) List() []string {
  725. devices.Lock()
  726. defer devices.Unlock()
  727. ids := make([]string, len(devices.Devices))
  728. i := 0
  729. for k := range devices.Devices {
  730. ids[i] = k
  731. i++
  732. }
  733. return ids
  734. }
  735. func (devices *DeviceSet) deviceStatus(devName string) (sizeInSectors, mappedSectors, highestMappedSector uint64, err error) {
  736. var params string
  737. _, sizeInSectors, _, params, err = getStatus(devName)
  738. if err != nil {
  739. return
  740. }
  741. if _, err = fmt.Sscanf(params, "%d %d", &mappedSectors, &highestMappedSector); err == nil {
  742. return
  743. }
  744. return
  745. }
  746. func (devices *DeviceSet) GetDeviceStatus(hash string) (*DevStatus, error) {
  747. devices.Lock()
  748. defer devices.Unlock()
  749. info := devices.Devices[hash]
  750. if info == nil {
  751. return nil, fmt.Errorf("No device %s", hash)
  752. }
  753. status := &DevStatus{
  754. DeviceId: info.DeviceId,
  755. Size: info.Size,
  756. TransactionId: info.TransactionId,
  757. }
  758. if err := devices.activateDeviceIfNeeded(hash); err != nil {
  759. return nil, fmt.Errorf("Error activating devmapper device for '%s': %s", hash, err)
  760. }
  761. if sizeInSectors, mappedSectors, highestMappedSector, err := devices.deviceStatus(info.DevName()); err != nil {
  762. return nil, err
  763. } else {
  764. status.SizeInSectors = sizeInSectors
  765. status.MappedSectors = mappedSectors
  766. status.HighestMappedSector = highestMappedSector
  767. }
  768. return status, nil
  769. }
  770. func (devices *DeviceSet) poolStatus() (totalSizeInSectors, transactionId, dataUsed, dataTotal, metadataUsed, metadataTotal uint64, err error) {
  771. var params string
  772. if _, totalSizeInSectors, _, params, err = getStatus(devices.getPoolName()); err == nil {
  773. _, err = fmt.Sscanf(params, "%d %d/%d %d/%d", &transactionId, &metadataUsed, &metadataTotal, &dataUsed, &dataTotal)
  774. }
  775. return
  776. }
  777. func (devices *DeviceSet) Status() *Status {
  778. devices.Lock()
  779. defer devices.Unlock()
  780. status := &Status{}
  781. status.PoolName = devices.getPoolName()
  782. status.DataLoopback = path.Join(devices.loopbackDir(), "data")
  783. status.MetadataLoopback = path.Join(devices.loopbackDir(), "metadata")
  784. totalSizeInSectors, _, dataUsed, dataTotal, metadataUsed, metadataTotal, err := devices.poolStatus()
  785. if err == nil {
  786. // Convert from blocks to bytes
  787. blockSizeInSectors := totalSizeInSectors / dataTotal
  788. status.Data.Used = dataUsed * blockSizeInSectors * 512
  789. status.Data.Total = dataTotal * blockSizeInSectors * 512
  790. // metadata blocks are always 4k
  791. status.Metadata.Used = metadataUsed * 4096
  792. status.Metadata.Total = metadataTotal * 4096
  793. status.SectorSize = blockSizeInSectors * 512
  794. }
  795. return status
  796. }
  797. func NewDeviceSet(root string, doInit bool) (*DeviceSet, error) {
  798. SetDevDir("/dev")
  799. devices := &DeviceSet{
  800. root: root,
  801. MetaData: MetaData{Devices: make(map[string]*DevInfo)},
  802. activeMounts: make(map[string]int),
  803. }
  804. if err := devices.initDevmapper(doInit); err != nil {
  805. return nil, err
  806. }
  807. return devices, nil
  808. }