deviceset.go 30 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189
  1. // +build linux,amd64
  2. package devmapper
  3. import (
  4. "encoding/json"
  5. "errors"
  6. "fmt"
  7. "io"
  8. "io/ioutil"
  9. "os"
  10. "os/exec"
  11. "path"
  12. "path/filepath"
  13. "strconv"
  14. "strings"
  15. "sync"
  16. "syscall"
  17. "time"
  18. "github.com/dotcloud/docker/daemon/graphdriver"
  19. "github.com/dotcloud/docker/pkg/label"
  20. "github.com/dotcloud/docker/pkg/units"
  21. "github.com/dotcloud/docker/utils"
  22. )
  23. var (
  24. DefaultDataLoopbackSize int64 = 100 * 1024 * 1024 * 1024
  25. DefaultMetaDataLoopbackSize int64 = 2 * 1024 * 1024 * 1024
  26. DefaultBaseFsSize uint64 = 10 * 1024 * 1024 * 1024
  27. )
  28. type DevInfo struct {
  29. Hash string `json:"-"`
  30. DeviceId int `json:"device_id"`
  31. Size uint64 `json:"size"`
  32. TransactionId uint64 `json:"transaction_id"`
  33. Initialized bool `json:"initialized"`
  34. devices *DeviceSet `json:"-"`
  35. mountCount int `json:"-"`
  36. mountPath string `json:"-"`
  37. // The global DeviceSet lock guarantees that we serialize all
  38. // the calls to libdevmapper (which is not threadsafe), but we
  39. // sometimes release that lock while sleeping. In that case
  40. // this per-device lock is still held, protecting against
  41. // other accesses to the device that we're doing the wait on.
  42. //
  43. // WARNING: In order to avoid AB-BA deadlocks when releasing
  44. // the global lock while holding the per-device locks all
  45. // device locks must be aquired *before* the device lock, and
  46. // multiple device locks should be aquired parent before child.
  47. lock sync.Mutex `json:"-"`
  48. }
  49. type MetaData struct {
  50. Devices map[string]*DevInfo `json:devices`
  51. devicesLock sync.Mutex `json:"-"` // Protects all read/writes to Devices map
  52. }
  53. type DeviceSet struct {
  54. MetaData
  55. sync.Mutex // Protects Devices map and serializes calls into libdevmapper
  56. root string
  57. devicePrefix string
  58. TransactionId uint64
  59. NewTransactionId uint64
  60. nextDeviceId int
  61. // Options
  62. dataLoopbackSize int64
  63. metaDataLoopbackSize int64
  64. baseFsSize uint64
  65. filesystem string
  66. mountOptions string
  67. mkfsArgs []string
  68. }
  69. type DiskUsage struct {
  70. Used uint64
  71. Total uint64
  72. }
  73. type Status struct {
  74. PoolName string
  75. DataLoopback string
  76. MetadataLoopback string
  77. Data DiskUsage
  78. Metadata DiskUsage
  79. SectorSize uint64
  80. }
  81. type DevStatus struct {
  82. DeviceId int
  83. Size uint64
  84. TransactionId uint64
  85. SizeInSectors uint64
  86. MappedSectors uint64
  87. HighestMappedSector uint64
  88. }
  89. func getDevName(name string) string {
  90. return "/dev/mapper/" + name
  91. }
  92. func (info *DevInfo) Name() string {
  93. hash := info.Hash
  94. if hash == "" {
  95. hash = "base"
  96. }
  97. return fmt.Sprintf("%s-%s", info.devices.devicePrefix, hash)
  98. }
  99. func (info *DevInfo) DevName() string {
  100. return getDevName(info.Name())
  101. }
  102. func (devices *DeviceSet) loopbackDir() string {
  103. return path.Join(devices.root, "devicemapper")
  104. }
  105. func (devices *DeviceSet) metadataDir() string {
  106. return path.Join(devices.root, "metadata")
  107. }
  108. func (devices *DeviceSet) metadataFile(info *DevInfo) string {
  109. file := info.Hash
  110. if file == "" {
  111. file = "base"
  112. }
  113. return path.Join(devices.metadataDir(), file)
  114. }
  115. func (devices *DeviceSet) oldMetadataFile() string {
  116. return path.Join(devices.loopbackDir(), "json")
  117. }
  118. func (devices *DeviceSet) getPoolName() string {
  119. return devices.devicePrefix + "-pool"
  120. }
  121. func (devices *DeviceSet) getPoolDevName() string {
  122. return getDevName(devices.getPoolName())
  123. }
  124. func (devices *DeviceSet) hasImage(name string) bool {
  125. dirname := devices.loopbackDir()
  126. filename := path.Join(dirname, name)
  127. _, err := os.Stat(filename)
  128. return err == nil
  129. }
  130. // ensureImage creates a sparse file of <size> bytes at the path
  131. // <root>/devicemapper/<name>.
  132. // If the file already exists, it does nothing.
  133. // Either way it returns the full path.
  134. func (devices *DeviceSet) ensureImage(name string, size int64) (string, error) {
  135. dirname := devices.loopbackDir()
  136. filename := path.Join(dirname, name)
  137. if err := os.MkdirAll(dirname, 0700); err != nil && !os.IsExist(err) {
  138. return "", err
  139. }
  140. if _, err := os.Stat(filename); err != nil {
  141. if !os.IsNotExist(err) {
  142. return "", err
  143. }
  144. utils.Debugf("Creating loopback file %s for device-manage use", filename)
  145. file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0600)
  146. if err != nil {
  147. return "", err
  148. }
  149. defer file.Close()
  150. if err = file.Truncate(size); err != nil {
  151. return "", err
  152. }
  153. }
  154. return filename, nil
  155. }
  156. func (devices *DeviceSet) allocateTransactionId() uint64 {
  157. devices.NewTransactionId = devices.NewTransactionId + 1
  158. return devices.NewTransactionId
  159. }
  160. func (devices *DeviceSet) removeMetadata(info *DevInfo) error {
  161. if err := os.RemoveAll(devices.metadataFile(info)); err != nil {
  162. return fmt.Errorf("Error removing metadata file %s: %s", devices.metadataFile(info), err)
  163. }
  164. return nil
  165. }
  166. func (devices *DeviceSet) saveMetadata(info *DevInfo) error {
  167. jsonData, err := json.Marshal(info)
  168. if err != nil {
  169. return fmt.Errorf("Error encoding metadata to json: %s", err)
  170. }
  171. tmpFile, err := ioutil.TempFile(devices.metadataDir(), ".tmp")
  172. if err != nil {
  173. return fmt.Errorf("Error creating metadata file: %s", err)
  174. }
  175. n, err := tmpFile.Write(jsonData)
  176. if err != nil {
  177. return fmt.Errorf("Error writing metadata to %s: %s", tmpFile.Name(), err)
  178. }
  179. if n < len(jsonData) {
  180. return io.ErrShortWrite
  181. }
  182. if err := tmpFile.Sync(); err != nil {
  183. return fmt.Errorf("Error syncing metadata file %s: %s", tmpFile.Name(), err)
  184. }
  185. if err := tmpFile.Close(); err != nil {
  186. return fmt.Errorf("Error closing metadata file %s: %s", tmpFile.Name(), err)
  187. }
  188. if err := os.Rename(tmpFile.Name(), devices.metadataFile(info)); err != nil {
  189. return fmt.Errorf("Error committing metadata file %s: %s", tmpFile.Name(), err)
  190. }
  191. if devices.NewTransactionId != devices.TransactionId {
  192. if err = setTransactionId(devices.getPoolDevName(), devices.TransactionId, devices.NewTransactionId); err != nil {
  193. return fmt.Errorf("Error setting devmapper transition ID: %s", err)
  194. }
  195. devices.TransactionId = devices.NewTransactionId
  196. }
  197. return nil
  198. }
  199. func (devices *DeviceSet) lookupDevice(hash string) (*DevInfo, error) {
  200. devices.devicesLock.Lock()
  201. defer devices.devicesLock.Unlock()
  202. info := devices.Devices[hash]
  203. if info == nil {
  204. info = devices.loadMetadata(hash)
  205. if info == nil {
  206. return nil, fmt.Errorf("Unknown device %s", hash)
  207. }
  208. devices.Devices[hash] = info
  209. }
  210. return info, nil
  211. }
  212. func (devices *DeviceSet) registerDevice(id int, hash string, size uint64) (*DevInfo, error) {
  213. utils.Debugf("registerDevice(%v, %v)", id, hash)
  214. info := &DevInfo{
  215. Hash: hash,
  216. DeviceId: id,
  217. Size: size,
  218. TransactionId: devices.allocateTransactionId(),
  219. Initialized: false,
  220. devices: devices,
  221. }
  222. devices.devicesLock.Lock()
  223. devices.Devices[hash] = info
  224. devices.devicesLock.Unlock()
  225. if err := devices.saveMetadata(info); err != nil {
  226. // Try to remove unused device
  227. devices.devicesLock.Lock()
  228. delete(devices.Devices, hash)
  229. devices.devicesLock.Unlock()
  230. return nil, err
  231. }
  232. return info, nil
  233. }
  234. func (devices *DeviceSet) activateDeviceIfNeeded(info *DevInfo) error {
  235. utils.Debugf("activateDeviceIfNeeded(%v)", info.Hash)
  236. if devinfo, _ := getInfo(info.Name()); devinfo != nil && devinfo.Exists != 0 {
  237. return nil
  238. }
  239. return activateDevice(devices.getPoolDevName(), info.Name(), info.DeviceId, info.Size)
  240. }
  241. func (devices *DeviceSet) createFilesystem(info *DevInfo) error {
  242. devname := info.DevName()
  243. args := []string{}
  244. for _, arg := range devices.mkfsArgs {
  245. args = append(args, arg)
  246. }
  247. args = append(args, devname)
  248. var err error
  249. switch devices.filesystem {
  250. case "xfs":
  251. err = exec.Command("mkfs.xfs", args...).Run()
  252. case "ext4":
  253. err = exec.Command("mkfs.ext4", append([]string{"-E", "nodiscard,lazy_itable_init=0,lazy_journal_init=0"}, args...)...).Run()
  254. if err != nil {
  255. err = exec.Command("mkfs.ext4", append([]string{"-E", "nodiscard,lazy_itable_init=0"}, args...)...).Run()
  256. }
  257. default:
  258. err = fmt.Errorf("Unsupported filesystem type %s", devices.filesystem)
  259. }
  260. if err != nil {
  261. utils.Debugf("\n--->Err: %s\n", err)
  262. return err
  263. }
  264. return nil
  265. }
  266. func (devices *DeviceSet) initMetaData() error {
  267. _, _, _, params, err := getStatus(devices.getPoolName())
  268. if err != nil {
  269. utils.Debugf("\n--->Err: %s\n", err)
  270. return err
  271. }
  272. if _, err := fmt.Sscanf(params, "%d", &devices.TransactionId); err != nil {
  273. utils.Debugf("\n--->Err: %s\n", err)
  274. return err
  275. }
  276. devices.NewTransactionId = devices.TransactionId
  277. // Migrate old metadatafile
  278. jsonData, err := ioutil.ReadFile(devices.oldMetadataFile())
  279. if err != nil && !os.IsNotExist(err) {
  280. utils.Debugf("\n--->Err: %s\n", err)
  281. return err
  282. }
  283. if jsonData != nil {
  284. m := MetaData{Devices: make(map[string]*DevInfo)}
  285. if err := json.Unmarshal(jsonData, &m); err != nil {
  286. utils.Debugf("\n--->Err: %s\n", err)
  287. return err
  288. }
  289. for hash, info := range m.Devices {
  290. info.Hash = hash
  291. // If the transaction id is larger than the actual one we lost the device due to some crash
  292. if info.TransactionId <= devices.TransactionId {
  293. devices.saveMetadata(info)
  294. }
  295. }
  296. if err := os.Rename(devices.oldMetadataFile(), devices.oldMetadataFile()+".migrated"); err != nil {
  297. return err
  298. }
  299. }
  300. return nil
  301. }
  302. func (devices *DeviceSet) loadMetadata(hash string) *DevInfo {
  303. info := &DevInfo{Hash: hash, devices: devices}
  304. jsonData, err := ioutil.ReadFile(devices.metadataFile(info))
  305. if err != nil {
  306. return nil
  307. }
  308. if err := json.Unmarshal(jsonData, &info); err != nil {
  309. return nil
  310. }
  311. // If the transaction id is larger than the actual one we lost the device due to some crash
  312. if info.TransactionId > devices.TransactionId {
  313. return nil
  314. }
  315. return info
  316. }
  317. func (devices *DeviceSet) setupBaseImage() error {
  318. oldInfo, _ := devices.lookupDevice("")
  319. if oldInfo != nil && oldInfo.Initialized {
  320. return nil
  321. }
  322. if oldInfo != nil && !oldInfo.Initialized {
  323. utils.Debugf("Removing uninitialized base image")
  324. if err := devices.deleteDevice(oldInfo); err != nil {
  325. utils.Debugf("\n--->Err: %s\n", err)
  326. return err
  327. }
  328. }
  329. utils.Debugf("Initializing base device-manager snapshot")
  330. id := devices.nextDeviceId
  331. // Create initial device
  332. if err := createDevice(devices.getPoolDevName(), &id); err != nil {
  333. utils.Debugf("\n--->Err: %s\n", err)
  334. return err
  335. }
  336. // Ids are 24bit, so wrap around
  337. devices.nextDeviceId = (id + 1) & 0xffffff
  338. utils.Debugf("Registering base device (id %v) with FS size %v", id, devices.baseFsSize)
  339. info, err := devices.registerDevice(id, "", devices.baseFsSize)
  340. if err != nil {
  341. _ = deleteDevice(devices.getPoolDevName(), id)
  342. utils.Debugf("\n--->Err: %s\n", err)
  343. return err
  344. }
  345. utils.Debugf("Creating filesystem on base device-manager snapshot")
  346. if err = devices.activateDeviceIfNeeded(info); err != nil {
  347. utils.Debugf("\n--->Err: %s\n", err)
  348. return err
  349. }
  350. if err := devices.createFilesystem(info); err != nil {
  351. utils.Debugf("\n--->Err: %s\n", err)
  352. return err
  353. }
  354. info.Initialized = true
  355. if err = devices.saveMetadata(info); err != nil {
  356. info.Initialized = false
  357. utils.Debugf("\n--->Err: %s\n", err)
  358. return err
  359. }
  360. return nil
  361. }
  362. func setCloseOnExec(name string) {
  363. if fileInfos, _ := ioutil.ReadDir("/proc/self/fd"); fileInfos != nil {
  364. for _, i := range fileInfos {
  365. link, _ := os.Readlink(filepath.Join("/proc/self/fd", i.Name()))
  366. if link == name {
  367. fd, err := strconv.Atoi(i.Name())
  368. if err == nil {
  369. syscall.CloseOnExec(fd)
  370. }
  371. }
  372. }
  373. }
  374. }
  375. func (devices *DeviceSet) log(level int, file string, line int, dmError int, message string) {
  376. if level >= 7 {
  377. return // Ignore _LOG_DEBUG
  378. }
  379. utils.Debugf("libdevmapper(%d): %s:%d (%d) %s", level, file, line, dmError, message)
  380. }
  381. func major(device uint64) uint64 {
  382. return (device >> 8) & 0xfff
  383. }
  384. func minor(device uint64) uint64 {
  385. return (device & 0xff) | ((device >> 12) & 0xfff00)
  386. }
  387. func (devices *DeviceSet) ResizePool(size int64) error {
  388. dirname := devices.loopbackDir()
  389. datafilename := path.Join(dirname, "data")
  390. metadatafilename := path.Join(dirname, "metadata")
  391. datafile, err := os.OpenFile(datafilename, os.O_RDWR, 0)
  392. if datafile == nil {
  393. return err
  394. }
  395. defer datafile.Close()
  396. fi, err := datafile.Stat()
  397. if fi == nil {
  398. return err
  399. }
  400. if fi.Size() > size {
  401. return fmt.Errorf("Can't shrink file")
  402. }
  403. dataloopback := FindLoopDeviceFor(datafile)
  404. if dataloopback == nil {
  405. return fmt.Errorf("Unable to find loopback mount for: %s", datafilename)
  406. }
  407. defer dataloopback.Close()
  408. metadatafile, err := os.OpenFile(metadatafilename, os.O_RDWR, 0)
  409. if metadatafile == nil {
  410. return err
  411. }
  412. defer metadatafile.Close()
  413. metadataloopback := FindLoopDeviceFor(metadatafile)
  414. if metadataloopback == nil {
  415. return fmt.Errorf("Unable to find loopback mount for: %s", metadatafilename)
  416. }
  417. defer metadataloopback.Close()
  418. // Grow loopback file
  419. if err := datafile.Truncate(size); err != nil {
  420. return fmt.Errorf("Unable to grow loopback file: %s", err)
  421. }
  422. // Reload size for loopback device
  423. if err := LoopbackSetCapacity(dataloopback); err != nil {
  424. return fmt.Errorf("Unable to update loopback capacity: %s", err)
  425. }
  426. // Suspend the pool
  427. if err := suspendDevice(devices.getPoolName()); err != nil {
  428. return fmt.Errorf("Unable to suspend pool: %s", err)
  429. }
  430. // Reload with the new block sizes
  431. if err := reloadPool(devices.getPoolName(), dataloopback, metadataloopback); err != nil {
  432. return fmt.Errorf("Unable to reload pool: %s", err)
  433. }
  434. // Resume the pool
  435. if err := resumeDevice(devices.getPoolName()); err != nil {
  436. return fmt.Errorf("Unable to resume pool: %s", err)
  437. }
  438. return nil
  439. }
  440. func (devices *DeviceSet) initDevmapper(doInit bool) error {
  441. logInit(devices)
  442. _, err := getDriverVersion()
  443. if err != nil {
  444. // Can't even get driver version, assume not supported
  445. return graphdriver.ErrNotSupported
  446. }
  447. if err := os.MkdirAll(devices.metadataDir(), 0700); err != nil && !os.IsExist(err) {
  448. return err
  449. }
  450. // Set the device prefix from the device id and inode of the docker root dir
  451. st, err := os.Stat(devices.root)
  452. if err != nil {
  453. return fmt.Errorf("Error looking up dir %s: %s", devices.root, err)
  454. }
  455. sysSt := st.Sys().(*syscall.Stat_t)
  456. // "reg-" stands for "regular file".
  457. // In the future we might use "dev-" for "device file", etc.
  458. // docker-maj,min[-inode] stands for:
  459. // - Managed by docker
  460. // - The target of this device is at major <maj> and minor <min>
  461. // - If <inode> is defined, use that file inside the device as a loopback image. Otherwise use the device itself.
  462. devices.devicePrefix = fmt.Sprintf("docker-%d:%d-%d", major(sysSt.Dev), minor(sysSt.Dev), sysSt.Ino)
  463. utils.Debugf("Generated prefix: %s", devices.devicePrefix)
  464. // Check for the existence of the device <prefix>-pool
  465. utils.Debugf("Checking for existence of the pool '%s'", devices.getPoolName())
  466. info, err := getInfo(devices.getPoolName())
  467. if info == nil {
  468. utils.Debugf("Error device getInfo: %s", err)
  469. return err
  470. }
  471. // It seems libdevmapper opens this without O_CLOEXEC, and go exec will not close files
  472. // that are not Close-on-exec, and lxc-start will die if it inherits any unexpected files,
  473. // so we add this badhack to make sure it closes itself
  474. setCloseOnExec("/dev/mapper/control")
  475. // Make sure the sparse images exist in <root>/devicemapper/data and
  476. // <root>/devicemapper/metadata
  477. createdLoopback := false
  478. // If the pool doesn't exist, create it
  479. if info.Exists == 0 {
  480. utils.Debugf("Pool doesn't exist. Creating it.")
  481. hasData := devices.hasImage("data")
  482. hasMetadata := devices.hasImage("metadata")
  483. if !doInit && !hasData {
  484. return errors.New("Loopback data file not found")
  485. }
  486. if !doInit && !hasMetadata {
  487. return errors.New("Loopback metadata file not found")
  488. }
  489. createdLoopback = !hasData || !hasMetadata
  490. data, err := devices.ensureImage("data", devices.dataLoopbackSize)
  491. if err != nil {
  492. utils.Debugf("Error device ensureImage (data): %s\n", err)
  493. return err
  494. }
  495. metadata, err := devices.ensureImage("metadata", devices.metaDataLoopbackSize)
  496. if err != nil {
  497. utils.Debugf("Error device ensureImage (metadata): %s\n", err)
  498. return err
  499. }
  500. dataFile, err := attachLoopDevice(data)
  501. if err != nil {
  502. utils.Debugf("\n--->Err: %s\n", err)
  503. return err
  504. }
  505. defer dataFile.Close()
  506. metadataFile, err := attachLoopDevice(metadata)
  507. if err != nil {
  508. utils.Debugf("\n--->Err: %s\n", err)
  509. return err
  510. }
  511. defer metadataFile.Close()
  512. if err := createPool(devices.getPoolName(), dataFile, metadataFile); err != nil {
  513. utils.Debugf("\n--->Err: %s\n", err)
  514. return err
  515. }
  516. }
  517. // If we didn't just create the data or metadata image, we need to
  518. // load the transaction id and migrate old metadata
  519. if !createdLoopback {
  520. if err = devices.initMetaData(); err != nil {
  521. utils.Debugf("\n--->Err: %s\n", err)
  522. return err
  523. }
  524. }
  525. // Setup the base image
  526. if doInit {
  527. if err := devices.setupBaseImage(); err != nil {
  528. utils.Debugf("Error device setupBaseImage: %s\n", err)
  529. return err
  530. }
  531. }
  532. return nil
  533. }
  534. func (devices *DeviceSet) AddDevice(hash, baseHash string) error {
  535. baseInfo, err := devices.lookupDevice(baseHash)
  536. if err != nil {
  537. return err
  538. }
  539. baseInfo.lock.Lock()
  540. defer baseInfo.lock.Unlock()
  541. devices.Lock()
  542. defer devices.Unlock()
  543. if info, _ := devices.lookupDevice(hash); info != nil {
  544. return fmt.Errorf("device %s already exists", hash)
  545. }
  546. deviceId := devices.nextDeviceId
  547. if err := createSnapDevice(devices.getPoolDevName(), &deviceId, baseInfo.Name(), baseInfo.DeviceId); err != nil {
  548. utils.Debugf("Error creating snap device: %s\n", err)
  549. return err
  550. }
  551. // Ids are 24bit, so wrap around
  552. devices.nextDeviceId = (deviceId + 1) & 0xffffff
  553. if _, err := devices.registerDevice(deviceId, hash, baseInfo.Size); err != nil {
  554. deleteDevice(devices.getPoolDevName(), deviceId)
  555. utils.Debugf("Error registering device: %s\n", err)
  556. return err
  557. }
  558. return nil
  559. }
  560. func (devices *DeviceSet) deleteDevice(info *DevInfo) error {
  561. // This is a workaround for the kernel not discarding block so
  562. // on the thin pool when we remove a thinp device, so we do it
  563. // manually
  564. if err := devices.activateDeviceIfNeeded(info); err == nil {
  565. if err := BlockDeviceDiscard(info.DevName()); err != nil {
  566. utils.Debugf("Error discarding block on device: %s (ignoring)\n", err)
  567. }
  568. }
  569. devinfo, _ := getInfo(info.Name())
  570. if devinfo != nil && devinfo.Exists != 0 {
  571. if err := devices.removeDeviceAndWait(info.Name()); err != nil {
  572. utils.Debugf("Error removing device: %s\n", err)
  573. return err
  574. }
  575. }
  576. if err := deleteDevice(devices.getPoolDevName(), info.DeviceId); err != nil {
  577. utils.Debugf("Error deleting device: %s\n", err)
  578. return err
  579. }
  580. devices.allocateTransactionId()
  581. devices.devicesLock.Lock()
  582. delete(devices.Devices, info.Hash)
  583. devices.devicesLock.Unlock()
  584. if err := devices.removeMetadata(info); err != nil {
  585. devices.devicesLock.Lock()
  586. devices.Devices[info.Hash] = info
  587. devices.devicesLock.Unlock()
  588. utils.Debugf("Error removing meta data: %s\n", err)
  589. return err
  590. }
  591. return nil
  592. }
  593. func (devices *DeviceSet) DeleteDevice(hash string) error {
  594. info, err := devices.lookupDevice(hash)
  595. if err != nil {
  596. return err
  597. }
  598. info.lock.Lock()
  599. defer info.lock.Unlock()
  600. devices.Lock()
  601. defer devices.Unlock()
  602. return devices.deleteDevice(info)
  603. }
  604. func (devices *DeviceSet) deactivatePool() error {
  605. utils.Debugf("[devmapper] deactivatePool()")
  606. defer utils.Debugf("[devmapper] deactivatePool END")
  607. devname := devices.getPoolDevName()
  608. devinfo, err := getInfo(devname)
  609. if err != nil {
  610. utils.Debugf("\n--->Err: %s\n", err)
  611. return err
  612. }
  613. if devinfo.Exists != 0 {
  614. return removeDevice(devname)
  615. }
  616. return nil
  617. }
  618. func (devices *DeviceSet) deactivateDevice(info *DevInfo) error {
  619. utils.Debugf("[devmapper] deactivateDevice(%s)", info.Hash)
  620. defer utils.Debugf("[devmapper] deactivateDevice END")
  621. // Wait for the unmount to be effective,
  622. // by watching the value of Info.OpenCount for the device
  623. if err := devices.waitClose(info); err != nil {
  624. utils.Errorf("Warning: error waiting for device %s to close: %s\n", info.Hash, err)
  625. }
  626. devinfo, err := getInfo(info.Name())
  627. if err != nil {
  628. utils.Debugf("\n--->Err: %s\n", err)
  629. return err
  630. }
  631. if devinfo.Exists != 0 {
  632. if err := devices.removeDeviceAndWait(info.Name()); err != nil {
  633. utils.Debugf("\n--->Err: %s\n", err)
  634. return err
  635. }
  636. }
  637. return nil
  638. }
  639. // Issues the underlying dm remove operation and then waits
  640. // for it to finish.
  641. func (devices *DeviceSet) removeDeviceAndWait(devname string) error {
  642. var err error
  643. for i := 0; i < 1000; i++ {
  644. err = removeDevice(devname)
  645. if err == nil {
  646. break
  647. }
  648. if err != ErrBusy {
  649. return err
  650. }
  651. // If we see EBUSY it may be a transient error,
  652. // sleep a bit a retry a few times.
  653. devices.Unlock()
  654. time.Sleep(10 * time.Millisecond)
  655. devices.Lock()
  656. }
  657. if err != nil {
  658. return err
  659. }
  660. if err := devices.waitRemove(devname); err != nil {
  661. return err
  662. }
  663. return nil
  664. }
  665. // waitRemove blocks until either:
  666. // a) the device registered at <device_set_prefix>-<hash> is removed,
  667. // or b) the 10 second timeout expires.
  668. func (devices *DeviceSet) waitRemove(devname string) error {
  669. utils.Debugf("[deviceset %s] waitRemove(%s)", devices.devicePrefix, devname)
  670. defer utils.Debugf("[deviceset %s] waitRemove(%s) END", devices.devicePrefix, devname)
  671. i := 0
  672. for ; i < 1000; i += 1 {
  673. devinfo, err := getInfo(devname)
  674. if err != nil {
  675. // If there is an error we assume the device doesn't exist.
  676. // The error might actually be something else, but we can't differentiate.
  677. return nil
  678. }
  679. if i%100 == 0 {
  680. utils.Debugf("Waiting for removal of %s: exists=%d", devname, devinfo.Exists)
  681. }
  682. if devinfo.Exists == 0 {
  683. break
  684. }
  685. devices.Unlock()
  686. time.Sleep(10 * time.Millisecond)
  687. devices.Lock()
  688. }
  689. if i == 1000 {
  690. return fmt.Errorf("Timeout while waiting for device %s to be removed", devname)
  691. }
  692. return nil
  693. }
  694. // waitClose blocks until either:
  695. // a) the device registered at <device_set_prefix>-<hash> is closed,
  696. // or b) the 10 second timeout expires.
  697. func (devices *DeviceSet) waitClose(info *DevInfo) error {
  698. i := 0
  699. for ; i < 1000; i += 1 {
  700. devinfo, err := getInfo(info.Name())
  701. if err != nil {
  702. return err
  703. }
  704. if i%100 == 0 {
  705. utils.Debugf("Waiting for unmount of %s: opencount=%d", info.Hash, devinfo.OpenCount)
  706. }
  707. if devinfo.OpenCount == 0 {
  708. break
  709. }
  710. devices.Unlock()
  711. time.Sleep(10 * time.Millisecond)
  712. devices.Lock()
  713. }
  714. if i == 1000 {
  715. return fmt.Errorf("Timeout while waiting for device %s to close", info.Hash)
  716. }
  717. return nil
  718. }
  719. func (devices *DeviceSet) Shutdown() error {
  720. utils.Debugf("[deviceset %s] shutdown()", devices.devicePrefix)
  721. utils.Debugf("[devmapper] Shutting down DeviceSet: %s", devices.root)
  722. defer utils.Debugf("[deviceset %s] shutdown END", devices.devicePrefix)
  723. var devs []*DevInfo
  724. devices.devicesLock.Lock()
  725. for _, info := range devices.Devices {
  726. devs = append(devs, info)
  727. }
  728. devices.devicesLock.Unlock()
  729. for _, info := range devs {
  730. info.lock.Lock()
  731. if info.mountCount > 0 {
  732. // We use MNT_DETACH here in case it is still busy in some running
  733. // container. This means it'll go away from the global scope directly,
  734. // and the device will be released when that container dies.
  735. if err := syscall.Unmount(info.mountPath, syscall.MNT_DETACH); err != nil {
  736. utils.Debugf("Shutdown unmounting %s, error: %s\n", info.mountPath, err)
  737. }
  738. devices.Lock()
  739. if err := devices.deactivateDevice(info); err != nil {
  740. utils.Debugf("Shutdown deactivate %s , error: %s\n", info.Hash, err)
  741. }
  742. devices.Unlock()
  743. }
  744. info.lock.Unlock()
  745. }
  746. info, _ := devices.lookupDevice("")
  747. if info != nil {
  748. info.lock.Lock()
  749. devices.Lock()
  750. if err := devices.deactivateDevice(info); err != nil {
  751. utils.Debugf("Shutdown deactivate base , error: %s\n", err)
  752. }
  753. devices.Unlock()
  754. info.lock.Unlock()
  755. }
  756. devices.Lock()
  757. if err := devices.deactivatePool(); err != nil {
  758. utils.Debugf("Shutdown deactivate pool , error: %s\n", err)
  759. }
  760. devices.Unlock()
  761. return nil
  762. }
  763. func (devices *DeviceSet) MountDevice(hash, path, mountLabel string) error {
  764. info, err := devices.lookupDevice(hash)
  765. if err != nil {
  766. return err
  767. }
  768. info.lock.Lock()
  769. defer info.lock.Unlock()
  770. devices.Lock()
  771. defer devices.Unlock()
  772. if info.mountCount > 0 {
  773. if path != info.mountPath {
  774. return fmt.Errorf("Trying to mount devmapper device in multple places (%s, %s)", info.mountPath, path)
  775. }
  776. info.mountCount++
  777. return nil
  778. }
  779. if err := devices.activateDeviceIfNeeded(info); err != nil {
  780. return fmt.Errorf("Error activating devmapper device for '%s': %s", hash, err)
  781. }
  782. var flags uintptr = syscall.MS_MGC_VAL
  783. fstype, err := ProbeFsType(info.DevName())
  784. if err != nil {
  785. return err
  786. }
  787. options := ""
  788. if fstype == "xfs" {
  789. // XFS needs nouuid or it can't mount filesystems with the same fs
  790. options = joinMountOptions(options, "nouuid")
  791. }
  792. options = joinMountOptions(options, devices.mountOptions)
  793. options = joinMountOptions(options, label.FormatMountLabel("", mountLabel))
  794. err = syscall.Mount(info.DevName(), path, fstype, flags, joinMountOptions("discard", options))
  795. if err != nil && err == syscall.EINVAL {
  796. err = syscall.Mount(info.DevName(), path, fstype, flags, options)
  797. }
  798. if err != nil {
  799. return fmt.Errorf("Error mounting '%s' on '%s': %s", info.DevName(), path, err)
  800. }
  801. info.mountCount = 1
  802. info.mountPath = path
  803. return nil
  804. }
  805. func (devices *DeviceSet) UnmountDevice(hash string) error {
  806. utils.Debugf("[devmapper] UnmountDevice(hash=%s)", hash)
  807. defer utils.Debugf("[devmapper] UnmountDevice END")
  808. info, err := devices.lookupDevice(hash)
  809. if err != nil {
  810. return err
  811. }
  812. info.lock.Lock()
  813. defer info.lock.Unlock()
  814. devices.Lock()
  815. defer devices.Unlock()
  816. if info.mountCount == 0 {
  817. return fmt.Errorf("UnmountDevice: device not-mounted id %s\n", hash)
  818. }
  819. info.mountCount--
  820. if info.mountCount > 0 {
  821. return nil
  822. }
  823. utils.Debugf("[devmapper] Unmount(%s)", info.mountPath)
  824. if err := syscall.Unmount(info.mountPath, 0); err != nil {
  825. utils.Debugf("\n--->Err: %s\n", err)
  826. return err
  827. }
  828. utils.Debugf("[devmapper] Unmount done")
  829. if err := devices.deactivateDevice(info); err != nil {
  830. return err
  831. }
  832. info.mountPath = ""
  833. return nil
  834. }
  835. func (devices *DeviceSet) HasDevice(hash string) bool {
  836. devices.Lock()
  837. defer devices.Unlock()
  838. info, _ := devices.lookupDevice(hash)
  839. return info != nil
  840. }
  841. func (devices *DeviceSet) HasActivatedDevice(hash string) bool {
  842. info, _ := devices.lookupDevice(hash)
  843. if info == nil {
  844. return false
  845. }
  846. info.lock.Lock()
  847. defer info.lock.Unlock()
  848. devices.Lock()
  849. defer devices.Unlock()
  850. devinfo, _ := getInfo(info.Name())
  851. return devinfo != nil && devinfo.Exists != 0
  852. }
  853. func (devices *DeviceSet) List() []string {
  854. devices.Lock()
  855. defer devices.Unlock()
  856. devices.devicesLock.Lock()
  857. ids := make([]string, len(devices.Devices))
  858. i := 0
  859. for k := range devices.Devices {
  860. ids[i] = k
  861. i++
  862. }
  863. devices.devicesLock.Unlock()
  864. return ids
  865. }
  866. func (devices *DeviceSet) deviceStatus(devName string) (sizeInSectors, mappedSectors, highestMappedSector uint64, err error) {
  867. var params string
  868. _, sizeInSectors, _, params, err = getStatus(devName)
  869. if err != nil {
  870. return
  871. }
  872. if _, err = fmt.Sscanf(params, "%d %d", &mappedSectors, &highestMappedSector); err == nil {
  873. return
  874. }
  875. return
  876. }
  877. func (devices *DeviceSet) GetDeviceStatus(hash string) (*DevStatus, error) {
  878. info, err := devices.lookupDevice(hash)
  879. if err != nil {
  880. return nil, err
  881. }
  882. info.lock.Lock()
  883. defer info.lock.Unlock()
  884. devices.Lock()
  885. defer devices.Unlock()
  886. status := &DevStatus{
  887. DeviceId: info.DeviceId,
  888. Size: info.Size,
  889. TransactionId: info.TransactionId,
  890. }
  891. if err := devices.activateDeviceIfNeeded(info); err != nil {
  892. return nil, fmt.Errorf("Error activating devmapper device for '%s': %s", hash, err)
  893. }
  894. if sizeInSectors, mappedSectors, highestMappedSector, err := devices.deviceStatus(info.DevName()); err != nil {
  895. return nil, err
  896. } else {
  897. status.SizeInSectors = sizeInSectors
  898. status.MappedSectors = mappedSectors
  899. status.HighestMappedSector = highestMappedSector
  900. }
  901. return status, nil
  902. }
  903. func (devices *DeviceSet) poolStatus() (totalSizeInSectors, transactionId, dataUsed, dataTotal, metadataUsed, metadataTotal uint64, err error) {
  904. var params string
  905. if _, totalSizeInSectors, _, params, err = getStatus(devices.getPoolName()); err == nil {
  906. _, err = fmt.Sscanf(params, "%d %d/%d %d/%d", &transactionId, &metadataUsed, &metadataTotal, &dataUsed, &dataTotal)
  907. }
  908. return
  909. }
  910. func (devices *DeviceSet) Status() *Status {
  911. devices.Lock()
  912. defer devices.Unlock()
  913. status := &Status{}
  914. status.PoolName = devices.getPoolName()
  915. status.DataLoopback = path.Join(devices.loopbackDir(), "data")
  916. status.MetadataLoopback = path.Join(devices.loopbackDir(), "metadata")
  917. totalSizeInSectors, _, dataUsed, dataTotal, metadataUsed, metadataTotal, err := devices.poolStatus()
  918. if err == nil {
  919. // Convert from blocks to bytes
  920. blockSizeInSectors := totalSizeInSectors / dataTotal
  921. status.Data.Used = dataUsed * blockSizeInSectors * 512
  922. status.Data.Total = dataTotal * blockSizeInSectors * 512
  923. // metadata blocks are always 4k
  924. status.Metadata.Used = metadataUsed * 4096
  925. status.Metadata.Total = metadataTotal * 4096
  926. status.SectorSize = blockSizeInSectors * 512
  927. }
  928. return status
  929. }
  930. func NewDeviceSet(root string, doInit bool, options []string) (*DeviceSet, error) {
  931. SetDevDir("/dev")
  932. devices := &DeviceSet{
  933. root: root,
  934. MetaData: MetaData{Devices: make(map[string]*DevInfo)},
  935. dataLoopbackSize: DefaultDataLoopbackSize,
  936. metaDataLoopbackSize: DefaultMetaDataLoopbackSize,
  937. baseFsSize: DefaultBaseFsSize,
  938. filesystem: "ext4",
  939. }
  940. for _, option := range options {
  941. key, val, err := utils.ParseKeyValueOpt(option)
  942. if err != nil {
  943. return nil, err
  944. }
  945. key = strings.ToLower(key)
  946. switch key {
  947. case "dm.basesize":
  948. size, err := units.FromHumanSize(val)
  949. if err != nil {
  950. return nil, err
  951. }
  952. devices.baseFsSize = uint64(size)
  953. case "dm.loopdatasize":
  954. size, err := units.FromHumanSize(val)
  955. if err != nil {
  956. return nil, err
  957. }
  958. devices.dataLoopbackSize = size
  959. case "dm.loopmetadatasize":
  960. size, err := units.FromHumanSize(val)
  961. if err != nil {
  962. return nil, err
  963. }
  964. devices.metaDataLoopbackSize = size
  965. case "dm.fs":
  966. if val != "ext4" && val != "xfs" {
  967. return nil, fmt.Errorf("Unsupported filesystem %s\n", val)
  968. }
  969. devices.filesystem = val
  970. case "dm.mkfsarg":
  971. devices.mkfsArgs = append(devices.mkfsArgs, val)
  972. case "dm.mountopt":
  973. devices.mountOptions = joinMountOptions(devices.mountOptions, val)
  974. default:
  975. return nil, fmt.Errorf("Unknown option %s\n", key)
  976. }
  977. }
  978. if err := devices.initDevmapper(doInit); err != nil {
  979. return nil, err
  980. }
  981. return devices, nil
  982. }