deviceset.go 57 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070
  1. // +build linux
  2. package devmapper
  3. import (
  4. "encoding/json"
  5. "errors"
  6. "fmt"
  7. "io"
  8. "io/ioutil"
  9. "os"
  10. "os/exec"
  11. "path"
  12. "path/filepath"
  13. "strconv"
  14. "strings"
  15. "sync"
  16. "syscall"
  17. "time"
  18. "github.com/Sirupsen/logrus"
  19. "github.com/docker/docker/daemon/graphdriver"
  20. "github.com/docker/docker/pkg/devicemapper"
  21. "github.com/docker/docker/pkg/parsers"
  22. "github.com/docker/docker/pkg/units"
  23. "github.com/opencontainers/runc/libcontainer/label"
  24. )
  25. var (
  26. defaultDataLoopbackSize int64 = 100 * 1024 * 1024 * 1024
  27. defaultMetaDataLoopbackSize int64 = 2 * 1024 * 1024 * 1024
  28. defaultBaseFsSize uint64 = 100 * 1024 * 1024 * 1024
  29. defaultThinpBlockSize uint32 = 128 // 64K = 128 512b sectors
  30. defaultUdevSyncOverride = false
  31. maxDeviceID = 0xffffff // 24 bit, pool limit
  32. deviceIDMapSz = (maxDeviceID + 1) / 8
  33. // We retry device removal so many a times that even error messages
  34. // will fill up console during normal operation. So only log Fatal
  35. // messages by default.
  36. logLevel = devicemapper.LogLevelFatal
  37. driverDeferredRemovalSupport = false
  38. enableDeferredRemoval = false
  39. )
  40. const deviceSetMetaFile string = "deviceset-metadata"
  41. const transactionMetaFile string = "transaction-metadata"
  42. type transaction struct {
  43. OpenTransactionID uint64 `json:"open_transaction_id"`
  44. DeviceIDHash string `json:"device_hash"`
  45. DeviceID int `json:"device_id"`
  46. }
  47. type devInfo struct {
  48. Hash string `json:"-"`
  49. DeviceID int `json:"device_id"`
  50. Size uint64 `json:"size"`
  51. TransactionID uint64 `json:"transaction_id"`
  52. Initialized bool `json:"initialized"`
  53. devices *DeviceSet
  54. mountCount int
  55. mountPath string
  56. // The global DeviceSet lock guarantees that we serialize all
  57. // the calls to libdevmapper (which is not threadsafe), but we
  58. // sometimes release that lock while sleeping. In that case
  59. // this per-device lock is still held, protecting against
  60. // other accesses to the device that we're doing the wait on.
  61. //
  62. // WARNING: In order to avoid AB-BA deadlocks when releasing
  63. // the global lock while holding the per-device locks all
  64. // device locks must be acquired *before* the device lock, and
  65. // multiple device locks should be acquired parent before child.
  66. lock sync.Mutex
  67. }
  68. type metaData struct {
  69. Devices map[string]*devInfo `json:"Devices"`
  70. devicesLock sync.Mutex // Protects all read/writes to Devices map
  71. }
  72. // DeviceSet holds information about list of devices
  73. type DeviceSet struct {
  74. metaData `json:"-"`
  75. sync.Mutex `json:"-"` // Protects Devices map and serializes calls into libdevmapper
  76. root string
  77. devicePrefix string
  78. TransactionID uint64 `json:"-"`
  79. NextDeviceID int `json:"next_device_id"`
  80. deviceIDMap []byte
  81. // Options
  82. dataLoopbackSize int64
  83. metaDataLoopbackSize int64
  84. baseFsSize uint64
  85. filesystem string
  86. mountOptions string
  87. mkfsArgs []string
  88. dataDevice string // block or loop dev
  89. dataLoopFile string // loopback file, if used
  90. metadataDevice string // block or loop dev
  91. metadataLoopFile string // loopback file, if used
  92. doBlkDiscard bool
  93. thinpBlockSize uint32
  94. thinPoolDevice string
  95. transaction `json:"-"`
  96. overrideUdevSyncCheck bool
  97. deferredRemove bool // use deferred removal
  98. BaseDeviceUUID string //save UUID of base device
  99. }
  100. // DiskUsage contains information about disk usage and is used when reporting Status of a device.
  101. type DiskUsage struct {
  102. // Used bytes on the disk.
  103. Used uint64
  104. // Total bytes on the disk.
  105. Total uint64
  106. // Available bytes on the disk.
  107. Available uint64
  108. }
  109. // Status returns the information about the device.
  110. type Status struct {
  111. // PoolName is the name of the data pool.
  112. PoolName string
  113. // DataFile is the actual block device for data.
  114. DataFile string
  115. // DataLoopback loopback file, if used.
  116. DataLoopback string
  117. // MetadataFile is the actual block device for metadata.
  118. MetadataFile string
  119. // MetadataLoopback is the loopback file, if used.
  120. MetadataLoopback string
  121. // Data is the disk used for data.
  122. Data DiskUsage
  123. // Metadata is the disk used for meta data.
  124. Metadata DiskUsage
  125. // SectorSize size of the vector.
  126. SectorSize uint64
  127. // UdevSyncSupported is true if sync is supported.
  128. UdevSyncSupported bool
  129. // DeferredRemoveEnabled is true then the device is not unmounted.
  130. DeferredRemoveEnabled bool
  131. }
  132. // Structure used to export image/container metadata in docker inspect.
  133. type deviceMetadata struct {
  134. deviceID int
  135. deviceSize uint64 // size in bytes
  136. deviceName string // Device name as used during activation
  137. }
  138. // DevStatus returns information about device mounted containing its id, size and sector information.
  139. type DevStatus struct {
  140. // DeviceID is the id of the device.
  141. DeviceID int
  142. // Size is the size of the filesystem.
  143. Size uint64
  144. // TransactionID is a unique integer per device set used to identify an operation on the file system, this number is incremental.
  145. TransactionID uint64
  146. // SizeInSectors indicates the size of the sectors allocated.
  147. SizeInSectors uint64
  148. // MappedSectors indicates number of mapped sectors.
  149. MappedSectors uint64
  150. // HighestMappedSector is the pointer to the highest mapped sector.
  151. HighestMappedSector uint64
  152. }
  153. func getDevName(name string) string {
  154. return "/dev/mapper/" + name
  155. }
  156. func (info *devInfo) Name() string {
  157. hash := info.Hash
  158. if hash == "" {
  159. hash = "base"
  160. }
  161. return fmt.Sprintf("%s-%s", info.devices.devicePrefix, hash)
  162. }
  163. func (info *devInfo) DevName() string {
  164. return getDevName(info.Name())
  165. }
  166. func (devices *DeviceSet) loopbackDir() string {
  167. return path.Join(devices.root, "devicemapper")
  168. }
  169. func (devices *DeviceSet) metadataDir() string {
  170. return path.Join(devices.root, "metadata")
  171. }
  172. func (devices *DeviceSet) metadataFile(info *devInfo) string {
  173. file := info.Hash
  174. if file == "" {
  175. file = "base"
  176. }
  177. return path.Join(devices.metadataDir(), file)
  178. }
  179. func (devices *DeviceSet) transactionMetaFile() string {
  180. return path.Join(devices.metadataDir(), transactionMetaFile)
  181. }
  182. func (devices *DeviceSet) deviceSetMetaFile() string {
  183. return path.Join(devices.metadataDir(), deviceSetMetaFile)
  184. }
  185. func (devices *DeviceSet) oldMetadataFile() string {
  186. return path.Join(devices.loopbackDir(), "json")
  187. }
  188. func (devices *DeviceSet) getPoolName() string {
  189. if devices.thinPoolDevice == "" {
  190. return devices.devicePrefix + "-pool"
  191. }
  192. return devices.thinPoolDevice
  193. }
  194. func (devices *DeviceSet) getPoolDevName() string {
  195. return getDevName(devices.getPoolName())
  196. }
  197. func (devices *DeviceSet) hasImage(name string) bool {
  198. dirname := devices.loopbackDir()
  199. filename := path.Join(dirname, name)
  200. _, err := os.Stat(filename)
  201. return err == nil
  202. }
  203. // ensureImage creates a sparse file of <size> bytes at the path
  204. // <root>/devicemapper/<name>.
  205. // If the file already exists and new size is larger than its current size, it grows to the new size.
  206. // Either way it returns the full path.
  207. func (devices *DeviceSet) ensureImage(name string, size int64) (string, error) {
  208. dirname := devices.loopbackDir()
  209. filename := path.Join(dirname, name)
  210. if err := os.MkdirAll(dirname, 0700); err != nil {
  211. return "", err
  212. }
  213. if fi, err := os.Stat(filename); err != nil {
  214. if !os.IsNotExist(err) {
  215. return "", err
  216. }
  217. logrus.Debugf("Creating loopback file %s for device-manage use", filename)
  218. file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0600)
  219. if err != nil {
  220. return "", err
  221. }
  222. defer file.Close()
  223. if err := file.Truncate(size); err != nil {
  224. return "", err
  225. }
  226. } else {
  227. if fi.Size() < size {
  228. file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0600)
  229. if err != nil {
  230. return "", err
  231. }
  232. defer file.Close()
  233. if err := file.Truncate(size); err != nil {
  234. return "", fmt.Errorf("Unable to grow loopback file %s: %v", filename, err)
  235. }
  236. } else if fi.Size() > size {
  237. logrus.Warnf("Can't shrink loopback file %s", filename)
  238. }
  239. }
  240. return filename, nil
  241. }
  242. func (devices *DeviceSet) allocateTransactionID() uint64 {
  243. devices.OpenTransactionID = devices.TransactionID + 1
  244. return devices.OpenTransactionID
  245. }
  246. func (devices *DeviceSet) updatePoolTransactionID() error {
  247. if err := devicemapper.SetTransactionID(devices.getPoolDevName(), devices.TransactionID, devices.OpenTransactionID); err != nil {
  248. return fmt.Errorf("Error setting devmapper transaction ID: %s", err)
  249. }
  250. devices.TransactionID = devices.OpenTransactionID
  251. return nil
  252. }
  253. func (devices *DeviceSet) removeMetadata(info *devInfo) error {
  254. if err := os.RemoveAll(devices.metadataFile(info)); err != nil {
  255. return fmt.Errorf("Error removing metadata file %s: %s", devices.metadataFile(info), err)
  256. }
  257. return nil
  258. }
  259. // Given json data and file path, write it to disk
  260. func (devices *DeviceSet) writeMetaFile(jsonData []byte, filePath string) error {
  261. tmpFile, err := ioutil.TempFile(devices.metadataDir(), ".tmp")
  262. if err != nil {
  263. return fmt.Errorf("Error creating metadata file: %s", err)
  264. }
  265. n, err := tmpFile.Write(jsonData)
  266. if err != nil {
  267. return fmt.Errorf("Error writing metadata to %s: %s", tmpFile.Name(), err)
  268. }
  269. if n < len(jsonData) {
  270. return io.ErrShortWrite
  271. }
  272. if err := tmpFile.Sync(); err != nil {
  273. return fmt.Errorf("Error syncing metadata file %s: %s", tmpFile.Name(), err)
  274. }
  275. if err := tmpFile.Close(); err != nil {
  276. return fmt.Errorf("Error closing metadata file %s: %s", tmpFile.Name(), err)
  277. }
  278. if err := os.Rename(tmpFile.Name(), filePath); err != nil {
  279. return fmt.Errorf("Error committing metadata file %s: %s", tmpFile.Name(), err)
  280. }
  281. return nil
  282. }
  283. func (devices *DeviceSet) saveMetadata(info *devInfo) error {
  284. jsonData, err := json.Marshal(info)
  285. if err != nil {
  286. return fmt.Errorf("Error encoding metadata to json: %s", err)
  287. }
  288. if err := devices.writeMetaFile(jsonData, devices.metadataFile(info)); err != nil {
  289. return err
  290. }
  291. return nil
  292. }
  293. func (devices *DeviceSet) markDeviceIDUsed(deviceID int) {
  294. var mask byte
  295. i := deviceID % 8
  296. mask = 1 << uint(i)
  297. devices.deviceIDMap[deviceID/8] = devices.deviceIDMap[deviceID/8] | mask
  298. }
  299. func (devices *DeviceSet) markDeviceIDFree(deviceID int) {
  300. var mask byte
  301. i := deviceID % 8
  302. mask = ^(1 << uint(i))
  303. devices.deviceIDMap[deviceID/8] = devices.deviceIDMap[deviceID/8] & mask
  304. }
  305. func (devices *DeviceSet) isDeviceIDFree(deviceID int) bool {
  306. var mask byte
  307. i := deviceID % 8
  308. mask = (1 << uint(i))
  309. if (devices.deviceIDMap[deviceID/8] & mask) != 0 {
  310. return false
  311. }
  312. return true
  313. }
  314. func (devices *DeviceSet) lookupDevice(hash string) (*devInfo, error) {
  315. devices.devicesLock.Lock()
  316. defer devices.devicesLock.Unlock()
  317. info := devices.Devices[hash]
  318. if info == nil {
  319. info = devices.loadMetadata(hash)
  320. if info == nil {
  321. return nil, fmt.Errorf("Unknown device %s", hash)
  322. }
  323. devices.Devices[hash] = info
  324. }
  325. return info, nil
  326. }
  327. func (devices *DeviceSet) deviceFileWalkFunction(path string, finfo os.FileInfo) error {
  328. // Skip some of the meta files which are not device files.
  329. if strings.HasSuffix(finfo.Name(), ".migrated") {
  330. logrus.Debugf("Skipping file %s", path)
  331. return nil
  332. }
  333. if strings.HasPrefix(finfo.Name(), ".") {
  334. logrus.Debugf("Skipping file %s", path)
  335. return nil
  336. }
  337. if finfo.Name() == deviceSetMetaFile {
  338. logrus.Debugf("Skipping file %s", path)
  339. return nil
  340. }
  341. if finfo.Name() == transactionMetaFile {
  342. logrus.Debugf("Skipping file %s", path)
  343. return nil
  344. }
  345. logrus.Debugf("Loading data for file %s", path)
  346. hash := finfo.Name()
  347. if hash == "base" {
  348. hash = ""
  349. }
  350. dinfo := devices.loadMetadata(hash)
  351. if dinfo == nil {
  352. return fmt.Errorf("Error loading device metadata file %s", hash)
  353. }
  354. devices.Lock()
  355. devices.markDeviceIDUsed(dinfo.DeviceID)
  356. devices.Unlock()
  357. logrus.Debugf("Added deviceID=%d to DeviceIDMap", dinfo.DeviceID)
  358. return nil
  359. }
  360. func (devices *DeviceSet) constructDeviceIDMap() error {
  361. logrus.Debugf("[deviceset] constructDeviceIDMap()")
  362. defer logrus.Debugf("[deviceset] constructDeviceIDMap() END")
  363. var scan = func(path string, info os.FileInfo, err error) error {
  364. if err != nil {
  365. logrus.Debugf("Can't walk the file %s", path)
  366. return nil
  367. }
  368. // Skip any directories
  369. if info.IsDir() {
  370. return nil
  371. }
  372. return devices.deviceFileWalkFunction(path, info)
  373. }
  374. return filepath.Walk(devices.metadataDir(), scan)
  375. }
  376. func (devices *DeviceSet) unregisterDevice(id int, hash string) error {
  377. logrus.Debugf("unregisterDevice(%v, %v)", id, hash)
  378. info := &devInfo{
  379. Hash: hash,
  380. DeviceID: id,
  381. }
  382. devices.devicesLock.Lock()
  383. delete(devices.Devices, hash)
  384. devices.devicesLock.Unlock()
  385. if err := devices.removeMetadata(info); err != nil {
  386. logrus.Debugf("Error removing metadata: %s", err)
  387. return err
  388. }
  389. return nil
  390. }
  391. func (devices *DeviceSet) registerDevice(id int, hash string, size uint64, transactionID uint64) (*devInfo, error) {
  392. logrus.Debugf("registerDevice(%v, %v)", id, hash)
  393. info := &devInfo{
  394. Hash: hash,
  395. DeviceID: id,
  396. Size: size,
  397. TransactionID: transactionID,
  398. Initialized: false,
  399. devices: devices,
  400. }
  401. devices.devicesLock.Lock()
  402. devices.Devices[hash] = info
  403. devices.devicesLock.Unlock()
  404. if err := devices.saveMetadata(info); err != nil {
  405. // Try to remove unused device
  406. devices.devicesLock.Lock()
  407. delete(devices.Devices, hash)
  408. devices.devicesLock.Unlock()
  409. return nil, err
  410. }
  411. return info, nil
  412. }
  413. func (devices *DeviceSet) activateDeviceIfNeeded(info *devInfo) error {
  414. logrus.Debugf("activateDeviceIfNeeded(%v)", info.Hash)
  415. // Make sure deferred removal on device is canceled, if one was
  416. // scheduled.
  417. if err := devices.cancelDeferredRemoval(info); err != nil {
  418. return fmt.Errorf("Deivce Deferred Removal Cancellation Failed: %s", err)
  419. }
  420. if devinfo, _ := devicemapper.GetInfo(info.Name()); devinfo != nil && devinfo.Exists != 0 {
  421. return nil
  422. }
  423. return devicemapper.ActivateDevice(devices.getPoolDevName(), info.Name(), info.DeviceID, info.Size)
  424. }
  425. func (devices *DeviceSet) createFilesystem(info *devInfo) error {
  426. devname := info.DevName()
  427. args := []string{}
  428. for _, arg := range devices.mkfsArgs {
  429. args = append(args, arg)
  430. }
  431. args = append(args, devname)
  432. var err error
  433. switch devices.filesystem {
  434. case "xfs":
  435. err = exec.Command("mkfs.xfs", args...).Run()
  436. case "ext4":
  437. err = exec.Command("mkfs.ext4", append([]string{"-E", "nodiscard,lazy_itable_init=0,lazy_journal_init=0"}, args...)...).Run()
  438. if err != nil {
  439. err = exec.Command("mkfs.ext4", append([]string{"-E", "nodiscard,lazy_itable_init=0"}, args...)...).Run()
  440. }
  441. if err != nil {
  442. return err
  443. }
  444. err = exec.Command("tune2fs", append([]string{"-c", "-1", "-i", "0"}, devname)...).Run()
  445. default:
  446. err = fmt.Errorf("Unsupported filesystem type %s", devices.filesystem)
  447. }
  448. if err != nil {
  449. return err
  450. }
  451. return nil
  452. }
  453. func (devices *DeviceSet) migrateOldMetaData() error {
  454. // Migrate old metadata file
  455. jsonData, err := ioutil.ReadFile(devices.oldMetadataFile())
  456. if err != nil && !os.IsNotExist(err) {
  457. return err
  458. }
  459. if jsonData != nil {
  460. m := metaData{Devices: make(map[string]*devInfo)}
  461. if err := json.Unmarshal(jsonData, &m); err != nil {
  462. return err
  463. }
  464. for hash, info := range m.Devices {
  465. info.Hash = hash
  466. devices.saveMetadata(info)
  467. }
  468. if err := os.Rename(devices.oldMetadataFile(), devices.oldMetadataFile()+".migrated"); err != nil {
  469. return err
  470. }
  471. }
  472. return nil
  473. }
  474. func (devices *DeviceSet) initMetaData() error {
  475. if err := devices.migrateOldMetaData(); err != nil {
  476. return err
  477. }
  478. _, transactionID, _, _, _, _, err := devices.poolStatus()
  479. if err != nil {
  480. return err
  481. }
  482. devices.TransactionID = transactionID
  483. if err := devices.constructDeviceIDMap(); err != nil {
  484. return err
  485. }
  486. if err := devices.processPendingTransaction(); err != nil {
  487. return err
  488. }
  489. return nil
  490. }
  491. func (devices *DeviceSet) incNextDeviceID() {
  492. // IDs are 24bit, so wrap around
  493. devices.NextDeviceID = (devices.NextDeviceID + 1) & maxDeviceID
  494. }
  495. func (devices *DeviceSet) getNextFreeDeviceID() (int, error) {
  496. devices.incNextDeviceID()
  497. for i := 0; i <= maxDeviceID; i++ {
  498. if devices.isDeviceIDFree(devices.NextDeviceID) {
  499. devices.markDeviceIDUsed(devices.NextDeviceID)
  500. return devices.NextDeviceID, nil
  501. }
  502. devices.incNextDeviceID()
  503. }
  504. return 0, fmt.Errorf("Unable to find a free device ID")
  505. }
  506. func (devices *DeviceSet) createRegisterDevice(hash string) (*devInfo, error) {
  507. deviceID, err := devices.getNextFreeDeviceID()
  508. if err != nil {
  509. return nil, err
  510. }
  511. if err := devices.openTransaction(hash, deviceID); err != nil {
  512. logrus.Debugf("Error opening transaction hash = %s deviceID = %d", hash, deviceID)
  513. devices.markDeviceIDFree(deviceID)
  514. return nil, err
  515. }
  516. for {
  517. if err := devicemapper.CreateDevice(devices.getPoolDevName(), deviceID); err != nil {
  518. if devicemapper.DeviceIDExists(err) {
  519. // Device ID already exists. This should not
  520. // happen. Now we have a mechianism to find
  521. // a free device ID. So something is not right.
  522. // Give a warning and continue.
  523. logrus.Errorf("Device ID %d exists in pool but it is supposed to be unused", deviceID)
  524. deviceID, err = devices.getNextFreeDeviceID()
  525. if err != nil {
  526. return nil, err
  527. }
  528. // Save new device id into transaction
  529. devices.refreshTransaction(deviceID)
  530. continue
  531. }
  532. logrus.Debugf("Error creating device: %s", err)
  533. devices.markDeviceIDFree(deviceID)
  534. return nil, err
  535. }
  536. break
  537. }
  538. logrus.Debugf("Registering device (id %v) with FS size %v", deviceID, devices.baseFsSize)
  539. info, err := devices.registerDevice(deviceID, hash, devices.baseFsSize, devices.OpenTransactionID)
  540. if err != nil {
  541. _ = devicemapper.DeleteDevice(devices.getPoolDevName(), deviceID)
  542. devices.markDeviceIDFree(deviceID)
  543. return nil, err
  544. }
  545. if err := devices.closeTransaction(); err != nil {
  546. devices.unregisterDevice(deviceID, hash)
  547. devicemapper.DeleteDevice(devices.getPoolDevName(), deviceID)
  548. devices.markDeviceIDFree(deviceID)
  549. return nil, err
  550. }
  551. return info, nil
  552. }
  553. func (devices *DeviceSet) createRegisterSnapDevice(hash string, baseInfo *devInfo) error {
  554. deviceID, err := devices.getNextFreeDeviceID()
  555. if err != nil {
  556. return err
  557. }
  558. if err := devices.openTransaction(hash, deviceID); err != nil {
  559. logrus.Debugf("Error opening transaction hash = %s deviceID = %d", hash, deviceID)
  560. devices.markDeviceIDFree(deviceID)
  561. return err
  562. }
  563. for {
  564. if err := devicemapper.CreateSnapDevice(devices.getPoolDevName(), deviceID, baseInfo.Name(), baseInfo.DeviceID); err != nil {
  565. if devicemapper.DeviceIDExists(err) {
  566. // Device ID already exists. This should not
  567. // happen. Now we have a mechianism to find
  568. // a free device ID. So something is not right.
  569. // Give a warning and continue.
  570. logrus.Errorf("Device ID %d exists in pool but it is supposed to be unused", deviceID)
  571. deviceID, err = devices.getNextFreeDeviceID()
  572. if err != nil {
  573. return err
  574. }
  575. // Save new device id into transaction
  576. devices.refreshTransaction(deviceID)
  577. continue
  578. }
  579. logrus.Debugf("Error creating snap device: %s", err)
  580. devices.markDeviceIDFree(deviceID)
  581. return err
  582. }
  583. break
  584. }
  585. if _, err := devices.registerDevice(deviceID, hash, baseInfo.Size, devices.OpenTransactionID); err != nil {
  586. devicemapper.DeleteDevice(devices.getPoolDevName(), deviceID)
  587. devices.markDeviceIDFree(deviceID)
  588. logrus.Debugf("Error registering device: %s", err)
  589. return err
  590. }
  591. if err := devices.closeTransaction(); err != nil {
  592. devices.unregisterDevice(deviceID, hash)
  593. devicemapper.DeleteDevice(devices.getPoolDevName(), deviceID)
  594. devices.markDeviceIDFree(deviceID)
  595. return err
  596. }
  597. return nil
  598. }
  599. func (devices *DeviceSet) loadMetadata(hash string) *devInfo {
  600. info := &devInfo{Hash: hash, devices: devices}
  601. jsonData, err := ioutil.ReadFile(devices.metadataFile(info))
  602. if err != nil {
  603. return nil
  604. }
  605. if err := json.Unmarshal(jsonData, &info); err != nil {
  606. return nil
  607. }
  608. if info.DeviceID > maxDeviceID {
  609. logrus.Errorf("Ignoring Invalid DeviceId=%d", info.DeviceID)
  610. return nil
  611. }
  612. return info
  613. }
  614. func getDeviceUUID(device string) (string, error) {
  615. out, err := exec.Command("blkid", "-s", "UUID", "-o", "value", device).Output()
  616. if err != nil {
  617. logrus.Debugf("Failed to find uuid for device %s:%v", device, err)
  618. return "", err
  619. }
  620. uuid := strings.TrimSuffix(string(out), "\n")
  621. uuid = strings.TrimSpace(uuid)
  622. logrus.Debugf("UUID for device: %s is:%s", device, uuid)
  623. return uuid, nil
  624. }
  625. func (devices *DeviceSet) verifyBaseDeviceUUID(baseInfo *devInfo) error {
  626. devices.Lock()
  627. defer devices.Unlock()
  628. if err := devices.activateDeviceIfNeeded(baseInfo); err != nil {
  629. return err
  630. }
  631. defer devices.deactivateDevice(baseInfo)
  632. uuid, err := getDeviceUUID(baseInfo.DevName())
  633. if err != nil {
  634. return err
  635. }
  636. if devices.BaseDeviceUUID != uuid {
  637. return fmt.Errorf("Current Base Device UUID:%s does not match with stored UUID:%s", uuid, devices.BaseDeviceUUID)
  638. }
  639. return nil
  640. }
  641. func (devices *DeviceSet) saveBaseDeviceUUID(baseInfo *devInfo) error {
  642. devices.Lock()
  643. defer devices.Unlock()
  644. if err := devices.activateDeviceIfNeeded(baseInfo); err != nil {
  645. return err
  646. }
  647. defer devices.deactivateDevice(baseInfo)
  648. uuid, err := getDeviceUUID(baseInfo.DevName())
  649. if err != nil {
  650. return err
  651. }
  652. devices.BaseDeviceUUID = uuid
  653. devices.saveDeviceSetMetaData()
  654. return nil
  655. }
  656. func (devices *DeviceSet) setupBaseImage() error {
  657. oldInfo, _ := devices.lookupDevice("")
  658. if oldInfo != nil && oldInfo.Initialized {
  659. // If BaseDeviceUUID is nil (upgrade case), save it and
  660. // return success.
  661. if devices.BaseDeviceUUID == "" {
  662. if err := devices.saveBaseDeviceUUID(oldInfo); err != nil {
  663. return fmt.Errorf("Could not query and save base device UUID:%v", err)
  664. }
  665. return nil
  666. }
  667. if err := devices.verifyBaseDeviceUUID(oldInfo); err != nil {
  668. return fmt.Errorf("Base Device UUID verification failed. Possibly using a different thin pool than last invocation:%v", err)
  669. }
  670. return nil
  671. }
  672. if oldInfo != nil && !oldInfo.Initialized {
  673. logrus.Debugf("Removing uninitialized base image")
  674. if err := devices.DeleteDevice(""); err != nil {
  675. return err
  676. }
  677. }
  678. if devices.thinPoolDevice != "" && oldInfo == nil {
  679. _, transactionID, dataUsed, _, _, _, err := devices.poolStatus()
  680. if err != nil {
  681. return err
  682. }
  683. if dataUsed != 0 {
  684. return fmt.Errorf("Unable to take ownership of thin-pool (%s) that already has used data blocks",
  685. devices.thinPoolDevice)
  686. }
  687. if transactionID != 0 {
  688. return fmt.Errorf("Unable to take ownership of thin-pool (%s) with non-zero transaction ID",
  689. devices.thinPoolDevice)
  690. }
  691. }
  692. logrus.Debugf("Initializing base device-mapper thin volume")
  693. // Create initial device
  694. info, err := devices.createRegisterDevice("")
  695. if err != nil {
  696. return err
  697. }
  698. logrus.Debugf("Creating filesystem on base device-mapper thin volume")
  699. if err := devices.activateDeviceIfNeeded(info); err != nil {
  700. return err
  701. }
  702. if err := devices.createFilesystem(info); err != nil {
  703. return err
  704. }
  705. info.Initialized = true
  706. if err := devices.saveMetadata(info); err != nil {
  707. info.Initialized = false
  708. return err
  709. }
  710. if err := devices.saveBaseDeviceUUID(info); err != nil {
  711. return fmt.Errorf("Could not query and save base device UUID:%v", err)
  712. }
  713. return nil
  714. }
  715. func setCloseOnExec(name string) {
  716. if fileInfos, _ := ioutil.ReadDir("/proc/self/fd"); fileInfos != nil {
  717. for _, i := range fileInfos {
  718. link, _ := os.Readlink(filepath.Join("/proc/self/fd", i.Name()))
  719. if link == name {
  720. fd, err := strconv.Atoi(i.Name())
  721. if err == nil {
  722. syscall.CloseOnExec(fd)
  723. }
  724. }
  725. }
  726. }
  727. }
  728. // DMLog implements logging using DevMapperLogger interface.
  729. func (devices *DeviceSet) DMLog(level int, file string, line int, dmError int, message string) {
  730. // By default libdm sends us all the messages including debug ones.
  731. // We need to filter out messages here and figure out which one
  732. // should be printed.
  733. if level > logLevel {
  734. return
  735. }
  736. // FIXME(vbatts) push this back into ./pkg/devicemapper/
  737. if level <= devicemapper.LogLevelErr {
  738. logrus.Errorf("libdevmapper(%d): %s:%d (%d) %s", level, file, line, dmError, message)
  739. } else if level <= devicemapper.LogLevelInfo {
  740. logrus.Infof("libdevmapper(%d): %s:%d (%d) %s", level, file, line, dmError, message)
  741. } else {
  742. // FIXME(vbatts) push this back into ./pkg/devicemapper/
  743. logrus.Debugf("libdevmapper(%d): %s:%d (%d) %s", level, file, line, dmError, message)
  744. }
  745. }
  746. func major(device uint64) uint64 {
  747. return (device >> 8) & 0xfff
  748. }
  749. func minor(device uint64) uint64 {
  750. return (device & 0xff) | ((device >> 12) & 0xfff00)
  751. }
  752. // ResizePool increases the size of the pool.
  753. func (devices *DeviceSet) ResizePool(size int64) error {
  754. dirname := devices.loopbackDir()
  755. datafilename := path.Join(dirname, "data")
  756. if len(devices.dataDevice) > 0 {
  757. datafilename = devices.dataDevice
  758. }
  759. metadatafilename := path.Join(dirname, "metadata")
  760. if len(devices.metadataDevice) > 0 {
  761. metadatafilename = devices.metadataDevice
  762. }
  763. datafile, err := os.OpenFile(datafilename, os.O_RDWR, 0)
  764. if datafile == nil {
  765. return err
  766. }
  767. defer datafile.Close()
  768. fi, err := datafile.Stat()
  769. if fi == nil {
  770. return err
  771. }
  772. if fi.Size() > size {
  773. return fmt.Errorf("Can't shrink file")
  774. }
  775. dataloopback := devicemapper.FindLoopDeviceFor(datafile)
  776. if dataloopback == nil {
  777. return fmt.Errorf("Unable to find loopback mount for: %s", datafilename)
  778. }
  779. defer dataloopback.Close()
  780. metadatafile, err := os.OpenFile(metadatafilename, os.O_RDWR, 0)
  781. if metadatafile == nil {
  782. return err
  783. }
  784. defer metadatafile.Close()
  785. metadataloopback := devicemapper.FindLoopDeviceFor(metadatafile)
  786. if metadataloopback == nil {
  787. return fmt.Errorf("Unable to find loopback mount for: %s", metadatafilename)
  788. }
  789. defer metadataloopback.Close()
  790. // Grow loopback file
  791. if err := datafile.Truncate(size); err != nil {
  792. return fmt.Errorf("Unable to grow loopback file: %s", err)
  793. }
  794. // Reload size for loopback device
  795. if err := devicemapper.LoopbackSetCapacity(dataloopback); err != nil {
  796. return fmt.Errorf("Unable to update loopback capacity: %s", err)
  797. }
  798. // Suspend the pool
  799. if err := devicemapper.SuspendDevice(devices.getPoolName()); err != nil {
  800. return fmt.Errorf("Unable to suspend pool: %s", err)
  801. }
  802. // Reload with the new block sizes
  803. if err := devicemapper.ReloadPool(devices.getPoolName(), dataloopback, metadataloopback, devices.thinpBlockSize); err != nil {
  804. return fmt.Errorf("Unable to reload pool: %s", err)
  805. }
  806. // Resume the pool
  807. if err := devicemapper.ResumeDevice(devices.getPoolName()); err != nil {
  808. return fmt.Errorf("Unable to resume pool: %s", err)
  809. }
  810. return nil
  811. }
  812. func (devices *DeviceSet) loadTransactionMetaData() error {
  813. jsonData, err := ioutil.ReadFile(devices.transactionMetaFile())
  814. if err != nil {
  815. // There is no active transaction. This will be the case
  816. // during upgrade.
  817. if os.IsNotExist(err) {
  818. devices.OpenTransactionID = devices.TransactionID
  819. return nil
  820. }
  821. return err
  822. }
  823. json.Unmarshal(jsonData, &devices.transaction)
  824. return nil
  825. }
  826. func (devices *DeviceSet) saveTransactionMetaData() error {
  827. jsonData, err := json.Marshal(&devices.transaction)
  828. if err != nil {
  829. return fmt.Errorf("Error encoding metadata to json: %s", err)
  830. }
  831. return devices.writeMetaFile(jsonData, devices.transactionMetaFile())
  832. }
  833. func (devices *DeviceSet) removeTransactionMetaData() error {
  834. if err := os.RemoveAll(devices.transactionMetaFile()); err != nil {
  835. return err
  836. }
  837. return nil
  838. }
  839. func (devices *DeviceSet) rollbackTransaction() error {
  840. logrus.Debugf("Rolling back open transaction: TransactionID=%d hash=%s device_id=%d", devices.OpenTransactionID, devices.DeviceIDHash, devices.DeviceID)
  841. // A device id might have already been deleted before transaction
  842. // closed. In that case this call will fail. Just leave a message
  843. // in case of failure.
  844. if err := devicemapper.DeleteDevice(devices.getPoolDevName(), devices.DeviceID); err != nil {
  845. logrus.Errorf("Unable to delete device: %s", err)
  846. }
  847. dinfo := &devInfo{Hash: devices.DeviceIDHash}
  848. if err := devices.removeMetadata(dinfo); err != nil {
  849. logrus.Errorf("Unable to remove metadata: %s", err)
  850. } else {
  851. devices.markDeviceIDFree(devices.DeviceID)
  852. }
  853. if err := devices.removeTransactionMetaData(); err != nil {
  854. logrus.Errorf("Unable to remove transaction meta file %s: %s", devices.transactionMetaFile(), err)
  855. }
  856. return nil
  857. }
  858. func (devices *DeviceSet) processPendingTransaction() error {
  859. if err := devices.loadTransactionMetaData(); err != nil {
  860. return err
  861. }
  862. // If there was open transaction but pool transaction ID is same
  863. // as open transaction ID, nothing to roll back.
  864. if devices.TransactionID == devices.OpenTransactionID {
  865. return nil
  866. }
  867. // If open transaction ID is less than pool transaction ID, something
  868. // is wrong. Bail out.
  869. if devices.OpenTransactionID < devices.TransactionID {
  870. logrus.Errorf("Open Transaction id %d is less than pool transaction id %d", devices.OpenTransactionID, devices.TransactionID)
  871. return nil
  872. }
  873. // Pool transaction ID is not same as open transaction. There is
  874. // a transaction which was not completed.
  875. if err := devices.rollbackTransaction(); err != nil {
  876. return fmt.Errorf("Rolling back open transaction failed: %s", err)
  877. }
  878. devices.OpenTransactionID = devices.TransactionID
  879. return nil
  880. }
  881. func (devices *DeviceSet) loadDeviceSetMetaData() error {
  882. jsonData, err := ioutil.ReadFile(devices.deviceSetMetaFile())
  883. if err != nil {
  884. // For backward compatibility return success if file does
  885. // not exist.
  886. if os.IsNotExist(err) {
  887. return nil
  888. }
  889. return err
  890. }
  891. return json.Unmarshal(jsonData, devices)
  892. }
  893. func (devices *DeviceSet) saveDeviceSetMetaData() error {
  894. jsonData, err := json.Marshal(devices)
  895. if err != nil {
  896. return fmt.Errorf("Error encoding metadata to json: %s", err)
  897. }
  898. return devices.writeMetaFile(jsonData, devices.deviceSetMetaFile())
  899. }
  900. func (devices *DeviceSet) openTransaction(hash string, DeviceID int) error {
  901. devices.allocateTransactionID()
  902. devices.DeviceIDHash = hash
  903. devices.DeviceID = DeviceID
  904. if err := devices.saveTransactionMetaData(); err != nil {
  905. return fmt.Errorf("Error saving transaction metadata: %s", err)
  906. }
  907. return nil
  908. }
  909. func (devices *DeviceSet) refreshTransaction(DeviceID int) error {
  910. devices.DeviceID = DeviceID
  911. if err := devices.saveTransactionMetaData(); err != nil {
  912. return fmt.Errorf("Error saving transaction metadata: %s", err)
  913. }
  914. return nil
  915. }
  916. func (devices *DeviceSet) closeTransaction() error {
  917. if err := devices.updatePoolTransactionID(); err != nil {
  918. logrus.Debugf("Failed to close Transaction")
  919. return err
  920. }
  921. return nil
  922. }
  923. func determineDriverCapabilities(version string) error {
  924. /*
  925. * Driver version 4.27.0 and greater support deferred activation
  926. * feature.
  927. */
  928. logrus.Debugf("devicemapper: driver version is %s", version)
  929. versionSplit := strings.Split(version, ".")
  930. major, err := strconv.Atoi(versionSplit[0])
  931. if err != nil {
  932. return graphdriver.ErrNotSupported
  933. }
  934. if major > 4 {
  935. driverDeferredRemovalSupport = true
  936. return nil
  937. }
  938. if major < 4 {
  939. return nil
  940. }
  941. minor, err := strconv.Atoi(versionSplit[1])
  942. if err != nil {
  943. return graphdriver.ErrNotSupported
  944. }
  945. /*
  946. * If major is 4 and minor is 27, then there is no need to
  947. * check for patch level as it can not be less than 0.
  948. */
  949. if minor >= 27 {
  950. driverDeferredRemovalSupport = true
  951. return nil
  952. }
  953. return nil
  954. }
  955. // Determine the major and minor number of loopback device
  956. func getDeviceMajorMinor(file *os.File) (uint64, uint64, error) {
  957. stat, err := file.Stat()
  958. if err != nil {
  959. return 0, 0, err
  960. }
  961. dev := stat.Sys().(*syscall.Stat_t).Rdev
  962. majorNum := major(dev)
  963. minorNum := minor(dev)
  964. logrus.Debugf("[devmapper]: Major:Minor for device: %s is:%v:%v", file.Name(), majorNum, minorNum)
  965. return majorNum, minorNum, nil
  966. }
  967. // Given a file which is backing file of a loop back device, find the
  968. // loopback device name and its major/minor number.
  969. func getLoopFileDeviceMajMin(filename string) (string, uint64, uint64, error) {
  970. file, err := os.Open(filename)
  971. if err != nil {
  972. logrus.Debugf("[devmapper]: Failed to open file %s", filename)
  973. return "", 0, 0, err
  974. }
  975. defer file.Close()
  976. loopbackDevice := devicemapper.FindLoopDeviceFor(file)
  977. if loopbackDevice == nil {
  978. return "", 0, 0, fmt.Errorf("[devmapper]: Unable to find loopback mount for: %s", filename)
  979. }
  980. defer loopbackDevice.Close()
  981. Major, Minor, err := getDeviceMajorMinor(loopbackDevice)
  982. if err != nil {
  983. return "", 0, 0, err
  984. }
  985. return loopbackDevice.Name(), Major, Minor, nil
  986. }
  987. // Get the major/minor numbers of thin pool data and metadata devices
  988. func (devices *DeviceSet) getThinPoolDataMetaMajMin() (uint64, uint64, uint64, uint64, error) {
  989. var params, poolDataMajMin, poolMetadataMajMin string
  990. _, _, _, params, err := devicemapper.GetTable(devices.getPoolName())
  991. if err != nil {
  992. return 0, 0, 0, 0, err
  993. }
  994. if _, err = fmt.Sscanf(params, "%s %s", &poolMetadataMajMin, &poolDataMajMin); err != nil {
  995. return 0, 0, 0, 0, err
  996. }
  997. logrus.Debugf("[devmapper]: poolDataMajMin=%s poolMetaMajMin=%s\n", poolDataMajMin, poolMetadataMajMin)
  998. poolDataMajMinorSplit := strings.Split(poolDataMajMin, ":")
  999. poolDataMajor, err := strconv.ParseUint(poolDataMajMinorSplit[0], 10, 32)
  1000. if err != nil {
  1001. return 0, 0, 0, 0, err
  1002. }
  1003. poolDataMinor, err := strconv.ParseUint(poolDataMajMinorSplit[1], 10, 32)
  1004. if err != nil {
  1005. return 0, 0, 0, 0, err
  1006. }
  1007. poolMetadataMajMinorSplit := strings.Split(poolMetadataMajMin, ":")
  1008. poolMetadataMajor, err := strconv.ParseUint(poolMetadataMajMinorSplit[0], 10, 32)
  1009. if err != nil {
  1010. return 0, 0, 0, 0, err
  1011. }
  1012. poolMetadataMinor, err := strconv.ParseUint(poolMetadataMajMinorSplit[1], 10, 32)
  1013. if err != nil {
  1014. return 0, 0, 0, 0, err
  1015. }
  1016. return poolDataMajor, poolDataMinor, poolMetadataMajor, poolMetadataMinor, nil
  1017. }
  1018. func (devices *DeviceSet) loadThinPoolLoopBackInfo() error {
  1019. poolDataMajor, poolDataMinor, poolMetadataMajor, poolMetadataMinor, err := devices.getThinPoolDataMetaMajMin()
  1020. if err != nil {
  1021. return err
  1022. }
  1023. dirname := devices.loopbackDir()
  1024. // data device has not been passed in. So there should be a data file
  1025. // which is being mounted as loop device.
  1026. if devices.dataDevice == "" {
  1027. datafilename := path.Join(dirname, "data")
  1028. dataLoopDevice, dataMajor, dataMinor, err := getLoopFileDeviceMajMin(datafilename)
  1029. if err != nil {
  1030. return err
  1031. }
  1032. // Compare the two
  1033. if poolDataMajor == dataMajor && poolDataMinor == dataMinor {
  1034. devices.dataDevice = dataLoopDevice
  1035. devices.dataLoopFile = datafilename
  1036. }
  1037. }
  1038. // metadata device has not been passed in. So there should be a
  1039. // metadata file which is being mounted as loop device.
  1040. if devices.metadataDevice == "" {
  1041. metadatafilename := path.Join(dirname, "metadata")
  1042. metadataLoopDevice, metadataMajor, metadataMinor, err := getLoopFileDeviceMajMin(metadatafilename)
  1043. if err != nil {
  1044. return err
  1045. }
  1046. if poolMetadataMajor == metadataMajor && poolMetadataMinor == metadataMinor {
  1047. devices.metadataDevice = metadataLoopDevice
  1048. devices.metadataLoopFile = metadatafilename
  1049. }
  1050. }
  1051. return nil
  1052. }
  1053. func (devices *DeviceSet) initDevmapper(doInit bool) error {
  1054. // give ourselves to libdm as a log handler
  1055. devicemapper.LogInit(devices)
  1056. version, err := devicemapper.GetDriverVersion()
  1057. if err != nil {
  1058. // Can't even get driver version, assume not supported
  1059. return graphdriver.ErrNotSupported
  1060. }
  1061. if err := determineDriverCapabilities(version); err != nil {
  1062. return graphdriver.ErrNotSupported
  1063. }
  1064. // If user asked for deferred removal and both library and driver
  1065. // supports deferred removal use it.
  1066. if enableDeferredRemoval && driverDeferredRemovalSupport && devicemapper.LibraryDeferredRemovalSupport == true {
  1067. logrus.Debugf("devmapper: Deferred removal support enabled.")
  1068. devices.deferredRemove = true
  1069. }
  1070. // https://github.com/docker/docker/issues/4036
  1071. if supported := devicemapper.UdevSetSyncSupport(true); !supported {
  1072. logrus.Warn("Udev sync is not supported. This will lead to unexpected behavior, data loss and errors. For more information, see https://docs.docker.com/reference/commandline/daemon/#daemon-storage-driver-option")
  1073. }
  1074. if err := os.MkdirAll(devices.metadataDir(), 0700); err != nil {
  1075. return err
  1076. }
  1077. // Set the device prefix from the device id and inode of the docker root dir
  1078. st, err := os.Stat(devices.root)
  1079. if err != nil {
  1080. return fmt.Errorf("Error looking up dir %s: %s", devices.root, err)
  1081. }
  1082. sysSt := st.Sys().(*syscall.Stat_t)
  1083. // "reg-" stands for "regular file".
  1084. // In the future we might use "dev-" for "device file", etc.
  1085. // docker-maj,min[-inode] stands for:
  1086. // - Managed by docker
  1087. // - The target of this device is at major <maj> and minor <min>
  1088. // - If <inode> is defined, use that file inside the device as a loopback image. Otherwise use the device itself.
  1089. devices.devicePrefix = fmt.Sprintf("docker-%d:%d-%d", major(sysSt.Dev), minor(sysSt.Dev), sysSt.Ino)
  1090. logrus.Debugf("Generated prefix: %s", devices.devicePrefix)
  1091. // Check for the existence of the thin-pool device
  1092. logrus.Debugf("Checking for existence of the pool '%s'", devices.getPoolName())
  1093. info, err := devicemapper.GetInfo(devices.getPoolName())
  1094. if info == nil {
  1095. logrus.Debugf("Error device devicemapper.GetInfo: %s", err)
  1096. return err
  1097. }
  1098. // It seems libdevmapper opens this without O_CLOEXEC, and go exec will not close files
  1099. // that are not Close-on-exec, and lxc-start will die if it inherits any unexpected files,
  1100. // so we add this badhack to make sure it closes itself
  1101. setCloseOnExec("/dev/mapper/control")
  1102. // Make sure the sparse images exist in <root>/devicemapper/data and
  1103. // <root>/devicemapper/metadata
  1104. createdLoopback := false
  1105. // If the pool doesn't exist, create it
  1106. if info.Exists == 0 && devices.thinPoolDevice == "" {
  1107. logrus.Debugf("Pool doesn't exist. Creating it.")
  1108. var (
  1109. dataFile *os.File
  1110. metadataFile *os.File
  1111. )
  1112. if devices.dataDevice == "" {
  1113. // Make sure the sparse images exist in <root>/devicemapper/data
  1114. hasData := devices.hasImage("data")
  1115. if !doInit && !hasData {
  1116. return errors.New("Loopback data file not found")
  1117. }
  1118. if !hasData {
  1119. createdLoopback = true
  1120. }
  1121. data, err := devices.ensureImage("data", devices.dataLoopbackSize)
  1122. if err != nil {
  1123. logrus.Debugf("Error device ensureImage (data): %s", err)
  1124. return err
  1125. }
  1126. dataFile, err = devicemapper.AttachLoopDevice(data)
  1127. if err != nil {
  1128. return err
  1129. }
  1130. devices.dataLoopFile = data
  1131. devices.dataDevice = dataFile.Name()
  1132. } else {
  1133. dataFile, err = os.OpenFile(devices.dataDevice, os.O_RDWR, 0600)
  1134. if err != nil {
  1135. return err
  1136. }
  1137. }
  1138. defer dataFile.Close()
  1139. if devices.metadataDevice == "" {
  1140. // Make sure the sparse images exist in <root>/devicemapper/metadata
  1141. hasMetadata := devices.hasImage("metadata")
  1142. if !doInit && !hasMetadata {
  1143. return errors.New("Loopback metadata file not found")
  1144. }
  1145. if !hasMetadata {
  1146. createdLoopback = true
  1147. }
  1148. metadata, err := devices.ensureImage("metadata", devices.metaDataLoopbackSize)
  1149. if err != nil {
  1150. logrus.Debugf("Error device ensureImage (metadata): %s", err)
  1151. return err
  1152. }
  1153. metadataFile, err = devicemapper.AttachLoopDevice(metadata)
  1154. if err != nil {
  1155. return err
  1156. }
  1157. devices.metadataLoopFile = metadata
  1158. devices.metadataDevice = metadataFile.Name()
  1159. } else {
  1160. metadataFile, err = os.OpenFile(devices.metadataDevice, os.O_RDWR, 0600)
  1161. if err != nil {
  1162. return err
  1163. }
  1164. }
  1165. defer metadataFile.Close()
  1166. if err := devicemapper.CreatePool(devices.getPoolName(), dataFile, metadataFile, devices.thinpBlockSize); err != nil {
  1167. return err
  1168. }
  1169. }
  1170. // Pool already exists and caller did not pass us a pool. That means
  1171. // we probably created pool earlier and could not remove it as some
  1172. // containers were still using it. Detect some of the properties of
  1173. // pool, like is it using loop devices.
  1174. if info.Exists != 0 && devices.thinPoolDevice == "" {
  1175. if err := devices.loadThinPoolLoopBackInfo(); err != nil {
  1176. logrus.Debugf("Failed to load thin pool loopback device information:%v", err)
  1177. return err
  1178. }
  1179. }
  1180. // If we didn't just create the data or metadata image, we need to
  1181. // load the transaction id and migrate old metadata
  1182. if !createdLoopback {
  1183. if err := devices.initMetaData(); err != nil {
  1184. return err
  1185. }
  1186. }
  1187. if devices.thinPoolDevice == "" {
  1188. if devices.metadataLoopFile != "" || devices.dataLoopFile != "" {
  1189. logrus.Warnf("Usage of loopback devices is strongly discouraged for production use. Please use `--storage-opt dm.thinpooldev` or use `man docker` to refer to dm.thinpooldev section.")
  1190. }
  1191. }
  1192. // Right now this loads only NextDeviceID. If there is more metadata
  1193. // down the line, we might have to move it earlier.
  1194. if err := devices.loadDeviceSetMetaData(); err != nil {
  1195. return err
  1196. }
  1197. // Setup the base image
  1198. if doInit {
  1199. if err := devices.setupBaseImage(); err != nil {
  1200. logrus.Debugf("Error device setupBaseImage: %s", err)
  1201. return err
  1202. }
  1203. }
  1204. return nil
  1205. }
  1206. // AddDevice adds a device and registers in the hash.
  1207. func (devices *DeviceSet) AddDevice(hash, baseHash string) error {
  1208. logrus.Debugf("[deviceset] AddDevice(hash=%s basehash=%s)", hash, baseHash)
  1209. defer logrus.Debugf("[deviceset] AddDevice(hash=%s basehash=%s) END", hash, baseHash)
  1210. baseInfo, err := devices.lookupDevice(baseHash)
  1211. if err != nil {
  1212. return err
  1213. }
  1214. baseInfo.lock.Lock()
  1215. defer baseInfo.lock.Unlock()
  1216. devices.Lock()
  1217. defer devices.Unlock()
  1218. if info, _ := devices.lookupDevice(hash); info != nil {
  1219. return fmt.Errorf("device %s already exists", hash)
  1220. }
  1221. if err := devices.createRegisterSnapDevice(hash, baseInfo); err != nil {
  1222. return err
  1223. }
  1224. return nil
  1225. }
  1226. func (devices *DeviceSet) deleteDevice(info *devInfo) error {
  1227. if devices.doBlkDiscard {
  1228. // This is a workaround for the kernel not discarding block so
  1229. // on the thin pool when we remove a thinp device, so we do it
  1230. // manually
  1231. if err := devices.activateDeviceIfNeeded(info); err == nil {
  1232. if err := devicemapper.BlockDeviceDiscard(info.DevName()); err != nil {
  1233. logrus.Debugf("Error discarding block on device: %s (ignoring)", err)
  1234. }
  1235. }
  1236. }
  1237. // Try to deactivate deivce in case it is active.
  1238. if err := devices.deactivateDevice(info); err != nil {
  1239. logrus.Debugf("Error deactivating device: %s", err)
  1240. return err
  1241. }
  1242. if err := devices.openTransaction(info.Hash, info.DeviceID); err != nil {
  1243. logrus.Debugf("Error opening transaction hash = %s deviceID = %d", "", info.DeviceID)
  1244. return err
  1245. }
  1246. if err := devicemapper.DeleteDevice(devices.getPoolDevName(), info.DeviceID); err != nil {
  1247. logrus.Debugf("Error deleting device: %s", err)
  1248. return err
  1249. }
  1250. if err := devices.unregisterDevice(info.DeviceID, info.Hash); err != nil {
  1251. return err
  1252. }
  1253. if err := devices.closeTransaction(); err != nil {
  1254. return err
  1255. }
  1256. devices.markDeviceIDFree(info.DeviceID)
  1257. return nil
  1258. }
  1259. // DeleteDevice deletes a device from the hash.
  1260. func (devices *DeviceSet) DeleteDevice(hash string) error {
  1261. info, err := devices.lookupDevice(hash)
  1262. if err != nil {
  1263. return err
  1264. }
  1265. info.lock.Lock()
  1266. defer info.lock.Unlock()
  1267. devices.Lock()
  1268. defer devices.Unlock()
  1269. // If mountcount is not zero, that means devices is still in use
  1270. // or has not been Put() properly. Fail device deletion.
  1271. if info.mountCount != 0 {
  1272. return fmt.Errorf("devmapper: Can't delete device %v as it is still mounted. mntCount=%v", info.Hash, info.mountCount)
  1273. }
  1274. return devices.deleteDevice(info)
  1275. }
  1276. func (devices *DeviceSet) deactivatePool() error {
  1277. logrus.Debugf("[devmapper] deactivatePool()")
  1278. defer logrus.Debugf("[devmapper] deactivatePool END")
  1279. devname := devices.getPoolDevName()
  1280. devinfo, err := devicemapper.GetInfo(devname)
  1281. if err != nil {
  1282. return err
  1283. }
  1284. if devinfo.Exists == 0 {
  1285. return nil
  1286. }
  1287. if err := devicemapper.RemoveDevice(devname); err != nil {
  1288. return err
  1289. }
  1290. if d, err := devicemapper.GetDeps(devname); err == nil {
  1291. logrus.Warnf("[devmapper] device %s still has %d active dependents", devname, d.Count)
  1292. }
  1293. return nil
  1294. }
  1295. func (devices *DeviceSet) deactivateDevice(info *devInfo) error {
  1296. logrus.Debugf("[devmapper] deactivateDevice(%s)", info.Hash)
  1297. defer logrus.Debugf("[devmapper] deactivateDevice END(%s)", info.Hash)
  1298. devinfo, err := devicemapper.GetInfo(info.Name())
  1299. if err != nil {
  1300. return err
  1301. }
  1302. if devinfo.Exists == 0 {
  1303. return nil
  1304. }
  1305. if devices.deferredRemove {
  1306. if err := devicemapper.RemoveDeviceDeferred(info.Name()); err != nil {
  1307. return err
  1308. }
  1309. } else {
  1310. if err := devices.removeDevice(info.Name()); err != nil {
  1311. return err
  1312. }
  1313. }
  1314. return nil
  1315. }
  1316. // Issues the underlying dm remove operation.
  1317. func (devices *DeviceSet) removeDevice(devname string) error {
  1318. var err error
  1319. logrus.Debugf("[devmapper] removeDevice START(%s)", devname)
  1320. defer logrus.Debugf("[devmapper] removeDevice END(%s)", devname)
  1321. for i := 0; i < 200; i++ {
  1322. err = devicemapper.RemoveDevice(devname)
  1323. if err == nil {
  1324. break
  1325. }
  1326. if err != devicemapper.ErrBusy {
  1327. return err
  1328. }
  1329. // If we see EBUSY it may be a transient error,
  1330. // sleep a bit a retry a few times.
  1331. devices.Unlock()
  1332. time.Sleep(100 * time.Millisecond)
  1333. devices.Lock()
  1334. }
  1335. return err
  1336. }
  1337. func (devices *DeviceSet) cancelDeferredRemoval(info *devInfo) error {
  1338. if !devices.deferredRemove {
  1339. return nil
  1340. }
  1341. logrus.Debugf("[devmapper] cancelDeferredRemoval START(%s)", info.Name())
  1342. defer logrus.Debugf("[devmapper] cancelDeferredRemoval END(%s)", info.Name())
  1343. devinfo, err := devicemapper.GetInfoWithDeferred(info.Name())
  1344. if devinfo != nil && devinfo.DeferredRemove == 0 {
  1345. return nil
  1346. }
  1347. // Cancel deferred remove
  1348. for i := 0; i < 100; i++ {
  1349. err = devicemapper.CancelDeferredRemove(info.Name())
  1350. if err == nil {
  1351. break
  1352. }
  1353. if err == devicemapper.ErrEnxio {
  1354. // Device is probably already gone. Return success.
  1355. return nil
  1356. }
  1357. if err != devicemapper.ErrBusy {
  1358. return err
  1359. }
  1360. // If we see EBUSY it may be a transient error,
  1361. // sleep a bit a retry a few times.
  1362. devices.Unlock()
  1363. time.Sleep(100 * time.Millisecond)
  1364. devices.Lock()
  1365. }
  1366. return err
  1367. }
  1368. // Shutdown shuts down the device by unmounting the root.
  1369. func (devices *DeviceSet) Shutdown() error {
  1370. logrus.Debugf("[deviceset %s] Shutdown()", devices.devicePrefix)
  1371. logrus.Debugf("[devmapper] Shutting down DeviceSet: %s", devices.root)
  1372. defer logrus.Debugf("[deviceset %s] Shutdown() END", devices.devicePrefix)
  1373. var devs []*devInfo
  1374. devices.devicesLock.Lock()
  1375. for _, info := range devices.Devices {
  1376. devs = append(devs, info)
  1377. }
  1378. devices.devicesLock.Unlock()
  1379. for _, info := range devs {
  1380. info.lock.Lock()
  1381. if info.mountCount > 0 {
  1382. // We use MNT_DETACH here in case it is still busy in some running
  1383. // container. This means it'll go away from the global scope directly,
  1384. // and the device will be released when that container dies.
  1385. if err := syscall.Unmount(info.mountPath, syscall.MNT_DETACH); err != nil {
  1386. logrus.Debugf("Shutdown unmounting %s, error: %s", info.mountPath, err)
  1387. }
  1388. devices.Lock()
  1389. if err := devices.deactivateDevice(info); err != nil {
  1390. logrus.Debugf("Shutdown deactivate %s , error: %s", info.Hash, err)
  1391. }
  1392. devices.Unlock()
  1393. }
  1394. info.lock.Unlock()
  1395. }
  1396. info, _ := devices.lookupDevice("")
  1397. if info != nil {
  1398. info.lock.Lock()
  1399. devices.Lock()
  1400. if err := devices.deactivateDevice(info); err != nil {
  1401. logrus.Debugf("Shutdown deactivate base , error: %s", err)
  1402. }
  1403. devices.Unlock()
  1404. info.lock.Unlock()
  1405. }
  1406. devices.Lock()
  1407. if devices.thinPoolDevice == "" {
  1408. if err := devices.deactivatePool(); err != nil {
  1409. logrus.Debugf("Shutdown deactivate pool , error: %s", err)
  1410. }
  1411. }
  1412. devices.saveDeviceSetMetaData()
  1413. devices.Unlock()
  1414. return nil
  1415. }
  1416. // MountDevice mounts the device if not already mounted.
  1417. func (devices *DeviceSet) MountDevice(hash, path, mountLabel string) error {
  1418. info, err := devices.lookupDevice(hash)
  1419. if err != nil {
  1420. return err
  1421. }
  1422. info.lock.Lock()
  1423. defer info.lock.Unlock()
  1424. devices.Lock()
  1425. defer devices.Unlock()
  1426. if info.mountCount > 0 {
  1427. if path != info.mountPath {
  1428. return fmt.Errorf("Trying to mount devmapper device in multiple places (%s, %s)", info.mountPath, path)
  1429. }
  1430. info.mountCount++
  1431. return nil
  1432. }
  1433. if err := devices.activateDeviceIfNeeded(info); err != nil {
  1434. return fmt.Errorf("Error activating devmapper device for '%s': %s", hash, err)
  1435. }
  1436. fstype, err := ProbeFsType(info.DevName())
  1437. if err != nil {
  1438. return err
  1439. }
  1440. options := ""
  1441. if fstype == "xfs" {
  1442. // XFS needs nouuid or it can't mount filesystems with the same fs
  1443. options = joinMountOptions(options, "nouuid")
  1444. }
  1445. options = joinMountOptions(options, devices.mountOptions)
  1446. options = joinMountOptions(options, label.FormatMountLabel("", mountLabel))
  1447. if err := syscall.Mount(info.DevName(), path, fstype, syscall.MS_MGC_VAL, options); err != nil {
  1448. return fmt.Errorf("Error mounting '%s' on '%s': %s", info.DevName(), path, err)
  1449. }
  1450. info.mountCount = 1
  1451. info.mountPath = path
  1452. return nil
  1453. }
  1454. // UnmountDevice unmounts the device and removes it from hash.
  1455. func (devices *DeviceSet) UnmountDevice(hash string) error {
  1456. logrus.Debugf("[devmapper] UnmountDevice(hash=%s)", hash)
  1457. defer logrus.Debugf("[devmapper] UnmountDevice(hash=%s) END", hash)
  1458. info, err := devices.lookupDevice(hash)
  1459. if err != nil {
  1460. return err
  1461. }
  1462. info.lock.Lock()
  1463. defer info.lock.Unlock()
  1464. devices.Lock()
  1465. defer devices.Unlock()
  1466. if info.mountCount == 0 {
  1467. return fmt.Errorf("UnmountDevice: device not-mounted id %s", hash)
  1468. }
  1469. info.mountCount--
  1470. if info.mountCount > 0 {
  1471. return nil
  1472. }
  1473. logrus.Debugf("[devmapper] Unmount(%s)", info.mountPath)
  1474. if err := syscall.Unmount(info.mountPath, syscall.MNT_DETACH); err != nil {
  1475. return err
  1476. }
  1477. logrus.Debugf("[devmapper] Unmount done")
  1478. if err := devices.deactivateDevice(info); err != nil {
  1479. return err
  1480. }
  1481. info.mountPath = ""
  1482. return nil
  1483. }
  1484. // HasDevice returns true if the device metadata exists.
  1485. func (devices *DeviceSet) HasDevice(hash string) bool {
  1486. devices.Lock()
  1487. defer devices.Unlock()
  1488. info, _ := devices.lookupDevice(hash)
  1489. return info != nil
  1490. }
  1491. // List returns a list of device ids.
  1492. func (devices *DeviceSet) List() []string {
  1493. devices.Lock()
  1494. defer devices.Unlock()
  1495. devices.devicesLock.Lock()
  1496. ids := make([]string, len(devices.Devices))
  1497. i := 0
  1498. for k := range devices.Devices {
  1499. ids[i] = k
  1500. i++
  1501. }
  1502. devices.devicesLock.Unlock()
  1503. return ids
  1504. }
  1505. func (devices *DeviceSet) deviceStatus(devName string) (sizeInSectors, mappedSectors, highestMappedSector uint64, err error) {
  1506. var params string
  1507. _, sizeInSectors, _, params, err = devicemapper.GetStatus(devName)
  1508. if err != nil {
  1509. return
  1510. }
  1511. if _, err = fmt.Sscanf(params, "%d %d", &mappedSectors, &highestMappedSector); err == nil {
  1512. return
  1513. }
  1514. return
  1515. }
  1516. // GetDeviceStatus provides size, mapped sectors
  1517. func (devices *DeviceSet) GetDeviceStatus(hash string) (*DevStatus, error) {
  1518. info, err := devices.lookupDevice(hash)
  1519. if err != nil {
  1520. return nil, err
  1521. }
  1522. info.lock.Lock()
  1523. defer info.lock.Unlock()
  1524. devices.Lock()
  1525. defer devices.Unlock()
  1526. status := &DevStatus{
  1527. DeviceID: info.DeviceID,
  1528. Size: info.Size,
  1529. TransactionID: info.TransactionID,
  1530. }
  1531. if err := devices.activateDeviceIfNeeded(info); err != nil {
  1532. return nil, fmt.Errorf("Error activating devmapper device for '%s': %s", hash, err)
  1533. }
  1534. sizeInSectors, mappedSectors, highestMappedSector, err := devices.deviceStatus(info.DevName())
  1535. if err != nil {
  1536. return nil, err
  1537. }
  1538. status.SizeInSectors = sizeInSectors
  1539. status.MappedSectors = mappedSectors
  1540. status.HighestMappedSector = highestMappedSector
  1541. return status, nil
  1542. }
  1543. func (devices *DeviceSet) poolStatus() (totalSizeInSectors, transactionID, dataUsed, dataTotal, metadataUsed, metadataTotal uint64, err error) {
  1544. var params string
  1545. if _, totalSizeInSectors, _, params, err = devicemapper.GetStatus(devices.getPoolName()); err == nil {
  1546. _, err = fmt.Sscanf(params, "%d %d/%d %d/%d", &transactionID, &metadataUsed, &metadataTotal, &dataUsed, &dataTotal)
  1547. }
  1548. return
  1549. }
  1550. // DataDevicePath returns the path to the data storage for this deviceset,
  1551. // regardless of loopback or block device
  1552. func (devices *DeviceSet) DataDevicePath() string {
  1553. return devices.dataDevice
  1554. }
  1555. // MetadataDevicePath returns the path to the metadata storage for this deviceset,
  1556. // regardless of loopback or block device
  1557. func (devices *DeviceSet) MetadataDevicePath() string {
  1558. return devices.metadataDevice
  1559. }
  1560. func (devices *DeviceSet) getUnderlyingAvailableSpace(loopFile string) (uint64, error) {
  1561. buf := new(syscall.Statfs_t)
  1562. if err := syscall.Statfs(loopFile, buf); err != nil {
  1563. logrus.Warnf("Couldn't stat loopfile filesystem %v: %v", loopFile, err)
  1564. return 0, err
  1565. }
  1566. return buf.Bfree * uint64(buf.Bsize), nil
  1567. }
  1568. func (devices *DeviceSet) isRealFile(loopFile string) (bool, error) {
  1569. if loopFile != "" {
  1570. fi, err := os.Stat(loopFile)
  1571. if err != nil {
  1572. logrus.Warnf("Couldn't stat loopfile %v: %v", loopFile, err)
  1573. return false, err
  1574. }
  1575. return fi.Mode().IsRegular(), nil
  1576. }
  1577. return false, nil
  1578. }
  1579. // Status returns the current status of this deviceset
  1580. func (devices *DeviceSet) Status() *Status {
  1581. devices.Lock()
  1582. defer devices.Unlock()
  1583. status := &Status{}
  1584. status.PoolName = devices.getPoolName()
  1585. status.DataFile = devices.DataDevicePath()
  1586. status.DataLoopback = devices.dataLoopFile
  1587. status.MetadataFile = devices.MetadataDevicePath()
  1588. status.MetadataLoopback = devices.metadataLoopFile
  1589. status.UdevSyncSupported = devicemapper.UdevSyncSupported()
  1590. status.DeferredRemoveEnabled = devices.deferredRemove
  1591. totalSizeInSectors, _, dataUsed, dataTotal, metadataUsed, metadataTotal, err := devices.poolStatus()
  1592. if err == nil {
  1593. // Convert from blocks to bytes
  1594. blockSizeInSectors := totalSizeInSectors / dataTotal
  1595. status.Data.Used = dataUsed * blockSizeInSectors * 512
  1596. status.Data.Total = dataTotal * blockSizeInSectors * 512
  1597. status.Data.Available = status.Data.Total - status.Data.Used
  1598. // metadata blocks are always 4k
  1599. status.Metadata.Used = metadataUsed * 4096
  1600. status.Metadata.Total = metadataTotal * 4096
  1601. status.Metadata.Available = status.Metadata.Total - status.Metadata.Used
  1602. status.SectorSize = blockSizeInSectors * 512
  1603. if check, _ := devices.isRealFile(devices.dataLoopFile); check {
  1604. actualSpace, err := devices.getUnderlyingAvailableSpace(devices.dataLoopFile)
  1605. if err == nil && actualSpace < status.Data.Available {
  1606. status.Data.Available = actualSpace
  1607. }
  1608. }
  1609. if check, _ := devices.isRealFile(devices.metadataLoopFile); check {
  1610. actualSpace, err := devices.getUnderlyingAvailableSpace(devices.metadataLoopFile)
  1611. if err == nil && actualSpace < status.Metadata.Available {
  1612. status.Metadata.Available = actualSpace
  1613. }
  1614. }
  1615. }
  1616. return status
  1617. }
  1618. // Status returns the current status of this deviceset
  1619. func (devices *DeviceSet) exportDeviceMetadata(hash string) (*deviceMetadata, error) {
  1620. info, err := devices.lookupDevice(hash)
  1621. if err != nil {
  1622. return nil, err
  1623. }
  1624. info.lock.Lock()
  1625. defer info.lock.Unlock()
  1626. metadata := &deviceMetadata{info.DeviceID, info.Size, info.Name()}
  1627. return metadata, nil
  1628. }
  1629. // NewDeviceSet creates the device set based on the options provided.
  1630. func NewDeviceSet(root string, doInit bool, options []string) (*DeviceSet, error) {
  1631. devicemapper.SetDevDir("/dev")
  1632. devices := &DeviceSet{
  1633. root: root,
  1634. metaData: metaData{Devices: make(map[string]*devInfo)},
  1635. dataLoopbackSize: defaultDataLoopbackSize,
  1636. metaDataLoopbackSize: defaultMetaDataLoopbackSize,
  1637. baseFsSize: defaultBaseFsSize,
  1638. overrideUdevSyncCheck: defaultUdevSyncOverride,
  1639. filesystem: "ext4",
  1640. doBlkDiscard: true,
  1641. thinpBlockSize: defaultThinpBlockSize,
  1642. deviceIDMap: make([]byte, deviceIDMapSz),
  1643. }
  1644. foundBlkDiscard := false
  1645. for _, option := range options {
  1646. key, val, err := parsers.ParseKeyValueOpt(option)
  1647. if err != nil {
  1648. return nil, err
  1649. }
  1650. key = strings.ToLower(key)
  1651. switch key {
  1652. case "dm.basesize":
  1653. size, err := units.RAMInBytes(val)
  1654. if err != nil {
  1655. return nil, err
  1656. }
  1657. devices.baseFsSize = uint64(size)
  1658. case "dm.loopdatasize":
  1659. size, err := units.RAMInBytes(val)
  1660. if err != nil {
  1661. return nil, err
  1662. }
  1663. devices.dataLoopbackSize = size
  1664. case "dm.loopmetadatasize":
  1665. size, err := units.RAMInBytes(val)
  1666. if err != nil {
  1667. return nil, err
  1668. }
  1669. devices.metaDataLoopbackSize = size
  1670. case "dm.fs":
  1671. if val != "ext4" && val != "xfs" {
  1672. return nil, fmt.Errorf("Unsupported filesystem %s\n", val)
  1673. }
  1674. devices.filesystem = val
  1675. case "dm.mkfsarg":
  1676. devices.mkfsArgs = append(devices.mkfsArgs, val)
  1677. case "dm.mountopt":
  1678. devices.mountOptions = joinMountOptions(devices.mountOptions, val)
  1679. case "dm.metadatadev":
  1680. devices.metadataDevice = val
  1681. case "dm.datadev":
  1682. devices.dataDevice = val
  1683. case "dm.thinpooldev":
  1684. devices.thinPoolDevice = strings.TrimPrefix(val, "/dev/mapper/")
  1685. case "dm.blkdiscard":
  1686. foundBlkDiscard = true
  1687. devices.doBlkDiscard, err = strconv.ParseBool(val)
  1688. if err != nil {
  1689. return nil, err
  1690. }
  1691. case "dm.blocksize":
  1692. size, err := units.RAMInBytes(val)
  1693. if err != nil {
  1694. return nil, err
  1695. }
  1696. // convert to 512b sectors
  1697. devices.thinpBlockSize = uint32(size) >> 9
  1698. case "dm.override_udev_sync_check":
  1699. devices.overrideUdevSyncCheck, err = strconv.ParseBool(val)
  1700. if err != nil {
  1701. return nil, err
  1702. }
  1703. case "dm.use_deferred_removal":
  1704. enableDeferredRemoval, err = strconv.ParseBool(val)
  1705. if err != nil {
  1706. return nil, err
  1707. }
  1708. default:
  1709. return nil, fmt.Errorf("Unknown option %s\n", key)
  1710. }
  1711. }
  1712. // By default, don't do blk discard hack on raw devices, its rarely useful and is expensive
  1713. if !foundBlkDiscard && (devices.dataDevice != "" || devices.thinPoolDevice != "") {
  1714. devices.doBlkDiscard = false
  1715. }
  1716. if err := devices.initDevmapper(doInit); err != nil {
  1717. return nil, err
  1718. }
  1719. return devices, nil
  1720. }