deviceset.go 71 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455
  1. // +build linux
  2. package devmapper
  3. import (
  4. "bufio"
  5. "encoding/json"
  6. "errors"
  7. "fmt"
  8. "io"
  9. "io/ioutil"
  10. "os"
  11. "os/exec"
  12. "path"
  13. "path/filepath"
  14. "strconv"
  15. "strings"
  16. "sync"
  17. "syscall"
  18. "time"
  19. "github.com/Sirupsen/logrus"
  20. "github.com/docker/docker/daemon/graphdriver"
  21. "github.com/docker/docker/pkg/devicemapper"
  22. "github.com/docker/docker/pkg/idtools"
  23. "github.com/docker/docker/pkg/mount"
  24. "github.com/docker/docker/pkg/parsers"
  25. "github.com/docker/go-units"
  26. "github.com/opencontainers/runc/libcontainer/label"
  27. )
  28. var (
  29. defaultDataLoopbackSize int64 = 100 * 1024 * 1024 * 1024
  30. defaultMetaDataLoopbackSize int64 = 2 * 1024 * 1024 * 1024
  31. defaultBaseFsSize uint64 = 100 * 1024 * 1024 * 1024
  32. defaultThinpBlockSize uint32 = 128 // 64K = 128 512b sectors
  33. defaultUdevSyncOverride = false
  34. maxDeviceID = 0xffffff // 24 bit, pool limit
  35. deviceIDMapSz = (maxDeviceID + 1) / 8
  36. // We retry device removal so many a times that even error messages
  37. // will fill up console during normal operation. So only log Fatal
  38. // messages by default.
  39. logLevel = devicemapper.LogLevelFatal
  40. driverDeferredRemovalSupport = false
  41. enableDeferredRemoval = false
  42. enableDeferredDeletion = false
  43. )
  44. const deviceSetMetaFile string = "deviceset-metadata"
  45. const transactionMetaFile string = "transaction-metadata"
  46. type transaction struct {
  47. OpenTransactionID uint64 `json:"open_transaction_id"`
  48. DeviceIDHash string `json:"device_hash"`
  49. DeviceID int `json:"device_id"`
  50. }
  51. type devInfo struct {
  52. Hash string `json:"-"`
  53. DeviceID int `json:"device_id"`
  54. Size uint64 `json:"size"`
  55. TransactionID uint64 `json:"transaction_id"`
  56. Initialized bool `json:"initialized"`
  57. Deleted bool `json:"deleted"`
  58. devices *DeviceSet
  59. mountCount int
  60. mountPath string
  61. // The global DeviceSet lock guarantees that we serialize all
  62. // the calls to libdevmapper (which is not threadsafe), but we
  63. // sometimes release that lock while sleeping. In that case
  64. // this per-device lock is still held, protecting against
  65. // other accesses to the device that we're doing the wait on.
  66. //
  67. // WARNING: In order to avoid AB-BA deadlocks when releasing
  68. // the global lock while holding the per-device locks all
  69. // device locks must be acquired *before* the device lock, and
  70. // multiple device locks should be acquired parent before child.
  71. lock sync.Mutex
  72. }
  73. type metaData struct {
  74. Devices map[string]*devInfo `json:"Devices"`
  75. }
  76. // DeviceSet holds information about list of devices
  77. type DeviceSet struct {
  78. metaData `json:"-"`
  79. sync.Mutex `json:"-"` // Protects all fields of DeviceSet and serializes calls into libdevmapper
  80. root string
  81. devicePrefix string
  82. TransactionID uint64 `json:"-"`
  83. NextDeviceID int `json:"next_device_id"`
  84. deviceIDMap []byte
  85. // Options
  86. dataLoopbackSize int64
  87. metaDataLoopbackSize int64
  88. baseFsSize uint64
  89. filesystem string
  90. mountOptions string
  91. mkfsArgs []string
  92. dataDevice string // block or loop dev
  93. dataLoopFile string // loopback file, if used
  94. metadataDevice string // block or loop dev
  95. metadataLoopFile string // loopback file, if used
  96. doBlkDiscard bool
  97. thinpBlockSize uint32
  98. thinPoolDevice string
  99. transaction `json:"-"`
  100. overrideUdevSyncCheck bool
  101. deferredRemove bool // use deferred removal
  102. deferredDelete bool // use deferred deletion
  103. BaseDeviceUUID string // save UUID of base device
  104. BaseDeviceFilesystem string // save filesystem of base device
  105. nrDeletedDevices uint // number of deleted devices
  106. deletionWorkerTicker *time.Ticker
  107. uidMaps []idtools.IDMap
  108. gidMaps []idtools.IDMap
  109. }
  110. // DiskUsage contains information about disk usage and is used when reporting Status of a device.
  111. type DiskUsage struct {
  112. // Used bytes on the disk.
  113. Used uint64
  114. // Total bytes on the disk.
  115. Total uint64
  116. // Available bytes on the disk.
  117. Available uint64
  118. }
  119. // Status returns the information about the device.
  120. type Status struct {
  121. // PoolName is the name of the data pool.
  122. PoolName string
  123. // DataFile is the actual block device for data.
  124. DataFile string
  125. // DataLoopback loopback file, if used.
  126. DataLoopback string
  127. // MetadataFile is the actual block device for metadata.
  128. MetadataFile string
  129. // MetadataLoopback is the loopback file, if used.
  130. MetadataLoopback string
  131. // Data is the disk used for data.
  132. Data DiskUsage
  133. // Metadata is the disk used for meta data.
  134. Metadata DiskUsage
  135. // BaseDeviceSize is base size of container and image
  136. BaseDeviceSize uint64
  137. // BaseDeviceFS is backing filesystem.
  138. BaseDeviceFS string
  139. // SectorSize size of the vector.
  140. SectorSize uint64
  141. // UdevSyncSupported is true if sync is supported.
  142. UdevSyncSupported bool
  143. // DeferredRemoveEnabled is true then the device is not unmounted.
  144. DeferredRemoveEnabled bool
  145. // True if deferred deletion is enabled. This is different from
  146. // deferred removal. "removal" means that device mapper device is
  147. // deactivated. Thin device is still in thin pool and can be activated
  148. // again. But "deletion" means that thin device will be deleted from
  149. // thin pool and it can't be activated again.
  150. DeferredDeleteEnabled bool
  151. DeferredDeletedDeviceCount uint
  152. }
  153. // Structure used to export image/container metadata in docker inspect.
  154. type deviceMetadata struct {
  155. deviceID int
  156. deviceSize uint64 // size in bytes
  157. deviceName string // Device name as used during activation
  158. }
  159. // DevStatus returns information about device mounted containing its id, size and sector information.
  160. type DevStatus struct {
  161. // DeviceID is the id of the device.
  162. DeviceID int
  163. // Size is the size of the filesystem.
  164. Size uint64
  165. // TransactionID is a unique integer per device set used to identify an operation on the file system, this number is incremental.
  166. TransactionID uint64
  167. // SizeInSectors indicates the size of the sectors allocated.
  168. SizeInSectors uint64
  169. // MappedSectors indicates number of mapped sectors.
  170. MappedSectors uint64
  171. // HighestMappedSector is the pointer to the highest mapped sector.
  172. HighestMappedSector uint64
  173. }
  174. func getDevName(name string) string {
  175. return "/dev/mapper/" + name
  176. }
  177. func (info *devInfo) Name() string {
  178. hash := info.Hash
  179. if hash == "" {
  180. hash = "base"
  181. }
  182. return fmt.Sprintf("%s-%s", info.devices.devicePrefix, hash)
  183. }
  184. func (info *devInfo) DevName() string {
  185. return getDevName(info.Name())
  186. }
  187. func (devices *DeviceSet) loopbackDir() string {
  188. return path.Join(devices.root, "devicemapper")
  189. }
  190. func (devices *DeviceSet) metadataDir() string {
  191. return path.Join(devices.root, "metadata")
  192. }
  193. func (devices *DeviceSet) metadataFile(info *devInfo) string {
  194. file := info.Hash
  195. if file == "" {
  196. file = "base"
  197. }
  198. return path.Join(devices.metadataDir(), file)
  199. }
  200. func (devices *DeviceSet) transactionMetaFile() string {
  201. return path.Join(devices.metadataDir(), transactionMetaFile)
  202. }
  203. func (devices *DeviceSet) deviceSetMetaFile() string {
  204. return path.Join(devices.metadataDir(), deviceSetMetaFile)
  205. }
  206. func (devices *DeviceSet) oldMetadataFile() string {
  207. return path.Join(devices.loopbackDir(), "json")
  208. }
  209. func (devices *DeviceSet) getPoolName() string {
  210. if devices.thinPoolDevice == "" {
  211. return devices.devicePrefix + "-pool"
  212. }
  213. return devices.thinPoolDevice
  214. }
  215. func (devices *DeviceSet) getPoolDevName() string {
  216. return getDevName(devices.getPoolName())
  217. }
  218. func (devices *DeviceSet) hasImage(name string) bool {
  219. dirname := devices.loopbackDir()
  220. filename := path.Join(dirname, name)
  221. _, err := os.Stat(filename)
  222. return err == nil
  223. }
  224. // ensureImage creates a sparse file of <size> bytes at the path
  225. // <root>/devicemapper/<name>.
  226. // If the file already exists and new size is larger than its current size, it grows to the new size.
  227. // Either way it returns the full path.
  228. func (devices *DeviceSet) ensureImage(name string, size int64) (string, error) {
  229. dirname := devices.loopbackDir()
  230. filename := path.Join(dirname, name)
  231. uid, gid, err := idtools.GetRootUIDGID(devices.uidMaps, devices.gidMaps)
  232. if err != nil {
  233. return "", err
  234. }
  235. if err := idtools.MkdirAllAs(dirname, 0700, uid, gid); err != nil && !os.IsExist(err) {
  236. return "", err
  237. }
  238. if fi, err := os.Stat(filename); err != nil {
  239. if !os.IsNotExist(err) {
  240. return "", err
  241. }
  242. logrus.Debugf("devmapper: Creating loopback file %s for device-manage use", filename)
  243. file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0600)
  244. if err != nil {
  245. return "", err
  246. }
  247. defer file.Close()
  248. if err := file.Truncate(size); err != nil {
  249. return "", err
  250. }
  251. } else {
  252. if fi.Size() < size {
  253. file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0600)
  254. if err != nil {
  255. return "", err
  256. }
  257. defer file.Close()
  258. if err := file.Truncate(size); err != nil {
  259. return "", fmt.Errorf("devmapper: Unable to grow loopback file %s: %v", filename, err)
  260. }
  261. } else if fi.Size() > size {
  262. logrus.Warnf("devmapper: Can't shrink loopback file %s", filename)
  263. }
  264. }
  265. return filename, nil
  266. }
  267. func (devices *DeviceSet) allocateTransactionID() uint64 {
  268. devices.OpenTransactionID = devices.TransactionID + 1
  269. return devices.OpenTransactionID
  270. }
  271. func (devices *DeviceSet) updatePoolTransactionID() error {
  272. if err := devicemapper.SetTransactionID(devices.getPoolDevName(), devices.TransactionID, devices.OpenTransactionID); err != nil {
  273. return fmt.Errorf("devmapper: Error setting devmapper transaction ID: %s", err)
  274. }
  275. devices.TransactionID = devices.OpenTransactionID
  276. return nil
  277. }
  278. func (devices *DeviceSet) removeMetadata(info *devInfo) error {
  279. if err := os.RemoveAll(devices.metadataFile(info)); err != nil {
  280. return fmt.Errorf("devmapper: Error removing metadata file %s: %s", devices.metadataFile(info), err)
  281. }
  282. return nil
  283. }
  284. // Given json data and file path, write it to disk
  285. func (devices *DeviceSet) writeMetaFile(jsonData []byte, filePath string) error {
  286. tmpFile, err := ioutil.TempFile(devices.metadataDir(), ".tmp")
  287. if err != nil {
  288. return fmt.Errorf("devmapper: Error creating metadata file: %s", err)
  289. }
  290. n, err := tmpFile.Write(jsonData)
  291. if err != nil {
  292. return fmt.Errorf("devmapper: Error writing metadata to %s: %s", tmpFile.Name(), err)
  293. }
  294. if n < len(jsonData) {
  295. return io.ErrShortWrite
  296. }
  297. if err := tmpFile.Sync(); err != nil {
  298. return fmt.Errorf("devmapper: Error syncing metadata file %s: %s", tmpFile.Name(), err)
  299. }
  300. if err := tmpFile.Close(); err != nil {
  301. return fmt.Errorf("devmapper: Error closing metadata file %s: %s", tmpFile.Name(), err)
  302. }
  303. if err := os.Rename(tmpFile.Name(), filePath); err != nil {
  304. return fmt.Errorf("devmapper: Error committing metadata file %s: %s", tmpFile.Name(), err)
  305. }
  306. return nil
  307. }
  308. func (devices *DeviceSet) saveMetadata(info *devInfo) error {
  309. jsonData, err := json.Marshal(info)
  310. if err != nil {
  311. return fmt.Errorf("devmapper: Error encoding metadata to json: %s", err)
  312. }
  313. if err := devices.writeMetaFile(jsonData, devices.metadataFile(info)); err != nil {
  314. return err
  315. }
  316. return nil
  317. }
  318. func (devices *DeviceSet) markDeviceIDUsed(deviceID int) {
  319. var mask byte
  320. i := deviceID % 8
  321. mask = 1 << uint(i)
  322. devices.deviceIDMap[deviceID/8] = devices.deviceIDMap[deviceID/8] | mask
  323. }
  324. func (devices *DeviceSet) markDeviceIDFree(deviceID int) {
  325. var mask byte
  326. i := deviceID % 8
  327. mask = ^(1 << uint(i))
  328. devices.deviceIDMap[deviceID/8] = devices.deviceIDMap[deviceID/8] & mask
  329. }
  330. func (devices *DeviceSet) isDeviceIDFree(deviceID int) bool {
  331. var mask byte
  332. i := deviceID % 8
  333. mask = (1 << uint(i))
  334. if (devices.deviceIDMap[deviceID/8] & mask) != 0 {
  335. return false
  336. }
  337. return true
  338. }
  339. // Should be called with devices.Lock() held.
  340. func (devices *DeviceSet) lookupDevice(hash string) (*devInfo, error) {
  341. info := devices.Devices[hash]
  342. if info == nil {
  343. info = devices.loadMetadata(hash)
  344. if info == nil {
  345. return nil, fmt.Errorf("devmapper: Unknown device %s", hash)
  346. }
  347. devices.Devices[hash] = info
  348. }
  349. return info, nil
  350. }
  351. func (devices *DeviceSet) lookupDeviceWithLock(hash string) (*devInfo, error) {
  352. devices.Lock()
  353. defer devices.Unlock()
  354. info, err := devices.lookupDevice(hash)
  355. return info, err
  356. }
  357. // This function relies on that device hash map has been loaded in advance.
  358. // Should be called with devices.Lock() held.
  359. func (devices *DeviceSet) constructDeviceIDMap() {
  360. logrus.Debugf("devmapper: constructDeviceIDMap()")
  361. defer logrus.Debugf("devmapper: constructDeviceIDMap() END")
  362. for _, info := range devices.Devices {
  363. devices.markDeviceIDUsed(info.DeviceID)
  364. logrus.Debugf("devmapper: Added deviceId=%d to DeviceIdMap", info.DeviceID)
  365. }
  366. }
  367. func (devices *DeviceSet) deviceFileWalkFunction(path string, finfo os.FileInfo) error {
  368. // Skip some of the meta files which are not device files.
  369. if strings.HasSuffix(finfo.Name(), ".migrated") {
  370. logrus.Debugf("devmapper: Skipping file %s", path)
  371. return nil
  372. }
  373. if strings.HasPrefix(finfo.Name(), ".") {
  374. logrus.Debugf("devmapper: Skipping file %s", path)
  375. return nil
  376. }
  377. if finfo.Name() == deviceSetMetaFile {
  378. logrus.Debugf("devmapper: Skipping file %s", path)
  379. return nil
  380. }
  381. if finfo.Name() == transactionMetaFile {
  382. logrus.Debugf("devmapper: Skipping file %s", path)
  383. return nil
  384. }
  385. logrus.Debugf("devmapper: Loading data for file %s", path)
  386. hash := finfo.Name()
  387. if hash == "base" {
  388. hash = ""
  389. }
  390. // Include deleted devices also as cleanup delete device logic
  391. // will go through it and see if there are any deleted devices.
  392. if _, err := devices.lookupDevice(hash); err != nil {
  393. return fmt.Errorf("devmapper: Error looking up device %s:%v", hash, err)
  394. }
  395. return nil
  396. }
  397. func (devices *DeviceSet) loadDeviceFilesOnStart() error {
  398. logrus.Debugf("devmapper: loadDeviceFilesOnStart()")
  399. defer logrus.Debugf("devmapper: loadDeviceFilesOnStart() END")
  400. var scan = func(path string, info os.FileInfo, err error) error {
  401. if err != nil {
  402. logrus.Debugf("devmapper: Can't walk the file %s", path)
  403. return nil
  404. }
  405. // Skip any directories
  406. if info.IsDir() {
  407. return nil
  408. }
  409. return devices.deviceFileWalkFunction(path, info)
  410. }
  411. return filepath.Walk(devices.metadataDir(), scan)
  412. }
  413. // Should be called with devices.Lock() held.
  414. func (devices *DeviceSet) unregisterDevice(id int, hash string) error {
  415. logrus.Debugf("devmapper: unregisterDevice(%v, %v)", id, hash)
  416. info := &devInfo{
  417. Hash: hash,
  418. DeviceID: id,
  419. }
  420. delete(devices.Devices, hash)
  421. if err := devices.removeMetadata(info); err != nil {
  422. logrus.Debugf("devmapper: Error removing metadata: %s", err)
  423. return err
  424. }
  425. return nil
  426. }
  427. // Should be called with devices.Lock() held.
  428. func (devices *DeviceSet) registerDevice(id int, hash string, size uint64, transactionID uint64) (*devInfo, error) {
  429. logrus.Debugf("devmapper: registerDevice(%v, %v)", id, hash)
  430. info := &devInfo{
  431. Hash: hash,
  432. DeviceID: id,
  433. Size: size,
  434. TransactionID: transactionID,
  435. Initialized: false,
  436. devices: devices,
  437. }
  438. devices.Devices[hash] = info
  439. if err := devices.saveMetadata(info); err != nil {
  440. // Try to remove unused device
  441. delete(devices.Devices, hash)
  442. return nil, err
  443. }
  444. return info, nil
  445. }
  446. func (devices *DeviceSet) activateDeviceIfNeeded(info *devInfo, ignoreDeleted bool) error {
  447. logrus.Debugf("devmapper: activateDeviceIfNeeded(%v)", info.Hash)
  448. if info.Deleted && !ignoreDeleted {
  449. return fmt.Errorf("devmapper: Can't activate device %v as it is marked for deletion", info.Hash)
  450. }
  451. // Make sure deferred removal on device is canceled, if one was
  452. // scheduled.
  453. if err := devices.cancelDeferredRemoval(info); err != nil {
  454. return fmt.Errorf("devmapper: Device Deferred Removal Cancellation Failed: %s", err)
  455. }
  456. if devinfo, _ := devicemapper.GetInfo(info.Name()); devinfo != nil && devinfo.Exists != 0 {
  457. return nil
  458. }
  459. return devicemapper.ActivateDevice(devices.getPoolDevName(), info.Name(), info.DeviceID, info.Size)
  460. }
  461. // Return true only if kernel supports xfs and mkfs.xfs is available
  462. func xfsSupported() bool {
  463. // Make sure mkfs.xfs is available
  464. if _, err := exec.LookPath("mkfs.xfs"); err != nil {
  465. return false
  466. }
  467. // Check if kernel supports xfs filesystem or not.
  468. exec.Command("modprobe", "xfs").Run()
  469. f, err := os.Open("/proc/filesystems")
  470. if err != nil {
  471. logrus.Warnf("devmapper: Could not check if xfs is supported: %v", err)
  472. return false
  473. }
  474. defer f.Close()
  475. s := bufio.NewScanner(f)
  476. for s.Scan() {
  477. if strings.HasSuffix(s.Text(), "\txfs") {
  478. return true
  479. }
  480. }
  481. if err := s.Err(); err != nil {
  482. logrus.Warnf("devmapper: Could not check if xfs is supported: %v", err)
  483. }
  484. return false
  485. }
  486. func determineDefaultFS() string {
  487. if xfsSupported() {
  488. return "xfs"
  489. }
  490. logrus.Warn("devmapper: XFS is not supported in your system. Either the kernel doesnt support it or mkfs.xfs is not in your PATH. Defaulting to ext4 filesystem")
  491. return "ext4"
  492. }
  493. func (devices *DeviceSet) createFilesystem(info *devInfo) (err error) {
  494. devname := info.DevName()
  495. args := []string{}
  496. for _, arg := range devices.mkfsArgs {
  497. args = append(args, arg)
  498. }
  499. args = append(args, devname)
  500. if devices.filesystem == "" {
  501. devices.filesystem = determineDefaultFS()
  502. }
  503. if err := devices.saveBaseDeviceFilesystem(devices.filesystem); err != nil {
  504. return err
  505. }
  506. logrus.Infof("devmapper: Creating filesystem %s on device %s", devices.filesystem, info.Name())
  507. defer func() {
  508. if err != nil {
  509. logrus.Infof("devmapper: Error while creating filesystem %s on device %s: %v", devices.filesystem, info.Name(), err)
  510. } else {
  511. logrus.Infof("devmapper: Successfully created filesystem %s on device %s", devices.filesystem, info.Name())
  512. }
  513. }()
  514. switch devices.filesystem {
  515. case "xfs":
  516. err = exec.Command("mkfs.xfs", args...).Run()
  517. case "ext4":
  518. err = exec.Command("mkfs.ext4", append([]string{"-E", "nodiscard,lazy_itable_init=0,lazy_journal_init=0"}, args...)...).Run()
  519. if err != nil {
  520. err = exec.Command("mkfs.ext4", append([]string{"-E", "nodiscard,lazy_itable_init=0"}, args...)...).Run()
  521. }
  522. if err != nil {
  523. return err
  524. }
  525. err = exec.Command("tune2fs", append([]string{"-c", "-1", "-i", "0"}, devname)...).Run()
  526. default:
  527. err = fmt.Errorf("devmapper: Unsupported filesystem type %s", devices.filesystem)
  528. }
  529. return
  530. }
  531. func (devices *DeviceSet) migrateOldMetaData() error {
  532. // Migrate old metadata file
  533. jsonData, err := ioutil.ReadFile(devices.oldMetadataFile())
  534. if err != nil && !os.IsNotExist(err) {
  535. return err
  536. }
  537. if jsonData != nil {
  538. m := metaData{Devices: make(map[string]*devInfo)}
  539. if err := json.Unmarshal(jsonData, &m); err != nil {
  540. return err
  541. }
  542. for hash, info := range m.Devices {
  543. info.Hash = hash
  544. devices.saveMetadata(info)
  545. }
  546. if err := os.Rename(devices.oldMetadataFile(), devices.oldMetadataFile()+".migrated"); err != nil {
  547. return err
  548. }
  549. }
  550. return nil
  551. }
  552. // Cleanup deleted devices. It assumes that all the devices have been
  553. // loaded in the hash table.
  554. func (devices *DeviceSet) cleanupDeletedDevices() error {
  555. devices.Lock()
  556. // If there are no deleted devices, there is nothing to do.
  557. if devices.nrDeletedDevices == 0 {
  558. devices.Unlock()
  559. return nil
  560. }
  561. var deletedDevices []*devInfo
  562. for _, info := range devices.Devices {
  563. if !info.Deleted {
  564. continue
  565. }
  566. logrus.Debugf("devmapper: Found deleted device %s.", info.Hash)
  567. deletedDevices = append(deletedDevices, info)
  568. }
  569. // Delete the deleted devices. DeleteDevice() first takes the info lock
  570. // and then devices.Lock(). So drop it to avoid deadlock.
  571. devices.Unlock()
  572. for _, info := range deletedDevices {
  573. // This will again try deferred deletion.
  574. if err := devices.DeleteDevice(info.Hash, false); err != nil {
  575. logrus.Warnf("devmapper: Deletion of device %s, device_id=%v failed:%v", info.Hash, info.DeviceID, err)
  576. }
  577. }
  578. return nil
  579. }
  580. func (devices *DeviceSet) countDeletedDevices() {
  581. for _, info := range devices.Devices {
  582. if !info.Deleted {
  583. continue
  584. }
  585. devices.nrDeletedDevices++
  586. }
  587. }
  588. func (devices *DeviceSet) startDeviceDeletionWorker() {
  589. // Deferred deletion is not enabled. Don't do anything.
  590. if !devices.deferredDelete {
  591. return
  592. }
  593. logrus.Debugf("devmapper: Worker to cleanup deleted devices started")
  594. for range devices.deletionWorkerTicker.C {
  595. devices.cleanupDeletedDevices()
  596. }
  597. }
  598. func (devices *DeviceSet) initMetaData() error {
  599. devices.Lock()
  600. defer devices.Unlock()
  601. if err := devices.migrateOldMetaData(); err != nil {
  602. return err
  603. }
  604. _, transactionID, _, _, _, _, err := devices.poolStatus()
  605. if err != nil {
  606. return err
  607. }
  608. devices.TransactionID = transactionID
  609. if err := devices.loadDeviceFilesOnStart(); err != nil {
  610. return fmt.Errorf("devmapper: Failed to load device files:%v", err)
  611. }
  612. devices.constructDeviceIDMap()
  613. devices.countDeletedDevices()
  614. if err := devices.processPendingTransaction(); err != nil {
  615. return err
  616. }
  617. // Start a goroutine to cleanup Deleted Devices
  618. go devices.startDeviceDeletionWorker()
  619. return nil
  620. }
  621. func (devices *DeviceSet) incNextDeviceID() {
  622. // IDs are 24bit, so wrap around
  623. devices.NextDeviceID = (devices.NextDeviceID + 1) & maxDeviceID
  624. }
  625. func (devices *DeviceSet) getNextFreeDeviceID() (int, error) {
  626. devices.incNextDeviceID()
  627. for i := 0; i <= maxDeviceID; i++ {
  628. if devices.isDeviceIDFree(devices.NextDeviceID) {
  629. devices.markDeviceIDUsed(devices.NextDeviceID)
  630. return devices.NextDeviceID, nil
  631. }
  632. devices.incNextDeviceID()
  633. }
  634. return 0, fmt.Errorf("devmapper: Unable to find a free device ID")
  635. }
  636. func (devices *DeviceSet) createRegisterDevice(hash string) (*devInfo, error) {
  637. devices.Lock()
  638. defer devices.Unlock()
  639. deviceID, err := devices.getNextFreeDeviceID()
  640. if err != nil {
  641. return nil, err
  642. }
  643. if err := devices.openTransaction(hash, deviceID); err != nil {
  644. logrus.Debugf("devmapper: Error opening transaction hash = %s deviceID = %d", hash, deviceID)
  645. devices.markDeviceIDFree(deviceID)
  646. return nil, err
  647. }
  648. for {
  649. if err := devicemapper.CreateDevice(devices.getPoolDevName(), deviceID); err != nil {
  650. if devicemapper.DeviceIDExists(err) {
  651. // Device ID already exists. This should not
  652. // happen. Now we have a mechanism to find
  653. // a free device ID. So something is not right.
  654. // Give a warning and continue.
  655. logrus.Errorf("devmapper: Device ID %d exists in pool but it is supposed to be unused", deviceID)
  656. deviceID, err = devices.getNextFreeDeviceID()
  657. if err != nil {
  658. return nil, err
  659. }
  660. // Save new device id into transaction
  661. devices.refreshTransaction(deviceID)
  662. continue
  663. }
  664. logrus.Debugf("devmapper: Error creating device: %s", err)
  665. devices.markDeviceIDFree(deviceID)
  666. return nil, err
  667. }
  668. break
  669. }
  670. logrus.Debugf("devmapper: Registering device (id %v) with FS size %v", deviceID, devices.baseFsSize)
  671. info, err := devices.registerDevice(deviceID, hash, devices.baseFsSize, devices.OpenTransactionID)
  672. if err != nil {
  673. _ = devicemapper.DeleteDevice(devices.getPoolDevName(), deviceID)
  674. devices.markDeviceIDFree(deviceID)
  675. return nil, err
  676. }
  677. if err := devices.closeTransaction(); err != nil {
  678. devices.unregisterDevice(deviceID, hash)
  679. devicemapper.DeleteDevice(devices.getPoolDevName(), deviceID)
  680. devices.markDeviceIDFree(deviceID)
  681. return nil, err
  682. }
  683. return info, nil
  684. }
  685. func (devices *DeviceSet) createRegisterSnapDevice(hash string, baseInfo *devInfo) error {
  686. deviceID, err := devices.getNextFreeDeviceID()
  687. if err != nil {
  688. return err
  689. }
  690. if err := devices.openTransaction(hash, deviceID); err != nil {
  691. logrus.Debugf("devmapper: Error opening transaction hash = %s deviceID = %d", hash, deviceID)
  692. devices.markDeviceIDFree(deviceID)
  693. return err
  694. }
  695. for {
  696. if err := devicemapper.CreateSnapDevice(devices.getPoolDevName(), deviceID, baseInfo.Name(), baseInfo.DeviceID); err != nil {
  697. if devicemapper.DeviceIDExists(err) {
  698. // Device ID already exists. This should not
  699. // happen. Now we have a mechanism to find
  700. // a free device ID. So something is not right.
  701. // Give a warning and continue.
  702. logrus.Errorf("devmapper: Device ID %d exists in pool but it is supposed to be unused", deviceID)
  703. deviceID, err = devices.getNextFreeDeviceID()
  704. if err != nil {
  705. return err
  706. }
  707. // Save new device id into transaction
  708. devices.refreshTransaction(deviceID)
  709. continue
  710. }
  711. logrus.Debugf("devmapper: Error creating snap device: %s", err)
  712. devices.markDeviceIDFree(deviceID)
  713. return err
  714. }
  715. break
  716. }
  717. if _, err := devices.registerDevice(deviceID, hash, baseInfo.Size, devices.OpenTransactionID); err != nil {
  718. devicemapper.DeleteDevice(devices.getPoolDevName(), deviceID)
  719. devices.markDeviceIDFree(deviceID)
  720. logrus.Debugf("devmapper: Error registering device: %s", err)
  721. return err
  722. }
  723. if err := devices.closeTransaction(); err != nil {
  724. devices.unregisterDevice(deviceID, hash)
  725. devicemapper.DeleteDevice(devices.getPoolDevName(), deviceID)
  726. devices.markDeviceIDFree(deviceID)
  727. return err
  728. }
  729. return nil
  730. }
  731. func (devices *DeviceSet) loadMetadata(hash string) *devInfo {
  732. info := &devInfo{Hash: hash, devices: devices}
  733. jsonData, err := ioutil.ReadFile(devices.metadataFile(info))
  734. if err != nil {
  735. return nil
  736. }
  737. if err := json.Unmarshal(jsonData, &info); err != nil {
  738. return nil
  739. }
  740. if info.DeviceID > maxDeviceID {
  741. logrus.Errorf("devmapper: Ignoring Invalid DeviceId=%d", info.DeviceID)
  742. return nil
  743. }
  744. return info
  745. }
  746. func getDeviceUUID(device string) (string, error) {
  747. out, err := exec.Command("blkid", "-s", "UUID", "-o", "value", device).Output()
  748. if err != nil {
  749. return "", fmt.Errorf("devmapper: Failed to find uuid for device %s:%v", device, err)
  750. }
  751. uuid := strings.TrimSuffix(string(out), "\n")
  752. uuid = strings.TrimSpace(uuid)
  753. logrus.Debugf("devmapper: UUID for device: %s is:%s", device, uuid)
  754. return uuid, nil
  755. }
  756. func (devices *DeviceSet) getBaseDeviceSize() uint64 {
  757. info, _ := devices.lookupDevice("")
  758. if info == nil {
  759. return 0
  760. }
  761. return info.Size
  762. }
  763. func (devices *DeviceSet) getBaseDeviceFS() string {
  764. return devices.BaseDeviceFilesystem
  765. }
  766. func (devices *DeviceSet) verifyBaseDeviceUUIDFS(baseInfo *devInfo) error {
  767. devices.Lock()
  768. defer devices.Unlock()
  769. if err := devices.activateDeviceIfNeeded(baseInfo, false); err != nil {
  770. return err
  771. }
  772. defer devices.deactivateDevice(baseInfo)
  773. uuid, err := getDeviceUUID(baseInfo.DevName())
  774. if err != nil {
  775. return err
  776. }
  777. if devices.BaseDeviceUUID != uuid {
  778. return fmt.Errorf("devmapper: Current Base Device UUID:%s does not match with stored UUID:%s. Possibly using a different thin pool than last invocation", uuid, devices.BaseDeviceUUID)
  779. }
  780. if devices.BaseDeviceFilesystem == "" {
  781. fsType, err := ProbeFsType(baseInfo.DevName())
  782. if err != nil {
  783. return err
  784. }
  785. if err := devices.saveBaseDeviceFilesystem(fsType); err != nil {
  786. return err
  787. }
  788. }
  789. // If user specified a filesystem using dm.fs option and current
  790. // file system of base image is not same, warn user that dm.fs
  791. // will be ignored.
  792. if devices.BaseDeviceFilesystem != devices.filesystem {
  793. logrus.Warnf("devmapper: Base device already exists and has filesystem %s on it. User specified filesystem %s will be ignored.", devices.BaseDeviceFilesystem, devices.filesystem)
  794. devices.filesystem = devices.BaseDeviceFilesystem
  795. }
  796. return nil
  797. }
  798. func (devices *DeviceSet) saveBaseDeviceFilesystem(fs string) error {
  799. devices.BaseDeviceFilesystem = fs
  800. return devices.saveDeviceSetMetaData()
  801. }
  802. func (devices *DeviceSet) saveBaseDeviceUUID(baseInfo *devInfo) error {
  803. devices.Lock()
  804. defer devices.Unlock()
  805. if err := devices.activateDeviceIfNeeded(baseInfo, false); err != nil {
  806. return err
  807. }
  808. defer devices.deactivateDevice(baseInfo)
  809. uuid, err := getDeviceUUID(baseInfo.DevName())
  810. if err != nil {
  811. return err
  812. }
  813. devices.BaseDeviceUUID = uuid
  814. return devices.saveDeviceSetMetaData()
  815. }
  816. func (devices *DeviceSet) createBaseImage() error {
  817. logrus.Debugf("devmapper: Initializing base device-mapper thin volume")
  818. // Create initial device
  819. info, err := devices.createRegisterDevice("")
  820. if err != nil {
  821. return err
  822. }
  823. logrus.Debugf("devmapper: Creating filesystem on base device-mapper thin volume")
  824. if err := devices.activateDeviceIfNeeded(info, false); err != nil {
  825. return err
  826. }
  827. if err := devices.createFilesystem(info); err != nil {
  828. return err
  829. }
  830. info.Initialized = true
  831. if err := devices.saveMetadata(info); err != nil {
  832. info.Initialized = false
  833. return err
  834. }
  835. if err := devices.saveBaseDeviceUUID(info); err != nil {
  836. return fmt.Errorf("devmapper: Could not query and save base device UUID:%v", err)
  837. }
  838. return nil
  839. }
  840. // Returns if thin pool device exists or not. If device exists, also makes
  841. // sure it is a thin pool device and not some other type of device.
  842. func (devices *DeviceSet) thinPoolExists(thinPoolDevice string) (bool, error) {
  843. logrus.Debugf("devmapper: Checking for existence of the pool %s", thinPoolDevice)
  844. info, err := devicemapper.GetInfo(thinPoolDevice)
  845. if err != nil {
  846. return false, fmt.Errorf("devmapper: GetInfo() on device %s failed: %v", thinPoolDevice, err)
  847. }
  848. // Device does not exist.
  849. if info.Exists == 0 {
  850. return false, nil
  851. }
  852. _, _, deviceType, _, err := devicemapper.GetStatus(thinPoolDevice)
  853. if err != nil {
  854. return false, fmt.Errorf("devmapper: GetStatus() on device %s failed: %v", thinPoolDevice, err)
  855. }
  856. if deviceType != "thin-pool" {
  857. return false, fmt.Errorf("devmapper: Device %s is not a thin pool", thinPoolDevice)
  858. }
  859. return true, nil
  860. }
  861. func (devices *DeviceSet) checkThinPool() error {
  862. _, transactionID, dataUsed, _, _, _, err := devices.poolStatus()
  863. if err != nil {
  864. return err
  865. }
  866. if dataUsed != 0 {
  867. return fmt.Errorf("devmapper: Unable to take ownership of thin-pool (%s) that already has used data blocks",
  868. devices.thinPoolDevice)
  869. }
  870. if transactionID != 0 {
  871. return fmt.Errorf("devmapper: Unable to take ownership of thin-pool (%s) with non-zero transaction ID",
  872. devices.thinPoolDevice)
  873. }
  874. return nil
  875. }
  876. // Base image is initialized properly. Either save UUID for first time (for
  877. // upgrade case or verify UUID.
  878. func (devices *DeviceSet) setupVerifyBaseImageUUIDFS(baseInfo *devInfo) error {
  879. // If BaseDeviceUUID is nil (upgrade case), save it and return success.
  880. if devices.BaseDeviceUUID == "" {
  881. if err := devices.saveBaseDeviceUUID(baseInfo); err != nil {
  882. return fmt.Errorf("devmapper: Could not query and save base device UUID:%v", err)
  883. }
  884. return nil
  885. }
  886. if err := devices.verifyBaseDeviceUUIDFS(baseInfo); err != nil {
  887. return fmt.Errorf("devmapper: Base Device UUID and Filesystem verification failed.%v", err)
  888. }
  889. return nil
  890. }
  891. func (devices *DeviceSet) setupBaseImage() error {
  892. oldInfo, _ := devices.lookupDeviceWithLock("")
  893. // base image already exists. If it is initialized properly, do UUID
  894. // verification and return. Otherwise remove image and set it up
  895. // fresh.
  896. if oldInfo != nil {
  897. if oldInfo.Initialized && !oldInfo.Deleted {
  898. if err := devices.setupVerifyBaseImageUUIDFS(oldInfo); err != nil {
  899. return err
  900. }
  901. if devices.baseFsSize != defaultBaseFsSize && devices.baseFsSize != devices.getBaseDeviceSize() {
  902. logrus.Warnf("devmapper: Base device is already initialized to size %s, new value of base device size %s will not take effect",
  903. units.HumanSize(float64(devices.getBaseDeviceSize())), units.HumanSize(float64(devices.baseFsSize)))
  904. }
  905. return nil
  906. }
  907. logrus.Debugf("devmapper: Removing uninitialized base image")
  908. // If previous base device is in deferred delete state,
  909. // that needs to be cleaned up first. So don't try
  910. // deferred deletion.
  911. if err := devices.DeleteDevice("", true); err != nil {
  912. return err
  913. }
  914. }
  915. // If we are setting up base image for the first time, make sure
  916. // thin pool is empty.
  917. if devices.thinPoolDevice != "" && oldInfo == nil {
  918. if err := devices.checkThinPool(); err != nil {
  919. return err
  920. }
  921. }
  922. // Create new base image device
  923. if err := devices.createBaseImage(); err != nil {
  924. return err
  925. }
  926. return nil
  927. }
  928. func setCloseOnExec(name string) {
  929. if fileInfos, _ := ioutil.ReadDir("/proc/self/fd"); fileInfos != nil {
  930. for _, i := range fileInfos {
  931. link, _ := os.Readlink(filepath.Join("/proc/self/fd", i.Name()))
  932. if link == name {
  933. fd, err := strconv.Atoi(i.Name())
  934. if err == nil {
  935. syscall.CloseOnExec(fd)
  936. }
  937. }
  938. }
  939. }
  940. }
  941. // DMLog implements logging using DevMapperLogger interface.
  942. func (devices *DeviceSet) DMLog(level int, file string, line int, dmError int, message string) {
  943. // By default libdm sends us all the messages including debug ones.
  944. // We need to filter out messages here and figure out which one
  945. // should be printed.
  946. if level > logLevel {
  947. return
  948. }
  949. // FIXME(vbatts) push this back into ./pkg/devicemapper/
  950. if level <= devicemapper.LogLevelErr {
  951. logrus.Errorf("libdevmapper(%d): %s:%d (%d) %s", level, file, line, dmError, message)
  952. } else if level <= devicemapper.LogLevelInfo {
  953. logrus.Infof("libdevmapper(%d): %s:%d (%d) %s", level, file, line, dmError, message)
  954. } else {
  955. // FIXME(vbatts) push this back into ./pkg/devicemapper/
  956. logrus.Debugf("libdevmapper(%d): %s:%d (%d) %s", level, file, line, dmError, message)
  957. }
  958. }
  959. func major(device uint64) uint64 {
  960. return (device >> 8) & 0xfff
  961. }
  962. func minor(device uint64) uint64 {
  963. return (device & 0xff) | ((device >> 12) & 0xfff00)
  964. }
  965. // ResizePool increases the size of the pool.
  966. func (devices *DeviceSet) ResizePool(size int64) error {
  967. dirname := devices.loopbackDir()
  968. datafilename := path.Join(dirname, "data")
  969. if len(devices.dataDevice) > 0 {
  970. datafilename = devices.dataDevice
  971. }
  972. metadatafilename := path.Join(dirname, "metadata")
  973. if len(devices.metadataDevice) > 0 {
  974. metadatafilename = devices.metadataDevice
  975. }
  976. datafile, err := os.OpenFile(datafilename, os.O_RDWR, 0)
  977. if datafile == nil {
  978. return err
  979. }
  980. defer datafile.Close()
  981. fi, err := datafile.Stat()
  982. if fi == nil {
  983. return err
  984. }
  985. if fi.Size() > size {
  986. return fmt.Errorf("devmapper: Can't shrink file")
  987. }
  988. dataloopback := devicemapper.FindLoopDeviceFor(datafile)
  989. if dataloopback == nil {
  990. return fmt.Errorf("devmapper: Unable to find loopback mount for: %s", datafilename)
  991. }
  992. defer dataloopback.Close()
  993. metadatafile, err := os.OpenFile(metadatafilename, os.O_RDWR, 0)
  994. if metadatafile == nil {
  995. return err
  996. }
  997. defer metadatafile.Close()
  998. metadataloopback := devicemapper.FindLoopDeviceFor(metadatafile)
  999. if metadataloopback == nil {
  1000. return fmt.Errorf("devmapper: Unable to find loopback mount for: %s", metadatafilename)
  1001. }
  1002. defer metadataloopback.Close()
  1003. // Grow loopback file
  1004. if err := datafile.Truncate(size); err != nil {
  1005. return fmt.Errorf("devmapper: Unable to grow loopback file: %s", err)
  1006. }
  1007. // Reload size for loopback device
  1008. if err := devicemapper.LoopbackSetCapacity(dataloopback); err != nil {
  1009. return fmt.Errorf("devmapper: Unable to update loopback capacity: %s", err)
  1010. }
  1011. // Suspend the pool
  1012. if err := devicemapper.SuspendDevice(devices.getPoolName()); err != nil {
  1013. return fmt.Errorf("devmapper: Unable to suspend pool: %s", err)
  1014. }
  1015. // Reload with the new block sizes
  1016. if err := devicemapper.ReloadPool(devices.getPoolName(), dataloopback, metadataloopback, devices.thinpBlockSize); err != nil {
  1017. return fmt.Errorf("devmapper: Unable to reload pool: %s", err)
  1018. }
  1019. // Resume the pool
  1020. if err := devicemapper.ResumeDevice(devices.getPoolName()); err != nil {
  1021. return fmt.Errorf("devmapper: Unable to resume pool: %s", err)
  1022. }
  1023. return nil
  1024. }
  1025. func (devices *DeviceSet) loadTransactionMetaData() error {
  1026. jsonData, err := ioutil.ReadFile(devices.transactionMetaFile())
  1027. if err != nil {
  1028. // There is no active transaction. This will be the case
  1029. // during upgrade.
  1030. if os.IsNotExist(err) {
  1031. devices.OpenTransactionID = devices.TransactionID
  1032. return nil
  1033. }
  1034. return err
  1035. }
  1036. json.Unmarshal(jsonData, &devices.transaction)
  1037. return nil
  1038. }
  1039. func (devices *DeviceSet) saveTransactionMetaData() error {
  1040. jsonData, err := json.Marshal(&devices.transaction)
  1041. if err != nil {
  1042. return fmt.Errorf("devmapper: Error encoding metadata to json: %s", err)
  1043. }
  1044. return devices.writeMetaFile(jsonData, devices.transactionMetaFile())
  1045. }
  1046. func (devices *DeviceSet) removeTransactionMetaData() error {
  1047. if err := os.RemoveAll(devices.transactionMetaFile()); err != nil {
  1048. return err
  1049. }
  1050. return nil
  1051. }
  1052. func (devices *DeviceSet) rollbackTransaction() error {
  1053. logrus.Debugf("devmapper: Rolling back open transaction: TransactionID=%d hash=%s device_id=%d", devices.OpenTransactionID, devices.DeviceIDHash, devices.DeviceID)
  1054. // A device id might have already been deleted before transaction
  1055. // closed. In that case this call will fail. Just leave a message
  1056. // in case of failure.
  1057. if err := devicemapper.DeleteDevice(devices.getPoolDevName(), devices.DeviceID); err != nil {
  1058. logrus.Errorf("devmapper: Unable to delete device: %s", err)
  1059. }
  1060. dinfo := &devInfo{Hash: devices.DeviceIDHash}
  1061. if err := devices.removeMetadata(dinfo); err != nil {
  1062. logrus.Errorf("devmapper: Unable to remove metadata: %s", err)
  1063. } else {
  1064. devices.markDeviceIDFree(devices.DeviceID)
  1065. }
  1066. if err := devices.removeTransactionMetaData(); err != nil {
  1067. logrus.Errorf("devmapper: Unable to remove transaction meta file %s: %s", devices.transactionMetaFile(), err)
  1068. }
  1069. return nil
  1070. }
  1071. func (devices *DeviceSet) processPendingTransaction() error {
  1072. if err := devices.loadTransactionMetaData(); err != nil {
  1073. return err
  1074. }
  1075. // If there was open transaction but pool transaction ID is same
  1076. // as open transaction ID, nothing to roll back.
  1077. if devices.TransactionID == devices.OpenTransactionID {
  1078. return nil
  1079. }
  1080. // If open transaction ID is less than pool transaction ID, something
  1081. // is wrong. Bail out.
  1082. if devices.OpenTransactionID < devices.TransactionID {
  1083. logrus.Errorf("devmapper: Open Transaction id %d is less than pool transaction id %d", devices.OpenTransactionID, devices.TransactionID)
  1084. return nil
  1085. }
  1086. // Pool transaction ID is not same as open transaction. There is
  1087. // a transaction which was not completed.
  1088. if err := devices.rollbackTransaction(); err != nil {
  1089. return fmt.Errorf("devmapper: Rolling back open transaction failed: %s", err)
  1090. }
  1091. devices.OpenTransactionID = devices.TransactionID
  1092. return nil
  1093. }
  1094. func (devices *DeviceSet) loadDeviceSetMetaData() error {
  1095. jsonData, err := ioutil.ReadFile(devices.deviceSetMetaFile())
  1096. if err != nil {
  1097. // For backward compatibility return success if file does
  1098. // not exist.
  1099. if os.IsNotExist(err) {
  1100. return nil
  1101. }
  1102. return err
  1103. }
  1104. return json.Unmarshal(jsonData, devices)
  1105. }
  1106. func (devices *DeviceSet) saveDeviceSetMetaData() error {
  1107. jsonData, err := json.Marshal(devices)
  1108. if err != nil {
  1109. return fmt.Errorf("devmapper: Error encoding metadata to json: %s", err)
  1110. }
  1111. return devices.writeMetaFile(jsonData, devices.deviceSetMetaFile())
  1112. }
  1113. func (devices *DeviceSet) openTransaction(hash string, DeviceID int) error {
  1114. devices.allocateTransactionID()
  1115. devices.DeviceIDHash = hash
  1116. devices.DeviceID = DeviceID
  1117. if err := devices.saveTransactionMetaData(); err != nil {
  1118. return fmt.Errorf("devmapper: Error saving transaction metadata: %s", err)
  1119. }
  1120. return nil
  1121. }
  1122. func (devices *DeviceSet) refreshTransaction(DeviceID int) error {
  1123. devices.DeviceID = DeviceID
  1124. if err := devices.saveTransactionMetaData(); err != nil {
  1125. return fmt.Errorf("devmapper: Error saving transaction metadata: %s", err)
  1126. }
  1127. return nil
  1128. }
  1129. func (devices *DeviceSet) closeTransaction() error {
  1130. if err := devices.updatePoolTransactionID(); err != nil {
  1131. logrus.Debugf("devmapper: Failed to close Transaction")
  1132. return err
  1133. }
  1134. return nil
  1135. }
  1136. func determineDriverCapabilities(version string) error {
  1137. /*
  1138. * Driver version 4.27.0 and greater support deferred activation
  1139. * feature.
  1140. */
  1141. logrus.Debugf("devicemapper: driver version is %s", version)
  1142. versionSplit := strings.Split(version, ".")
  1143. major, err := strconv.Atoi(versionSplit[0])
  1144. if err != nil {
  1145. return graphdriver.ErrNotSupported
  1146. }
  1147. if major > 4 {
  1148. driverDeferredRemovalSupport = true
  1149. return nil
  1150. }
  1151. if major < 4 {
  1152. return nil
  1153. }
  1154. minor, err := strconv.Atoi(versionSplit[1])
  1155. if err != nil {
  1156. return graphdriver.ErrNotSupported
  1157. }
  1158. /*
  1159. * If major is 4 and minor is 27, then there is no need to
  1160. * check for patch level as it can not be less than 0.
  1161. */
  1162. if minor >= 27 {
  1163. driverDeferredRemovalSupport = true
  1164. return nil
  1165. }
  1166. return nil
  1167. }
  1168. // Determine the major and minor number of loopback device
  1169. func getDeviceMajorMinor(file *os.File) (uint64, uint64, error) {
  1170. stat, err := file.Stat()
  1171. if err != nil {
  1172. return 0, 0, err
  1173. }
  1174. dev := stat.Sys().(*syscall.Stat_t).Rdev
  1175. majorNum := major(dev)
  1176. minorNum := minor(dev)
  1177. logrus.Debugf("devmapper: Major:Minor for device: %s is:%v:%v", file.Name(), majorNum, minorNum)
  1178. return majorNum, minorNum, nil
  1179. }
  1180. // Given a file which is backing file of a loop back device, find the
  1181. // loopback device name and its major/minor number.
  1182. func getLoopFileDeviceMajMin(filename string) (string, uint64, uint64, error) {
  1183. file, err := os.Open(filename)
  1184. if err != nil {
  1185. logrus.Debugf("devmapper: Failed to open file %s", filename)
  1186. return "", 0, 0, err
  1187. }
  1188. defer file.Close()
  1189. loopbackDevice := devicemapper.FindLoopDeviceFor(file)
  1190. if loopbackDevice == nil {
  1191. return "", 0, 0, fmt.Errorf("devmapper: Unable to find loopback mount for: %s", filename)
  1192. }
  1193. defer loopbackDevice.Close()
  1194. Major, Minor, err := getDeviceMajorMinor(loopbackDevice)
  1195. if err != nil {
  1196. return "", 0, 0, err
  1197. }
  1198. return loopbackDevice.Name(), Major, Minor, nil
  1199. }
  1200. // Get the major/minor numbers of thin pool data and metadata devices
  1201. func (devices *DeviceSet) getThinPoolDataMetaMajMin() (uint64, uint64, uint64, uint64, error) {
  1202. var params, poolDataMajMin, poolMetadataMajMin string
  1203. _, _, _, params, err := devicemapper.GetTable(devices.getPoolName())
  1204. if err != nil {
  1205. return 0, 0, 0, 0, err
  1206. }
  1207. if _, err = fmt.Sscanf(params, "%s %s", &poolMetadataMajMin, &poolDataMajMin); err != nil {
  1208. return 0, 0, 0, 0, err
  1209. }
  1210. logrus.Debugf("devmapper: poolDataMajMin=%s poolMetaMajMin=%s\n", poolDataMajMin, poolMetadataMajMin)
  1211. poolDataMajMinorSplit := strings.Split(poolDataMajMin, ":")
  1212. poolDataMajor, err := strconv.ParseUint(poolDataMajMinorSplit[0], 10, 32)
  1213. if err != nil {
  1214. return 0, 0, 0, 0, err
  1215. }
  1216. poolDataMinor, err := strconv.ParseUint(poolDataMajMinorSplit[1], 10, 32)
  1217. if err != nil {
  1218. return 0, 0, 0, 0, err
  1219. }
  1220. poolMetadataMajMinorSplit := strings.Split(poolMetadataMajMin, ":")
  1221. poolMetadataMajor, err := strconv.ParseUint(poolMetadataMajMinorSplit[0], 10, 32)
  1222. if err != nil {
  1223. return 0, 0, 0, 0, err
  1224. }
  1225. poolMetadataMinor, err := strconv.ParseUint(poolMetadataMajMinorSplit[1], 10, 32)
  1226. if err != nil {
  1227. return 0, 0, 0, 0, err
  1228. }
  1229. return poolDataMajor, poolDataMinor, poolMetadataMajor, poolMetadataMinor, nil
  1230. }
  1231. func (devices *DeviceSet) loadThinPoolLoopBackInfo() error {
  1232. poolDataMajor, poolDataMinor, poolMetadataMajor, poolMetadataMinor, err := devices.getThinPoolDataMetaMajMin()
  1233. if err != nil {
  1234. return err
  1235. }
  1236. dirname := devices.loopbackDir()
  1237. // data device has not been passed in. So there should be a data file
  1238. // which is being mounted as loop device.
  1239. if devices.dataDevice == "" {
  1240. datafilename := path.Join(dirname, "data")
  1241. dataLoopDevice, dataMajor, dataMinor, err := getLoopFileDeviceMajMin(datafilename)
  1242. if err != nil {
  1243. return err
  1244. }
  1245. // Compare the two
  1246. if poolDataMajor == dataMajor && poolDataMinor == dataMinor {
  1247. devices.dataDevice = dataLoopDevice
  1248. devices.dataLoopFile = datafilename
  1249. }
  1250. }
  1251. // metadata device has not been passed in. So there should be a
  1252. // metadata file which is being mounted as loop device.
  1253. if devices.metadataDevice == "" {
  1254. metadatafilename := path.Join(dirname, "metadata")
  1255. metadataLoopDevice, metadataMajor, metadataMinor, err := getLoopFileDeviceMajMin(metadatafilename)
  1256. if err != nil {
  1257. return err
  1258. }
  1259. if poolMetadataMajor == metadataMajor && poolMetadataMinor == metadataMinor {
  1260. devices.metadataDevice = metadataLoopDevice
  1261. devices.metadataLoopFile = metadatafilename
  1262. }
  1263. }
  1264. return nil
  1265. }
  1266. func (devices *DeviceSet) initDevmapper(doInit bool) error {
  1267. // give ourselves to libdm as a log handler
  1268. devicemapper.LogInit(devices)
  1269. version, err := devicemapper.GetDriverVersion()
  1270. if err != nil {
  1271. // Can't even get driver version, assume not supported
  1272. return graphdriver.ErrNotSupported
  1273. }
  1274. if err := determineDriverCapabilities(version); err != nil {
  1275. return graphdriver.ErrNotSupported
  1276. }
  1277. // If user asked for deferred removal then check both libdm library
  1278. // and kernel driver support deferred removal otherwise error out.
  1279. if enableDeferredRemoval {
  1280. if !driverDeferredRemovalSupport {
  1281. return fmt.Errorf("devmapper: Deferred removal can not be enabled as kernel does not support it")
  1282. }
  1283. if !devicemapper.LibraryDeferredRemovalSupport {
  1284. return fmt.Errorf("devmapper: Deferred removal can not be enabled as libdm does not support it")
  1285. }
  1286. logrus.Debugf("devmapper: Deferred removal support enabled.")
  1287. devices.deferredRemove = true
  1288. }
  1289. if enableDeferredDeletion {
  1290. if !devices.deferredRemove {
  1291. return fmt.Errorf("devmapper: Deferred deletion can not be enabled as deferred removal is not enabled. Enable deferred removal using --storage-opt dm.use_deferred_removal=true parameter")
  1292. }
  1293. logrus.Debugf("devmapper: Deferred deletion support enabled.")
  1294. devices.deferredDelete = true
  1295. }
  1296. // https://github.com/docker/docker/issues/4036
  1297. if supported := devicemapper.UdevSetSyncSupport(true); !supported {
  1298. logrus.Warn("devmapper: Udev sync is not supported. This will lead to unexpected behavior, data loss and errors. For more information, see https://docs.docker.com/reference/commandline/daemon/#daemon-storage-driver-option")
  1299. }
  1300. //create the root dir of the devmapper driver ownership to match this
  1301. //daemon's remapped root uid/gid so containers can start properly
  1302. uid, gid, err := idtools.GetRootUIDGID(devices.uidMaps, devices.gidMaps)
  1303. if err != nil {
  1304. return err
  1305. }
  1306. if err := idtools.MkdirAs(devices.root, 0700, uid, gid); err != nil && !os.IsExist(err) {
  1307. return err
  1308. }
  1309. if err := os.MkdirAll(devices.metadataDir(), 0700); err != nil && !os.IsExist(err) {
  1310. return err
  1311. }
  1312. // Set the device prefix from the device id and inode of the docker root dir
  1313. st, err := os.Stat(devices.root)
  1314. if err != nil {
  1315. return fmt.Errorf("devmapper: Error looking up dir %s: %s", devices.root, err)
  1316. }
  1317. sysSt := st.Sys().(*syscall.Stat_t)
  1318. // "reg-" stands for "regular file".
  1319. // In the future we might use "dev-" for "device file", etc.
  1320. // docker-maj,min[-inode] stands for:
  1321. // - Managed by docker
  1322. // - The target of this device is at major <maj> and minor <min>
  1323. // - If <inode> is defined, use that file inside the device as a loopback image. Otherwise use the device itself.
  1324. devices.devicePrefix = fmt.Sprintf("docker-%d:%d-%d", major(sysSt.Dev), minor(sysSt.Dev), sysSt.Ino)
  1325. logrus.Debugf("devmapper: Generated prefix: %s", devices.devicePrefix)
  1326. // Check for the existence of the thin-pool device
  1327. poolExists, err := devices.thinPoolExists(devices.getPoolName())
  1328. if err != nil {
  1329. return err
  1330. }
  1331. // It seems libdevmapper opens this without O_CLOEXEC, and go exec will not close files
  1332. // that are not Close-on-exec,
  1333. // so we add this badhack to make sure it closes itself
  1334. setCloseOnExec("/dev/mapper/control")
  1335. // Make sure the sparse images exist in <root>/devicemapper/data and
  1336. // <root>/devicemapper/metadata
  1337. createdLoopback := false
  1338. // If the pool doesn't exist, create it
  1339. if !poolExists && devices.thinPoolDevice == "" {
  1340. logrus.Debugf("devmapper: Pool doesn't exist. Creating it.")
  1341. var (
  1342. dataFile *os.File
  1343. metadataFile *os.File
  1344. )
  1345. if devices.dataDevice == "" {
  1346. // Make sure the sparse images exist in <root>/devicemapper/data
  1347. hasData := devices.hasImage("data")
  1348. if !doInit && !hasData {
  1349. return errors.New("Loopback data file not found")
  1350. }
  1351. if !hasData {
  1352. createdLoopback = true
  1353. }
  1354. data, err := devices.ensureImage("data", devices.dataLoopbackSize)
  1355. if err != nil {
  1356. logrus.Debugf("devmapper: Error device ensureImage (data): %s", err)
  1357. return err
  1358. }
  1359. dataFile, err = devicemapper.AttachLoopDevice(data)
  1360. if err != nil {
  1361. return err
  1362. }
  1363. devices.dataLoopFile = data
  1364. devices.dataDevice = dataFile.Name()
  1365. } else {
  1366. dataFile, err = os.OpenFile(devices.dataDevice, os.O_RDWR, 0600)
  1367. if err != nil {
  1368. return err
  1369. }
  1370. }
  1371. defer dataFile.Close()
  1372. if devices.metadataDevice == "" {
  1373. // Make sure the sparse images exist in <root>/devicemapper/metadata
  1374. hasMetadata := devices.hasImage("metadata")
  1375. if !doInit && !hasMetadata {
  1376. return errors.New("Loopback metadata file not found")
  1377. }
  1378. if !hasMetadata {
  1379. createdLoopback = true
  1380. }
  1381. metadata, err := devices.ensureImage("metadata", devices.metaDataLoopbackSize)
  1382. if err != nil {
  1383. logrus.Debugf("devmapper: Error device ensureImage (metadata): %s", err)
  1384. return err
  1385. }
  1386. metadataFile, err = devicemapper.AttachLoopDevice(metadata)
  1387. if err != nil {
  1388. return err
  1389. }
  1390. devices.metadataLoopFile = metadata
  1391. devices.metadataDevice = metadataFile.Name()
  1392. } else {
  1393. metadataFile, err = os.OpenFile(devices.metadataDevice, os.O_RDWR, 0600)
  1394. if err != nil {
  1395. return err
  1396. }
  1397. }
  1398. defer metadataFile.Close()
  1399. if err := devicemapper.CreatePool(devices.getPoolName(), dataFile, metadataFile, devices.thinpBlockSize); err != nil {
  1400. return err
  1401. }
  1402. }
  1403. // Pool already exists and caller did not pass us a pool. That means
  1404. // we probably created pool earlier and could not remove it as some
  1405. // containers were still using it. Detect some of the properties of
  1406. // pool, like is it using loop devices.
  1407. if poolExists && devices.thinPoolDevice == "" {
  1408. if err := devices.loadThinPoolLoopBackInfo(); err != nil {
  1409. logrus.Debugf("devmapper: Failed to load thin pool loopback device information:%v", err)
  1410. return err
  1411. }
  1412. }
  1413. // If we didn't just create the data or metadata image, we need to
  1414. // load the transaction id and migrate old metadata
  1415. if !createdLoopback {
  1416. if err := devices.initMetaData(); err != nil {
  1417. return err
  1418. }
  1419. }
  1420. if devices.thinPoolDevice == "" {
  1421. if devices.metadataLoopFile != "" || devices.dataLoopFile != "" {
  1422. logrus.Warnf("devmapper: Usage of loopback devices is strongly discouraged for production use. Please use `--storage-opt dm.thinpooldev` or use `man docker` to refer to dm.thinpooldev section.")
  1423. }
  1424. }
  1425. // Right now this loads only NextDeviceID. If there is more metadata
  1426. // down the line, we might have to move it earlier.
  1427. if err := devices.loadDeviceSetMetaData(); err != nil {
  1428. return err
  1429. }
  1430. // Setup the base image
  1431. if doInit {
  1432. if err := devices.setupBaseImage(); err != nil {
  1433. logrus.Debugf("devmapper: Error device setupBaseImage: %s", err)
  1434. return err
  1435. }
  1436. }
  1437. return nil
  1438. }
  1439. // AddDevice adds a device and registers in the hash.
  1440. func (devices *DeviceSet) AddDevice(hash, baseHash string) error {
  1441. logrus.Debugf("devmapper: AddDevice(hash=%s basehash=%s)", hash, baseHash)
  1442. defer logrus.Debugf("devmapper: AddDevice(hash=%s basehash=%s) END", hash, baseHash)
  1443. // If a deleted device exists, return error.
  1444. baseInfo, err := devices.lookupDeviceWithLock(baseHash)
  1445. if err != nil {
  1446. return err
  1447. }
  1448. if baseInfo.Deleted {
  1449. return fmt.Errorf("devmapper: Base device %v has been marked for deferred deletion", baseInfo.Hash)
  1450. }
  1451. baseInfo.lock.Lock()
  1452. defer baseInfo.lock.Unlock()
  1453. devices.Lock()
  1454. defer devices.Unlock()
  1455. // Also include deleted devices in case hash of new device is
  1456. // same as one of the deleted devices.
  1457. if info, _ := devices.lookupDevice(hash); info != nil {
  1458. return fmt.Errorf("devmapper: device %s already exists. Deleted=%v", hash, info.Deleted)
  1459. }
  1460. if err := devices.createRegisterSnapDevice(hash, baseInfo); err != nil {
  1461. return err
  1462. }
  1463. return nil
  1464. }
  1465. func (devices *DeviceSet) markForDeferredDeletion(info *devInfo) error {
  1466. // If device is already in deleted state, there is nothing to be done.
  1467. if info.Deleted {
  1468. return nil
  1469. }
  1470. logrus.Debugf("devmapper: Marking device %s for deferred deletion.", info.Hash)
  1471. info.Deleted = true
  1472. // save device metadata to reflect deleted state.
  1473. if err := devices.saveMetadata(info); err != nil {
  1474. info.Deleted = false
  1475. return err
  1476. }
  1477. devices.nrDeletedDevices++
  1478. return nil
  1479. }
  1480. // Should be called with devices.Lock() held.
  1481. func (devices *DeviceSet) deleteTransaction(info *devInfo, syncDelete bool) error {
  1482. if err := devices.openTransaction(info.Hash, info.DeviceID); err != nil {
  1483. logrus.Debugf("devmapper: Error opening transaction hash = %s deviceId = %d", "", info.DeviceID)
  1484. return err
  1485. }
  1486. defer devices.closeTransaction()
  1487. err := devicemapper.DeleteDevice(devices.getPoolDevName(), info.DeviceID)
  1488. if err != nil {
  1489. // If syncDelete is true, we want to return error. If deferred
  1490. // deletion is not enabled, we return an error. If error is
  1491. // something other then EBUSY, return an error.
  1492. if syncDelete || !devices.deferredDelete || err != devicemapper.ErrBusy {
  1493. logrus.Debugf("devmapper: Error deleting device: %s", err)
  1494. return err
  1495. }
  1496. }
  1497. if err == nil {
  1498. if err := devices.unregisterDevice(info.DeviceID, info.Hash); err != nil {
  1499. return err
  1500. }
  1501. // If device was already in deferred delete state that means
  1502. // deletion was being tried again later. Reduce the deleted
  1503. // device count.
  1504. if info.Deleted {
  1505. devices.nrDeletedDevices--
  1506. }
  1507. } else {
  1508. if err := devices.markForDeferredDeletion(info); err != nil {
  1509. return err
  1510. }
  1511. }
  1512. return nil
  1513. }
  1514. // Issue discard only if device open count is zero.
  1515. func (devices *DeviceSet) issueDiscard(info *devInfo) error {
  1516. logrus.Debugf("devmapper: issueDiscard(device: %s). START", info.Hash)
  1517. defer logrus.Debugf("devmapper: issueDiscard(device: %s). END", info.Hash)
  1518. // This is a workaround for the kernel not discarding block so
  1519. // on the thin pool when we remove a thinp device, so we do it
  1520. // manually.
  1521. // Even if device is deferred deleted, activate it and issue
  1522. // discards.
  1523. if err := devices.activateDeviceIfNeeded(info, true); err != nil {
  1524. return err
  1525. }
  1526. devinfo, err := devicemapper.GetInfo(info.Name())
  1527. if err != nil {
  1528. return err
  1529. }
  1530. if devinfo.OpenCount != 0 {
  1531. logrus.Debugf("devmapper: Device: %s is in use. OpenCount=%d. Not issuing discards.", info.Hash, devinfo.OpenCount)
  1532. return nil
  1533. }
  1534. if err := devicemapper.BlockDeviceDiscard(info.DevName()); err != nil {
  1535. logrus.Debugf("devmapper: Error discarding block on device: %s (ignoring)", err)
  1536. }
  1537. return nil
  1538. }
  1539. // Should be called with devices.Lock() held.
  1540. func (devices *DeviceSet) deleteDevice(info *devInfo, syncDelete bool) error {
  1541. if devices.doBlkDiscard {
  1542. devices.issueDiscard(info)
  1543. }
  1544. // Try to deactivate device in case it is active.
  1545. if err := devices.deactivateDevice(info); err != nil {
  1546. logrus.Debugf("devmapper: Error deactivating device: %s", err)
  1547. return err
  1548. }
  1549. if err := devices.deleteTransaction(info, syncDelete); err != nil {
  1550. return err
  1551. }
  1552. devices.markDeviceIDFree(info.DeviceID)
  1553. return nil
  1554. }
  1555. // DeleteDevice will return success if device has been marked for deferred
  1556. // removal. If one wants to override that and want DeleteDevice() to fail if
  1557. // device was busy and could not be deleted, set syncDelete=true.
  1558. func (devices *DeviceSet) DeleteDevice(hash string, syncDelete bool) error {
  1559. logrus.Debugf("devmapper: DeleteDevice(hash=%v syncDelete=%v) START", hash, syncDelete)
  1560. defer logrus.Debugf("devmapper: DeleteDevice(hash=%v syncDelete=%v) END", hash, syncDelete)
  1561. info, err := devices.lookupDeviceWithLock(hash)
  1562. if err != nil {
  1563. return err
  1564. }
  1565. info.lock.Lock()
  1566. defer info.lock.Unlock()
  1567. devices.Lock()
  1568. defer devices.Unlock()
  1569. // If mountcount is not zero, that means devices is still in use
  1570. // or has not been Put() properly. Fail device deletion.
  1571. if info.mountCount != 0 {
  1572. return fmt.Errorf("devmapper: Can't delete device %v as it is still mounted. mntCount=%v", info.Hash, info.mountCount)
  1573. }
  1574. return devices.deleteDevice(info, syncDelete)
  1575. }
  1576. func (devices *DeviceSet) deactivatePool() error {
  1577. logrus.Debugf("devmapper: deactivatePool()")
  1578. defer logrus.Debugf("devmapper: deactivatePool END")
  1579. devname := devices.getPoolDevName()
  1580. devinfo, err := devicemapper.GetInfo(devname)
  1581. if err != nil {
  1582. return err
  1583. }
  1584. if devinfo.Exists == 0 {
  1585. return nil
  1586. }
  1587. if err := devicemapper.RemoveDevice(devname); err != nil {
  1588. return err
  1589. }
  1590. if d, err := devicemapper.GetDeps(devname); err == nil {
  1591. logrus.Warnf("devmapper: device %s still has %d active dependents", devname, d.Count)
  1592. }
  1593. return nil
  1594. }
  1595. func (devices *DeviceSet) deactivateDevice(info *devInfo) error {
  1596. logrus.Debugf("devmapper: deactivateDevice(%s)", info.Hash)
  1597. defer logrus.Debugf("devmapper: deactivateDevice END(%s)", info.Hash)
  1598. devinfo, err := devicemapper.GetInfo(info.Name())
  1599. if err != nil {
  1600. return err
  1601. }
  1602. if devinfo.Exists == 0 {
  1603. return nil
  1604. }
  1605. if devices.deferredRemove {
  1606. if err := devicemapper.RemoveDeviceDeferred(info.Name()); err != nil {
  1607. return err
  1608. }
  1609. } else {
  1610. if err := devices.removeDevice(info.Name()); err != nil {
  1611. return err
  1612. }
  1613. }
  1614. return nil
  1615. }
  1616. // Issues the underlying dm remove operation.
  1617. func (devices *DeviceSet) removeDevice(devname string) error {
  1618. var err error
  1619. logrus.Debugf("devmapper: removeDevice START(%s)", devname)
  1620. defer logrus.Debugf("devmapper: removeDevice END(%s)", devname)
  1621. for i := 0; i < 200; i++ {
  1622. err = devicemapper.RemoveDevice(devname)
  1623. if err == nil {
  1624. break
  1625. }
  1626. if err != devicemapper.ErrBusy {
  1627. return err
  1628. }
  1629. // If we see EBUSY it may be a transient error,
  1630. // sleep a bit a retry a few times.
  1631. devices.Unlock()
  1632. time.Sleep(100 * time.Millisecond)
  1633. devices.Lock()
  1634. }
  1635. return err
  1636. }
  1637. func (devices *DeviceSet) cancelDeferredRemoval(info *devInfo) error {
  1638. if !devices.deferredRemove {
  1639. return nil
  1640. }
  1641. logrus.Debugf("devmapper: cancelDeferredRemoval START(%s)", info.Name())
  1642. defer logrus.Debugf("devmapper: cancelDeferredRemoval END(%s)", info.Name())
  1643. devinfo, err := devicemapper.GetInfoWithDeferred(info.Name())
  1644. if devinfo != nil && devinfo.DeferredRemove == 0 {
  1645. return nil
  1646. }
  1647. // Cancel deferred remove
  1648. for i := 0; i < 100; i++ {
  1649. err = devicemapper.CancelDeferredRemove(info.Name())
  1650. if err == nil {
  1651. break
  1652. }
  1653. if err == devicemapper.ErrEnxio {
  1654. // Device is probably already gone. Return success.
  1655. return nil
  1656. }
  1657. if err != devicemapper.ErrBusy {
  1658. return err
  1659. }
  1660. // If we see EBUSY it may be a transient error,
  1661. // sleep a bit a retry a few times.
  1662. devices.Unlock()
  1663. time.Sleep(100 * time.Millisecond)
  1664. devices.Lock()
  1665. }
  1666. return err
  1667. }
  1668. // Shutdown shuts down the device by unmounting the root.
  1669. func (devices *DeviceSet) Shutdown() error {
  1670. logrus.Debugf("devmapper: [deviceset %s] Shutdown()", devices.devicePrefix)
  1671. logrus.Debugf("devmapper: Shutting down DeviceSet: %s", devices.root)
  1672. defer logrus.Debugf("devmapper: [deviceset %s] Shutdown() END", devices.devicePrefix)
  1673. var devs []*devInfo
  1674. // Stop deletion worker. This should start delivering new events to
  1675. // ticker channel. That means no new instance of cleanupDeletedDevice()
  1676. // will run after this call. If one instance is already running at
  1677. // the time of the call, it must be holding devices.Lock() and
  1678. // we will block on this lock till cleanup function exits.
  1679. devices.deletionWorkerTicker.Stop()
  1680. devices.Lock()
  1681. // Save DeviceSet Metadata first. Docker kills all threads if they
  1682. // don't finish in certain time. It is possible that Shutdown()
  1683. // routine does not finish in time as we loop trying to deactivate
  1684. // some devices while these are busy. In that case shutdown() routine
  1685. // will be killed and we will not get a chance to save deviceset
  1686. // metadata. Hence save this early before trying to deactivate devices.
  1687. devices.saveDeviceSetMetaData()
  1688. for _, info := range devices.Devices {
  1689. devs = append(devs, info)
  1690. }
  1691. devices.Unlock()
  1692. for _, info := range devs {
  1693. info.lock.Lock()
  1694. if info.mountCount > 0 {
  1695. // We use MNT_DETACH here in case it is still busy in some running
  1696. // container. This means it'll go away from the global scope directly,
  1697. // and the device will be released when that container dies.
  1698. if err := syscall.Unmount(info.mountPath, syscall.MNT_DETACH); err != nil {
  1699. logrus.Debugf("devmapper: Shutdown unmounting %s, error: %s", info.mountPath, err)
  1700. }
  1701. devices.Lock()
  1702. if err := devices.deactivateDevice(info); err != nil {
  1703. logrus.Debugf("devmapper: Shutdown deactivate %s , error: %s", info.Hash, err)
  1704. }
  1705. devices.Unlock()
  1706. }
  1707. info.lock.Unlock()
  1708. }
  1709. info, _ := devices.lookupDeviceWithLock("")
  1710. if info != nil {
  1711. info.lock.Lock()
  1712. devices.Lock()
  1713. if err := devices.deactivateDevice(info); err != nil {
  1714. logrus.Debugf("devmapper: Shutdown deactivate base , error: %s", err)
  1715. }
  1716. devices.Unlock()
  1717. info.lock.Unlock()
  1718. }
  1719. devices.Lock()
  1720. if devices.thinPoolDevice == "" {
  1721. if err := devices.deactivatePool(); err != nil {
  1722. logrus.Debugf("devmapper: Shutdown deactivate pool , error: %s", err)
  1723. }
  1724. }
  1725. devices.Unlock()
  1726. return nil
  1727. }
  1728. // MountDevice mounts the device if not already mounted.
  1729. func (devices *DeviceSet) MountDevice(hash, path, mountLabel string) error {
  1730. info, err := devices.lookupDeviceWithLock(hash)
  1731. if err != nil {
  1732. return err
  1733. }
  1734. if info.Deleted {
  1735. return fmt.Errorf("devmapper: Can't mount device %v as it has been marked for deferred deletion", info.Hash)
  1736. }
  1737. info.lock.Lock()
  1738. defer info.lock.Unlock()
  1739. devices.Lock()
  1740. defer devices.Unlock()
  1741. if info.mountCount > 0 {
  1742. if path != info.mountPath {
  1743. return fmt.Errorf("devmapper: Trying to mount devmapper device in multiple places (%s, %s)", info.mountPath, path)
  1744. }
  1745. info.mountCount++
  1746. return nil
  1747. }
  1748. if err := devices.activateDeviceIfNeeded(info, false); err != nil {
  1749. return fmt.Errorf("devmapper: Error activating devmapper device for '%s': %s", hash, err)
  1750. }
  1751. fstype, err := ProbeFsType(info.DevName())
  1752. if err != nil {
  1753. return err
  1754. }
  1755. options := ""
  1756. if fstype == "xfs" {
  1757. // XFS needs nouuid or it can't mount filesystems with the same fs
  1758. options = joinMountOptions(options, "nouuid")
  1759. }
  1760. options = joinMountOptions(options, devices.mountOptions)
  1761. options = joinMountOptions(options, label.FormatMountLabel("", mountLabel))
  1762. if err := mount.Mount(info.DevName(), path, fstype, options); err != nil {
  1763. return fmt.Errorf("devmapper: Error mounting '%s' on '%s': %s", info.DevName(), path, err)
  1764. }
  1765. info.mountCount = 1
  1766. info.mountPath = path
  1767. return nil
  1768. }
  1769. // UnmountDevice unmounts the device and removes it from hash.
  1770. func (devices *DeviceSet) UnmountDevice(hash, mountPath string) error {
  1771. logrus.Debugf("devmapper: UnmountDevice(hash=%s)", hash)
  1772. defer logrus.Debugf("devmapper: UnmountDevice(hash=%s) END", hash)
  1773. info, err := devices.lookupDeviceWithLock(hash)
  1774. if err != nil {
  1775. return err
  1776. }
  1777. info.lock.Lock()
  1778. defer info.lock.Unlock()
  1779. devices.Lock()
  1780. defer devices.Unlock()
  1781. // If there are running containers when daemon crashes, during daemon
  1782. // restarting, it will kill running containers and will finally call
  1783. // Put() without calling Get(). So info.MountCount may become negative.
  1784. // if info.mountCount goes negative, we do the unmount and assign
  1785. // it to 0.
  1786. info.mountCount--
  1787. if info.mountCount > 0 {
  1788. return nil
  1789. } else if info.mountCount < 0 {
  1790. logrus.Warnf("devmapper: Mount count of device went negative. Put() called without matching Get(). Resetting count to 0")
  1791. info.mountCount = 0
  1792. }
  1793. logrus.Debugf("devmapper: Unmount(%s)", mountPath)
  1794. if err := syscall.Unmount(mountPath, syscall.MNT_DETACH); err != nil {
  1795. return err
  1796. }
  1797. logrus.Debugf("devmapper: Unmount done")
  1798. if err := devices.deactivateDevice(info); err != nil {
  1799. return err
  1800. }
  1801. info.mountPath = ""
  1802. return nil
  1803. }
  1804. // HasDevice returns true if the device metadata exists.
  1805. func (devices *DeviceSet) HasDevice(hash string) bool {
  1806. info, _ := devices.lookupDeviceWithLock(hash)
  1807. return info != nil
  1808. }
  1809. // List returns a list of device ids.
  1810. func (devices *DeviceSet) List() []string {
  1811. devices.Lock()
  1812. defer devices.Unlock()
  1813. ids := make([]string, len(devices.Devices))
  1814. i := 0
  1815. for k := range devices.Devices {
  1816. ids[i] = k
  1817. i++
  1818. }
  1819. return ids
  1820. }
  1821. func (devices *DeviceSet) deviceStatus(devName string) (sizeInSectors, mappedSectors, highestMappedSector uint64, err error) {
  1822. var params string
  1823. _, sizeInSectors, _, params, err = devicemapper.GetStatus(devName)
  1824. if err != nil {
  1825. return
  1826. }
  1827. if _, err = fmt.Sscanf(params, "%d %d", &mappedSectors, &highestMappedSector); err == nil {
  1828. return
  1829. }
  1830. return
  1831. }
  1832. // GetDeviceStatus provides size, mapped sectors
  1833. func (devices *DeviceSet) GetDeviceStatus(hash string) (*DevStatus, error) {
  1834. info, err := devices.lookupDeviceWithLock(hash)
  1835. if err != nil {
  1836. return nil, err
  1837. }
  1838. info.lock.Lock()
  1839. defer info.lock.Unlock()
  1840. devices.Lock()
  1841. defer devices.Unlock()
  1842. status := &DevStatus{
  1843. DeviceID: info.DeviceID,
  1844. Size: info.Size,
  1845. TransactionID: info.TransactionID,
  1846. }
  1847. if err := devices.activateDeviceIfNeeded(info, false); err != nil {
  1848. return nil, fmt.Errorf("devmapper: Error activating devmapper device for '%s': %s", hash, err)
  1849. }
  1850. sizeInSectors, mappedSectors, highestMappedSector, err := devices.deviceStatus(info.DevName())
  1851. if err != nil {
  1852. return nil, err
  1853. }
  1854. status.SizeInSectors = sizeInSectors
  1855. status.MappedSectors = mappedSectors
  1856. status.HighestMappedSector = highestMappedSector
  1857. return status, nil
  1858. }
  1859. func (devices *DeviceSet) poolStatus() (totalSizeInSectors, transactionID, dataUsed, dataTotal, metadataUsed, metadataTotal uint64, err error) {
  1860. var params string
  1861. if _, totalSizeInSectors, _, params, err = devicemapper.GetStatus(devices.getPoolName()); err == nil {
  1862. _, err = fmt.Sscanf(params, "%d %d/%d %d/%d", &transactionID, &metadataUsed, &metadataTotal, &dataUsed, &dataTotal)
  1863. }
  1864. return
  1865. }
  1866. // DataDevicePath returns the path to the data storage for this deviceset,
  1867. // regardless of loopback or block device
  1868. func (devices *DeviceSet) DataDevicePath() string {
  1869. return devices.dataDevice
  1870. }
  1871. // MetadataDevicePath returns the path to the metadata storage for this deviceset,
  1872. // regardless of loopback or block device
  1873. func (devices *DeviceSet) MetadataDevicePath() string {
  1874. return devices.metadataDevice
  1875. }
  1876. func (devices *DeviceSet) getUnderlyingAvailableSpace(loopFile string) (uint64, error) {
  1877. buf := new(syscall.Statfs_t)
  1878. if err := syscall.Statfs(loopFile, buf); err != nil {
  1879. logrus.Warnf("devmapper: Couldn't stat loopfile filesystem %v: %v", loopFile, err)
  1880. return 0, err
  1881. }
  1882. return buf.Bfree * uint64(buf.Bsize), nil
  1883. }
  1884. func (devices *DeviceSet) isRealFile(loopFile string) (bool, error) {
  1885. if loopFile != "" {
  1886. fi, err := os.Stat(loopFile)
  1887. if err != nil {
  1888. logrus.Warnf("devmapper: Couldn't stat loopfile %v: %v", loopFile, err)
  1889. return false, err
  1890. }
  1891. return fi.Mode().IsRegular(), nil
  1892. }
  1893. return false, nil
  1894. }
  1895. // Status returns the current status of this deviceset
  1896. func (devices *DeviceSet) Status() *Status {
  1897. devices.Lock()
  1898. defer devices.Unlock()
  1899. status := &Status{}
  1900. status.PoolName = devices.getPoolName()
  1901. status.DataFile = devices.DataDevicePath()
  1902. status.DataLoopback = devices.dataLoopFile
  1903. status.MetadataFile = devices.MetadataDevicePath()
  1904. status.MetadataLoopback = devices.metadataLoopFile
  1905. status.UdevSyncSupported = devicemapper.UdevSyncSupported()
  1906. status.DeferredRemoveEnabled = devices.deferredRemove
  1907. status.DeferredDeleteEnabled = devices.deferredDelete
  1908. status.DeferredDeletedDeviceCount = devices.nrDeletedDevices
  1909. status.BaseDeviceSize = devices.getBaseDeviceSize()
  1910. status.BaseDeviceFS = devices.getBaseDeviceFS()
  1911. totalSizeInSectors, _, dataUsed, dataTotal, metadataUsed, metadataTotal, err := devices.poolStatus()
  1912. if err == nil {
  1913. // Convert from blocks to bytes
  1914. blockSizeInSectors := totalSizeInSectors / dataTotal
  1915. status.Data.Used = dataUsed * blockSizeInSectors * 512
  1916. status.Data.Total = dataTotal * blockSizeInSectors * 512
  1917. status.Data.Available = status.Data.Total - status.Data.Used
  1918. // metadata blocks are always 4k
  1919. status.Metadata.Used = metadataUsed * 4096
  1920. status.Metadata.Total = metadataTotal * 4096
  1921. status.Metadata.Available = status.Metadata.Total - status.Metadata.Used
  1922. status.SectorSize = blockSizeInSectors * 512
  1923. if check, _ := devices.isRealFile(devices.dataLoopFile); check {
  1924. actualSpace, err := devices.getUnderlyingAvailableSpace(devices.dataLoopFile)
  1925. if err == nil && actualSpace < status.Data.Available {
  1926. status.Data.Available = actualSpace
  1927. }
  1928. }
  1929. if check, _ := devices.isRealFile(devices.metadataLoopFile); check {
  1930. actualSpace, err := devices.getUnderlyingAvailableSpace(devices.metadataLoopFile)
  1931. if err == nil && actualSpace < status.Metadata.Available {
  1932. status.Metadata.Available = actualSpace
  1933. }
  1934. }
  1935. }
  1936. return status
  1937. }
  1938. // Status returns the current status of this deviceset
  1939. func (devices *DeviceSet) exportDeviceMetadata(hash string) (*deviceMetadata, error) {
  1940. info, err := devices.lookupDeviceWithLock(hash)
  1941. if err != nil {
  1942. return nil, err
  1943. }
  1944. info.lock.Lock()
  1945. defer info.lock.Unlock()
  1946. metadata := &deviceMetadata{info.DeviceID, info.Size, info.Name()}
  1947. return metadata, nil
  1948. }
  1949. // NewDeviceSet creates the device set based on the options provided.
  1950. func NewDeviceSet(root string, doInit bool, options []string, uidMaps, gidMaps []idtools.IDMap) (*DeviceSet, error) {
  1951. devicemapper.SetDevDir("/dev")
  1952. devices := &DeviceSet{
  1953. root: root,
  1954. metaData: metaData{Devices: make(map[string]*devInfo)},
  1955. dataLoopbackSize: defaultDataLoopbackSize,
  1956. metaDataLoopbackSize: defaultMetaDataLoopbackSize,
  1957. baseFsSize: defaultBaseFsSize,
  1958. overrideUdevSyncCheck: defaultUdevSyncOverride,
  1959. doBlkDiscard: true,
  1960. thinpBlockSize: defaultThinpBlockSize,
  1961. deviceIDMap: make([]byte, deviceIDMapSz),
  1962. deletionWorkerTicker: time.NewTicker(time.Second * 30),
  1963. uidMaps: uidMaps,
  1964. gidMaps: gidMaps,
  1965. }
  1966. foundBlkDiscard := false
  1967. for _, option := range options {
  1968. key, val, err := parsers.ParseKeyValueOpt(option)
  1969. if err != nil {
  1970. return nil, err
  1971. }
  1972. key = strings.ToLower(key)
  1973. switch key {
  1974. case "dm.basesize":
  1975. size, err := units.RAMInBytes(val)
  1976. if err != nil {
  1977. return nil, err
  1978. }
  1979. devices.baseFsSize = uint64(size)
  1980. case "dm.loopdatasize":
  1981. size, err := units.RAMInBytes(val)
  1982. if err != nil {
  1983. return nil, err
  1984. }
  1985. devices.dataLoopbackSize = size
  1986. case "dm.loopmetadatasize":
  1987. size, err := units.RAMInBytes(val)
  1988. if err != nil {
  1989. return nil, err
  1990. }
  1991. devices.metaDataLoopbackSize = size
  1992. case "dm.fs":
  1993. if val != "ext4" && val != "xfs" {
  1994. return nil, fmt.Errorf("devmapper: Unsupported filesystem %s\n", val)
  1995. }
  1996. devices.filesystem = val
  1997. case "dm.mkfsarg":
  1998. devices.mkfsArgs = append(devices.mkfsArgs, val)
  1999. case "dm.mountopt":
  2000. devices.mountOptions = joinMountOptions(devices.mountOptions, val)
  2001. case "dm.metadatadev":
  2002. devices.metadataDevice = val
  2003. case "dm.datadev":
  2004. devices.dataDevice = val
  2005. case "dm.thinpooldev":
  2006. devices.thinPoolDevice = strings.TrimPrefix(val, "/dev/mapper/")
  2007. case "dm.blkdiscard":
  2008. foundBlkDiscard = true
  2009. devices.doBlkDiscard, err = strconv.ParseBool(val)
  2010. if err != nil {
  2011. return nil, err
  2012. }
  2013. case "dm.blocksize":
  2014. size, err := units.RAMInBytes(val)
  2015. if err != nil {
  2016. return nil, err
  2017. }
  2018. // convert to 512b sectors
  2019. devices.thinpBlockSize = uint32(size) >> 9
  2020. case "dm.override_udev_sync_check":
  2021. devices.overrideUdevSyncCheck, err = strconv.ParseBool(val)
  2022. if err != nil {
  2023. return nil, err
  2024. }
  2025. case "dm.use_deferred_removal":
  2026. enableDeferredRemoval, err = strconv.ParseBool(val)
  2027. if err != nil {
  2028. return nil, err
  2029. }
  2030. case "dm.use_deferred_deletion":
  2031. enableDeferredDeletion, err = strconv.ParseBool(val)
  2032. if err != nil {
  2033. return nil, err
  2034. }
  2035. default:
  2036. return nil, fmt.Errorf("devmapper: Unknown option %s\n", key)
  2037. }
  2038. }
  2039. // By default, don't do blk discard hack on raw devices, its rarely useful and is expensive
  2040. if !foundBlkDiscard && (devices.dataDevice != "" || devices.thinPoolDevice != "") {
  2041. devices.doBlkDiscard = false
  2042. }
  2043. if err := devices.initDevmapper(doInit); err != nil {
  2044. return nil, err
  2045. }
  2046. return devices, nil
  2047. }