deviceset.go 82 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806
  1. // +build linux
  2. package devmapper
  3. import (
  4. "bufio"
  5. "encoding/json"
  6. "fmt"
  7. "io"
  8. "io/ioutil"
  9. "os"
  10. "os/exec"
  11. "path"
  12. "path/filepath"
  13. "reflect"
  14. "strconv"
  15. "strings"
  16. "sync"
  17. "syscall"
  18. "time"
  19. "github.com/Sirupsen/logrus"
  20. "github.com/docker/docker/daemon/graphdriver"
  21. "github.com/docker/docker/dockerversion"
  22. "github.com/docker/docker/pkg/devicemapper"
  23. "github.com/docker/docker/pkg/idtools"
  24. "github.com/docker/docker/pkg/loopback"
  25. "github.com/docker/docker/pkg/mount"
  26. "github.com/docker/docker/pkg/parsers"
  27. units "github.com/docker/go-units"
  28. "github.com/pkg/errors"
  29. "github.com/opencontainers/selinux/go-selinux/label"
  30. )
  31. var (
  32. defaultDataLoopbackSize int64 = 100 * 1024 * 1024 * 1024
  33. defaultMetaDataLoopbackSize int64 = 2 * 1024 * 1024 * 1024
  34. defaultBaseFsSize uint64 = 10 * 1024 * 1024 * 1024
  35. defaultThinpBlockSize uint32 = 128 // 64K = 128 512b sectors
  36. defaultUdevSyncOverride = false
  37. maxDeviceID = 0xffffff // 24 bit, pool limit
  38. deviceIDMapSz = (maxDeviceID + 1) / 8
  39. // We retry device removal so many a times that even error messages
  40. // will fill up console during normal operation. So only log Fatal
  41. // messages by default.
  42. logLevel = devicemapper.LogLevelFatal
  43. driverDeferredRemovalSupport = false
  44. enableDeferredRemoval = false
  45. enableDeferredDeletion = false
  46. userBaseSize = false
  47. defaultMinFreeSpacePercent uint32 = 10
  48. lvmSetupConfigForce bool
  49. )
  50. const deviceSetMetaFile string = "deviceset-metadata"
  51. const transactionMetaFile string = "transaction-metadata"
  52. type transaction struct {
  53. OpenTransactionID uint64 `json:"open_transaction_id"`
  54. DeviceIDHash string `json:"device_hash"`
  55. DeviceID int `json:"device_id"`
  56. }
  57. type devInfo struct {
  58. Hash string `json:"-"`
  59. DeviceID int `json:"device_id"`
  60. Size uint64 `json:"size"`
  61. TransactionID uint64 `json:"transaction_id"`
  62. Initialized bool `json:"initialized"`
  63. Deleted bool `json:"deleted"`
  64. devices *DeviceSet
  65. // The global DeviceSet lock guarantees that we serialize all
  66. // the calls to libdevmapper (which is not threadsafe), but we
  67. // sometimes release that lock while sleeping. In that case
  68. // this per-device lock is still held, protecting against
  69. // other accesses to the device that we're doing the wait on.
  70. //
  71. // WARNING: In order to avoid AB-BA deadlocks when releasing
  72. // the global lock while holding the per-device locks all
  73. // device locks must be acquired *before* the device lock, and
  74. // multiple device locks should be acquired parent before child.
  75. lock sync.Mutex
  76. }
  77. type metaData struct {
  78. Devices map[string]*devInfo `json:"Devices"`
  79. }
  80. // DeviceSet holds information about list of devices
  81. type DeviceSet struct {
  82. metaData `json:"-"`
  83. sync.Mutex `json:"-"` // Protects all fields of DeviceSet and serializes calls into libdevmapper
  84. root string
  85. devicePrefix string
  86. TransactionID uint64 `json:"-"`
  87. NextDeviceID int `json:"next_device_id"`
  88. deviceIDMap []byte
  89. // Options
  90. dataLoopbackSize int64
  91. metaDataLoopbackSize int64
  92. baseFsSize uint64
  93. filesystem string
  94. mountOptions string
  95. mkfsArgs []string
  96. dataDevice string // block or loop dev
  97. dataLoopFile string // loopback file, if used
  98. metadataDevice string // block or loop dev
  99. metadataLoopFile string // loopback file, if used
  100. doBlkDiscard bool
  101. thinpBlockSize uint32
  102. thinPoolDevice string
  103. transaction `json:"-"`
  104. overrideUdevSyncCheck bool
  105. deferredRemove bool // use deferred removal
  106. deferredDelete bool // use deferred deletion
  107. BaseDeviceUUID string // save UUID of base device
  108. BaseDeviceFilesystem string // save filesystem of base device
  109. nrDeletedDevices uint // number of deleted devices
  110. deletionWorkerTicker *time.Ticker
  111. uidMaps []idtools.IDMap
  112. gidMaps []idtools.IDMap
  113. minFreeSpacePercent uint32 //min free space percentage in thinpool
  114. xfsNospaceRetries string // max retries when xfs receives ENOSPC
  115. lvmSetupConfig directLVMConfig
  116. }
  117. // DiskUsage contains information about disk usage and is used when reporting Status of a device.
  118. type DiskUsage struct {
  119. // Used bytes on the disk.
  120. Used uint64
  121. // Total bytes on the disk.
  122. Total uint64
  123. // Available bytes on the disk.
  124. Available uint64
  125. }
  126. // Status returns the information about the device.
  127. type Status struct {
  128. // PoolName is the name of the data pool.
  129. PoolName string
  130. // DataFile is the actual block device for data.
  131. DataFile string
  132. // DataLoopback loopback file, if used.
  133. DataLoopback string
  134. // MetadataFile is the actual block device for metadata.
  135. MetadataFile string
  136. // MetadataLoopback is the loopback file, if used.
  137. MetadataLoopback string
  138. // Data is the disk used for data.
  139. Data DiskUsage
  140. // Metadata is the disk used for meta data.
  141. Metadata DiskUsage
  142. // BaseDeviceSize is base size of container and image
  143. BaseDeviceSize uint64
  144. // BaseDeviceFS is backing filesystem.
  145. BaseDeviceFS string
  146. // SectorSize size of the vector.
  147. SectorSize uint64
  148. // UdevSyncSupported is true if sync is supported.
  149. UdevSyncSupported bool
  150. // DeferredRemoveEnabled is true then the device is not unmounted.
  151. DeferredRemoveEnabled bool
  152. // True if deferred deletion is enabled. This is different from
  153. // deferred removal. "removal" means that device mapper device is
  154. // deactivated. Thin device is still in thin pool and can be activated
  155. // again. But "deletion" means that thin device will be deleted from
  156. // thin pool and it can't be activated again.
  157. DeferredDeleteEnabled bool
  158. DeferredDeletedDeviceCount uint
  159. MinFreeSpace uint64
  160. }
  161. // Structure used to export image/container metadata in docker inspect.
  162. type deviceMetadata struct {
  163. deviceID int
  164. deviceSize uint64 // size in bytes
  165. deviceName string // Device name as used during activation
  166. }
  167. // DevStatus returns information about device mounted containing its id, size and sector information.
  168. type DevStatus struct {
  169. // DeviceID is the id of the device.
  170. DeviceID int
  171. // Size is the size of the filesystem.
  172. Size uint64
  173. // TransactionID is a unique integer per device set used to identify an operation on the file system, this number is incremental.
  174. TransactionID uint64
  175. // SizeInSectors indicates the size of the sectors allocated.
  176. SizeInSectors uint64
  177. // MappedSectors indicates number of mapped sectors.
  178. MappedSectors uint64
  179. // HighestMappedSector is the pointer to the highest mapped sector.
  180. HighestMappedSector uint64
  181. }
  182. func getDevName(name string) string {
  183. return "/dev/mapper/" + name
  184. }
  185. func (info *devInfo) Name() string {
  186. hash := info.Hash
  187. if hash == "" {
  188. hash = "base"
  189. }
  190. return fmt.Sprintf("%s-%s", info.devices.devicePrefix, hash)
  191. }
  192. func (info *devInfo) DevName() string {
  193. return getDevName(info.Name())
  194. }
  195. func (devices *DeviceSet) loopbackDir() string {
  196. return path.Join(devices.root, "devicemapper")
  197. }
  198. func (devices *DeviceSet) metadataDir() string {
  199. return path.Join(devices.root, "metadata")
  200. }
  201. func (devices *DeviceSet) metadataFile(info *devInfo) string {
  202. file := info.Hash
  203. if file == "" {
  204. file = "base"
  205. }
  206. return path.Join(devices.metadataDir(), file)
  207. }
  208. func (devices *DeviceSet) transactionMetaFile() string {
  209. return path.Join(devices.metadataDir(), transactionMetaFile)
  210. }
  211. func (devices *DeviceSet) deviceSetMetaFile() string {
  212. return path.Join(devices.metadataDir(), deviceSetMetaFile)
  213. }
  214. func (devices *DeviceSet) oldMetadataFile() string {
  215. return path.Join(devices.loopbackDir(), "json")
  216. }
  217. func (devices *DeviceSet) getPoolName() string {
  218. if devices.thinPoolDevice == "" {
  219. return devices.devicePrefix + "-pool"
  220. }
  221. return devices.thinPoolDevice
  222. }
  223. func (devices *DeviceSet) getPoolDevName() string {
  224. return getDevName(devices.getPoolName())
  225. }
  226. func (devices *DeviceSet) hasImage(name string) bool {
  227. dirname := devices.loopbackDir()
  228. filename := path.Join(dirname, name)
  229. _, err := os.Stat(filename)
  230. return err == nil
  231. }
  232. // ensureImage creates a sparse file of <size> bytes at the path
  233. // <root>/devicemapper/<name>.
  234. // If the file already exists and new size is larger than its current size, it grows to the new size.
  235. // Either way it returns the full path.
  236. func (devices *DeviceSet) ensureImage(name string, size int64) (string, error) {
  237. dirname := devices.loopbackDir()
  238. filename := path.Join(dirname, name)
  239. uid, gid, err := idtools.GetRootUIDGID(devices.uidMaps, devices.gidMaps)
  240. if err != nil {
  241. return "", err
  242. }
  243. if err := idtools.MkdirAllAs(dirname, 0700, uid, gid); err != nil && !os.IsExist(err) {
  244. return "", err
  245. }
  246. if fi, err := os.Stat(filename); err != nil {
  247. if !os.IsNotExist(err) {
  248. return "", err
  249. }
  250. logrus.Debugf("devmapper: Creating loopback file %s for device-manage use", filename)
  251. file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0600)
  252. if err != nil {
  253. return "", err
  254. }
  255. defer file.Close()
  256. if err := file.Truncate(size); err != nil {
  257. return "", err
  258. }
  259. } else {
  260. if fi.Size() < size {
  261. file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0600)
  262. if err != nil {
  263. return "", err
  264. }
  265. defer file.Close()
  266. if err := file.Truncate(size); err != nil {
  267. return "", fmt.Errorf("devmapper: Unable to grow loopback file %s: %v", filename, err)
  268. }
  269. } else if fi.Size() > size {
  270. logrus.Warnf("devmapper: Can't shrink loopback file %s", filename)
  271. }
  272. }
  273. return filename, nil
  274. }
  275. func (devices *DeviceSet) allocateTransactionID() uint64 {
  276. devices.OpenTransactionID = devices.TransactionID + 1
  277. return devices.OpenTransactionID
  278. }
  279. func (devices *DeviceSet) updatePoolTransactionID() error {
  280. if err := devicemapper.SetTransactionID(devices.getPoolDevName(), devices.TransactionID, devices.OpenTransactionID); err != nil {
  281. return fmt.Errorf("devmapper: Error setting devmapper transaction ID: %s", err)
  282. }
  283. devices.TransactionID = devices.OpenTransactionID
  284. return nil
  285. }
  286. func (devices *DeviceSet) removeMetadata(info *devInfo) error {
  287. if err := os.RemoveAll(devices.metadataFile(info)); err != nil {
  288. return fmt.Errorf("devmapper: Error removing metadata file %s: %s", devices.metadataFile(info), err)
  289. }
  290. return nil
  291. }
  292. // Given json data and file path, write it to disk
  293. func (devices *DeviceSet) writeMetaFile(jsonData []byte, filePath string) error {
  294. tmpFile, err := ioutil.TempFile(devices.metadataDir(), ".tmp")
  295. if err != nil {
  296. return fmt.Errorf("devmapper: Error creating metadata file: %s", err)
  297. }
  298. n, err := tmpFile.Write(jsonData)
  299. if err != nil {
  300. return fmt.Errorf("devmapper: Error writing metadata to %s: %s", tmpFile.Name(), err)
  301. }
  302. if n < len(jsonData) {
  303. return io.ErrShortWrite
  304. }
  305. if err := tmpFile.Sync(); err != nil {
  306. return fmt.Errorf("devmapper: Error syncing metadata file %s: %s", tmpFile.Name(), err)
  307. }
  308. if err := tmpFile.Close(); err != nil {
  309. return fmt.Errorf("devmapper: Error closing metadata file %s: %s", tmpFile.Name(), err)
  310. }
  311. if err := os.Rename(tmpFile.Name(), filePath); err != nil {
  312. return fmt.Errorf("devmapper: Error committing metadata file %s: %s", tmpFile.Name(), err)
  313. }
  314. return nil
  315. }
  316. func (devices *DeviceSet) saveMetadata(info *devInfo) error {
  317. jsonData, err := json.Marshal(info)
  318. if err != nil {
  319. return fmt.Errorf("devmapper: Error encoding metadata to json: %s", err)
  320. }
  321. if err := devices.writeMetaFile(jsonData, devices.metadataFile(info)); err != nil {
  322. return err
  323. }
  324. return nil
  325. }
  326. func (devices *DeviceSet) markDeviceIDUsed(deviceID int) {
  327. var mask byte
  328. i := deviceID % 8
  329. mask = 1 << uint(i)
  330. devices.deviceIDMap[deviceID/8] = devices.deviceIDMap[deviceID/8] | mask
  331. }
  332. func (devices *DeviceSet) markDeviceIDFree(deviceID int) {
  333. var mask byte
  334. i := deviceID % 8
  335. mask = ^(1 << uint(i))
  336. devices.deviceIDMap[deviceID/8] = devices.deviceIDMap[deviceID/8] & mask
  337. }
  338. func (devices *DeviceSet) isDeviceIDFree(deviceID int) bool {
  339. var mask byte
  340. i := deviceID % 8
  341. mask = (1 << uint(i))
  342. return (devices.deviceIDMap[deviceID/8] & mask) == 0
  343. }
  344. // Should be called with devices.Lock() held.
  345. func (devices *DeviceSet) lookupDevice(hash string) (*devInfo, error) {
  346. info := devices.Devices[hash]
  347. if info == nil {
  348. info = devices.loadMetadata(hash)
  349. if info == nil {
  350. return nil, fmt.Errorf("devmapper: Unknown device %s", hash)
  351. }
  352. devices.Devices[hash] = info
  353. }
  354. return info, nil
  355. }
  356. func (devices *DeviceSet) lookupDeviceWithLock(hash string) (*devInfo, error) {
  357. devices.Lock()
  358. defer devices.Unlock()
  359. info, err := devices.lookupDevice(hash)
  360. return info, err
  361. }
  362. // This function relies on that device hash map has been loaded in advance.
  363. // Should be called with devices.Lock() held.
  364. func (devices *DeviceSet) constructDeviceIDMap() {
  365. logrus.Debug("devmapper: constructDeviceIDMap()")
  366. defer logrus.Debug("devmapper: constructDeviceIDMap() END")
  367. for _, info := range devices.Devices {
  368. devices.markDeviceIDUsed(info.DeviceID)
  369. logrus.Debugf("devmapper: Added deviceId=%d to DeviceIdMap", info.DeviceID)
  370. }
  371. }
  372. func (devices *DeviceSet) deviceFileWalkFunction(path string, finfo os.FileInfo) error {
  373. // Skip some of the meta files which are not device files.
  374. if strings.HasSuffix(finfo.Name(), ".migrated") {
  375. logrus.Debugf("devmapper: Skipping file %s", path)
  376. return nil
  377. }
  378. if strings.HasPrefix(finfo.Name(), ".") {
  379. logrus.Debugf("devmapper: Skipping file %s", path)
  380. return nil
  381. }
  382. if finfo.Name() == deviceSetMetaFile {
  383. logrus.Debugf("devmapper: Skipping file %s", path)
  384. return nil
  385. }
  386. if finfo.Name() == transactionMetaFile {
  387. logrus.Debugf("devmapper: Skipping file %s", path)
  388. return nil
  389. }
  390. logrus.Debugf("devmapper: Loading data for file %s", path)
  391. hash := finfo.Name()
  392. if hash == "base" {
  393. hash = ""
  394. }
  395. // Include deleted devices also as cleanup delete device logic
  396. // will go through it and see if there are any deleted devices.
  397. if _, err := devices.lookupDevice(hash); err != nil {
  398. return fmt.Errorf("devmapper: Error looking up device %s:%v", hash, err)
  399. }
  400. return nil
  401. }
  402. func (devices *DeviceSet) loadDeviceFilesOnStart() error {
  403. logrus.Debug("devmapper: loadDeviceFilesOnStart()")
  404. defer logrus.Debug("devmapper: loadDeviceFilesOnStart() END")
  405. var scan = func(path string, info os.FileInfo, err error) error {
  406. if err != nil {
  407. logrus.Debugf("devmapper: Can't walk the file %s", path)
  408. return nil
  409. }
  410. // Skip any directories
  411. if info.IsDir() {
  412. return nil
  413. }
  414. return devices.deviceFileWalkFunction(path, info)
  415. }
  416. return filepath.Walk(devices.metadataDir(), scan)
  417. }
  418. // Should be called with devices.Lock() held.
  419. func (devices *DeviceSet) unregisterDevice(hash string) error {
  420. logrus.Debugf("devmapper: unregisterDevice(%v)", hash)
  421. info := &devInfo{
  422. Hash: hash,
  423. }
  424. delete(devices.Devices, hash)
  425. if err := devices.removeMetadata(info); err != nil {
  426. logrus.Debugf("devmapper: Error removing metadata: %s", err)
  427. return err
  428. }
  429. return nil
  430. }
  431. // Should be called with devices.Lock() held.
  432. func (devices *DeviceSet) registerDevice(id int, hash string, size uint64, transactionID uint64) (*devInfo, error) {
  433. logrus.Debugf("devmapper: registerDevice(%v, %v)", id, hash)
  434. info := &devInfo{
  435. Hash: hash,
  436. DeviceID: id,
  437. Size: size,
  438. TransactionID: transactionID,
  439. Initialized: false,
  440. devices: devices,
  441. }
  442. devices.Devices[hash] = info
  443. if err := devices.saveMetadata(info); err != nil {
  444. // Try to remove unused device
  445. delete(devices.Devices, hash)
  446. return nil, err
  447. }
  448. return info, nil
  449. }
  450. func (devices *DeviceSet) activateDeviceIfNeeded(info *devInfo, ignoreDeleted bool) error {
  451. logrus.Debugf("devmapper: activateDeviceIfNeeded(%v)", info.Hash)
  452. if info.Deleted && !ignoreDeleted {
  453. return fmt.Errorf("devmapper: Can't activate device %v as it is marked for deletion", info.Hash)
  454. }
  455. // Make sure deferred removal on device is canceled, if one was
  456. // scheduled.
  457. if err := devices.cancelDeferredRemovalIfNeeded(info); err != nil {
  458. return fmt.Errorf("devmapper: Device Deferred Removal Cancellation Failed: %s", err)
  459. }
  460. if devinfo, _ := devicemapper.GetInfo(info.Name()); devinfo != nil && devinfo.Exists != 0 {
  461. return nil
  462. }
  463. return devicemapper.ActivateDevice(devices.getPoolDevName(), info.Name(), info.DeviceID, info.Size)
  464. }
  465. // Return true only if kernel supports xfs and mkfs.xfs is available
  466. func xfsSupported() bool {
  467. // Make sure mkfs.xfs is available
  468. if _, err := exec.LookPath("mkfs.xfs"); err != nil {
  469. return false
  470. }
  471. // Check if kernel supports xfs filesystem or not.
  472. exec.Command("modprobe", "xfs").Run()
  473. f, err := os.Open("/proc/filesystems")
  474. if err != nil {
  475. logrus.Warnf("devmapper: Could not check if xfs is supported: %v", err)
  476. return false
  477. }
  478. defer f.Close()
  479. s := bufio.NewScanner(f)
  480. for s.Scan() {
  481. if strings.HasSuffix(s.Text(), "\txfs") {
  482. return true
  483. }
  484. }
  485. if err := s.Err(); err != nil {
  486. logrus.Warnf("devmapper: Could not check if xfs is supported: %v", err)
  487. }
  488. return false
  489. }
  490. func determineDefaultFS() string {
  491. if xfsSupported() {
  492. return "xfs"
  493. }
  494. logrus.Warn("devmapper: XFS is not supported in your system. Either the kernel doesn't support it or mkfs.xfs is not in your PATH. Defaulting to ext4 filesystem")
  495. return "ext4"
  496. }
  497. func (devices *DeviceSet) createFilesystem(info *devInfo) (err error) {
  498. devname := info.DevName()
  499. args := []string{}
  500. args = append(args, devices.mkfsArgs...)
  501. args = append(args, devname)
  502. if devices.filesystem == "" {
  503. devices.filesystem = determineDefaultFS()
  504. }
  505. if err := devices.saveBaseDeviceFilesystem(devices.filesystem); err != nil {
  506. return err
  507. }
  508. logrus.Infof("devmapper: Creating filesystem %s on device %s", devices.filesystem, info.Name())
  509. defer func() {
  510. if err != nil {
  511. logrus.Infof("devmapper: Error while creating filesystem %s on device %s: %v", devices.filesystem, info.Name(), err)
  512. } else {
  513. logrus.Infof("devmapper: Successfully created filesystem %s on device %s", devices.filesystem, info.Name())
  514. }
  515. }()
  516. switch devices.filesystem {
  517. case "xfs":
  518. err = exec.Command("mkfs.xfs", args...).Run()
  519. case "ext4":
  520. err = exec.Command("mkfs.ext4", append([]string{"-E", "nodiscard,lazy_itable_init=0,lazy_journal_init=0"}, args...)...).Run()
  521. if err != nil {
  522. err = exec.Command("mkfs.ext4", append([]string{"-E", "nodiscard,lazy_itable_init=0"}, args...)...).Run()
  523. }
  524. if err != nil {
  525. return err
  526. }
  527. err = exec.Command("tune2fs", append([]string{"-c", "-1", "-i", "0"}, devname)...).Run()
  528. default:
  529. err = fmt.Errorf("devmapper: Unsupported filesystem type %s", devices.filesystem)
  530. }
  531. return
  532. }
  533. func (devices *DeviceSet) migrateOldMetaData() error {
  534. // Migrate old metadata file
  535. jsonData, err := ioutil.ReadFile(devices.oldMetadataFile())
  536. if err != nil && !os.IsNotExist(err) {
  537. return err
  538. }
  539. if jsonData != nil {
  540. m := metaData{Devices: make(map[string]*devInfo)}
  541. if err := json.Unmarshal(jsonData, &m); err != nil {
  542. return err
  543. }
  544. for hash, info := range m.Devices {
  545. info.Hash = hash
  546. devices.saveMetadata(info)
  547. }
  548. if err := os.Rename(devices.oldMetadataFile(), devices.oldMetadataFile()+".migrated"); err != nil {
  549. return err
  550. }
  551. }
  552. return nil
  553. }
  554. // Cleanup deleted devices. It assumes that all the devices have been
  555. // loaded in the hash table.
  556. func (devices *DeviceSet) cleanupDeletedDevices() error {
  557. devices.Lock()
  558. // If there are no deleted devices, there is nothing to do.
  559. if devices.nrDeletedDevices == 0 {
  560. devices.Unlock()
  561. return nil
  562. }
  563. var deletedDevices []*devInfo
  564. for _, info := range devices.Devices {
  565. if !info.Deleted {
  566. continue
  567. }
  568. logrus.Debugf("devmapper: Found deleted device %s.", info.Hash)
  569. deletedDevices = append(deletedDevices, info)
  570. }
  571. // Delete the deleted devices. DeleteDevice() first takes the info lock
  572. // and then devices.Lock(). So drop it to avoid deadlock.
  573. devices.Unlock()
  574. for _, info := range deletedDevices {
  575. // This will again try deferred deletion.
  576. if err := devices.DeleteDevice(info.Hash, false); err != nil {
  577. logrus.Warnf("devmapper: Deletion of device %s, device_id=%v failed:%v", info.Hash, info.DeviceID, err)
  578. }
  579. }
  580. return nil
  581. }
  582. func (devices *DeviceSet) countDeletedDevices() {
  583. for _, info := range devices.Devices {
  584. if !info.Deleted {
  585. continue
  586. }
  587. devices.nrDeletedDevices++
  588. }
  589. }
  590. func (devices *DeviceSet) startDeviceDeletionWorker() {
  591. // Deferred deletion is not enabled. Don't do anything.
  592. if !devices.deferredDelete {
  593. return
  594. }
  595. logrus.Debug("devmapper: Worker to cleanup deleted devices started")
  596. for range devices.deletionWorkerTicker.C {
  597. devices.cleanupDeletedDevices()
  598. }
  599. }
  600. func (devices *DeviceSet) initMetaData() error {
  601. devices.Lock()
  602. defer devices.Unlock()
  603. if err := devices.migrateOldMetaData(); err != nil {
  604. return err
  605. }
  606. _, transactionID, _, _, _, _, err := devices.poolStatus()
  607. if err != nil {
  608. return err
  609. }
  610. devices.TransactionID = transactionID
  611. if err := devices.loadDeviceFilesOnStart(); err != nil {
  612. return fmt.Errorf("devmapper: Failed to load device files:%v", err)
  613. }
  614. devices.constructDeviceIDMap()
  615. devices.countDeletedDevices()
  616. if err := devices.processPendingTransaction(); err != nil {
  617. return err
  618. }
  619. // Start a goroutine to cleanup Deleted Devices
  620. go devices.startDeviceDeletionWorker()
  621. return nil
  622. }
  623. func (devices *DeviceSet) incNextDeviceID() {
  624. // IDs are 24bit, so wrap around
  625. devices.NextDeviceID = (devices.NextDeviceID + 1) & maxDeviceID
  626. }
  627. func (devices *DeviceSet) getNextFreeDeviceID() (int, error) {
  628. devices.incNextDeviceID()
  629. for i := 0; i <= maxDeviceID; i++ {
  630. if devices.isDeviceIDFree(devices.NextDeviceID) {
  631. devices.markDeviceIDUsed(devices.NextDeviceID)
  632. return devices.NextDeviceID, nil
  633. }
  634. devices.incNextDeviceID()
  635. }
  636. return 0, fmt.Errorf("devmapper: Unable to find a free device ID")
  637. }
  638. func (devices *DeviceSet) poolHasFreeSpace() error {
  639. if devices.minFreeSpacePercent == 0 {
  640. return nil
  641. }
  642. _, _, dataUsed, dataTotal, metadataUsed, metadataTotal, err := devices.poolStatus()
  643. if err != nil {
  644. return err
  645. }
  646. minFreeData := (dataTotal * uint64(devices.minFreeSpacePercent)) / 100
  647. if minFreeData < 1 {
  648. minFreeData = 1
  649. }
  650. dataFree := dataTotal - dataUsed
  651. if dataFree < minFreeData {
  652. return fmt.Errorf("devmapper: Thin Pool has %v free data blocks which is less than minimum required %v free data blocks. Create more free space in thin pool or use dm.min_free_space option to change behavior", (dataTotal - dataUsed), minFreeData)
  653. }
  654. minFreeMetadata := (metadataTotal * uint64(devices.minFreeSpacePercent)) / 100
  655. if minFreeMetadata < 1 {
  656. minFreeMetadata = 1
  657. }
  658. metadataFree := metadataTotal - metadataUsed
  659. if metadataFree < minFreeMetadata {
  660. return fmt.Errorf("devmapper: Thin Pool has %v free metadata blocks which is less than minimum required %v free metadata blocks. Create more free metadata space in thin pool or use dm.min_free_space option to change behavior", (metadataTotal - metadataUsed), minFreeMetadata)
  661. }
  662. return nil
  663. }
  664. func (devices *DeviceSet) createRegisterDevice(hash string) (*devInfo, error) {
  665. devices.Lock()
  666. defer devices.Unlock()
  667. deviceID, err := devices.getNextFreeDeviceID()
  668. if err != nil {
  669. return nil, err
  670. }
  671. if err := devices.openTransaction(hash, deviceID); err != nil {
  672. logrus.Debugf("devmapper: Error opening transaction hash = %s deviceID = %d", hash, deviceID)
  673. devices.markDeviceIDFree(deviceID)
  674. return nil, err
  675. }
  676. for {
  677. if err := devicemapper.CreateDevice(devices.getPoolDevName(), deviceID); err != nil {
  678. if devicemapper.DeviceIDExists(err) {
  679. // Device ID already exists. This should not
  680. // happen. Now we have a mechanism to find
  681. // a free device ID. So something is not right.
  682. // Give a warning and continue.
  683. logrus.Errorf("devmapper: Device ID %d exists in pool but it is supposed to be unused", deviceID)
  684. deviceID, err = devices.getNextFreeDeviceID()
  685. if err != nil {
  686. return nil, err
  687. }
  688. // Save new device id into transaction
  689. devices.refreshTransaction(deviceID)
  690. continue
  691. }
  692. logrus.Debugf("devmapper: Error creating device: %s", err)
  693. devices.markDeviceIDFree(deviceID)
  694. return nil, err
  695. }
  696. break
  697. }
  698. logrus.Debugf("devmapper: Registering device (id %v) with FS size %v", deviceID, devices.baseFsSize)
  699. info, err := devices.registerDevice(deviceID, hash, devices.baseFsSize, devices.OpenTransactionID)
  700. if err != nil {
  701. _ = devicemapper.DeleteDevice(devices.getPoolDevName(), deviceID)
  702. devices.markDeviceIDFree(deviceID)
  703. return nil, err
  704. }
  705. if err := devices.closeTransaction(); err != nil {
  706. devices.unregisterDevice(hash)
  707. devicemapper.DeleteDevice(devices.getPoolDevName(), deviceID)
  708. devices.markDeviceIDFree(deviceID)
  709. return nil, err
  710. }
  711. return info, nil
  712. }
  713. func (devices *DeviceSet) takeSnapshot(hash string, baseInfo *devInfo, size uint64) error {
  714. var (
  715. devinfo *devicemapper.Info
  716. err error
  717. )
  718. if err = devices.poolHasFreeSpace(); err != nil {
  719. return err
  720. }
  721. if devices.deferredRemove {
  722. devinfo, err = devicemapper.GetInfoWithDeferred(baseInfo.Name())
  723. if err != nil {
  724. return err
  725. }
  726. if devinfo != nil && devinfo.DeferredRemove != 0 {
  727. err = devices.cancelDeferredRemoval(baseInfo)
  728. if err != nil {
  729. // If Error is ErrEnxio. Device is probably already gone. Continue.
  730. if err != devicemapper.ErrEnxio {
  731. return err
  732. }
  733. devinfo = nil
  734. } else {
  735. defer devices.deactivateDevice(baseInfo)
  736. }
  737. }
  738. } else {
  739. devinfo, err = devicemapper.GetInfo(baseInfo.Name())
  740. if err != nil {
  741. return err
  742. }
  743. }
  744. doSuspend := devinfo != nil && devinfo.Exists != 0
  745. if doSuspend {
  746. if err = devicemapper.SuspendDevice(baseInfo.Name()); err != nil {
  747. return err
  748. }
  749. defer devicemapper.ResumeDevice(baseInfo.Name())
  750. }
  751. if err = devices.createRegisterSnapDevice(hash, baseInfo, size); err != nil {
  752. return err
  753. }
  754. return nil
  755. }
  756. func (devices *DeviceSet) createRegisterSnapDevice(hash string, baseInfo *devInfo, size uint64) error {
  757. deviceID, err := devices.getNextFreeDeviceID()
  758. if err != nil {
  759. return err
  760. }
  761. if err := devices.openTransaction(hash, deviceID); err != nil {
  762. logrus.Debugf("devmapper: Error opening transaction hash = %s deviceID = %d", hash, deviceID)
  763. devices.markDeviceIDFree(deviceID)
  764. return err
  765. }
  766. for {
  767. if err := devicemapper.CreateSnapDeviceRaw(devices.getPoolDevName(), deviceID, baseInfo.DeviceID); err != nil {
  768. if devicemapper.DeviceIDExists(err) {
  769. // Device ID already exists. This should not
  770. // happen. Now we have a mechanism to find
  771. // a free device ID. So something is not right.
  772. // Give a warning and continue.
  773. logrus.Errorf("devmapper: Device ID %d exists in pool but it is supposed to be unused", deviceID)
  774. deviceID, err = devices.getNextFreeDeviceID()
  775. if err != nil {
  776. return err
  777. }
  778. // Save new device id into transaction
  779. devices.refreshTransaction(deviceID)
  780. continue
  781. }
  782. logrus.Debugf("devmapper: Error creating snap device: %s", err)
  783. devices.markDeviceIDFree(deviceID)
  784. return err
  785. }
  786. break
  787. }
  788. if _, err := devices.registerDevice(deviceID, hash, size, devices.OpenTransactionID); err != nil {
  789. devicemapper.DeleteDevice(devices.getPoolDevName(), deviceID)
  790. devices.markDeviceIDFree(deviceID)
  791. logrus.Debugf("devmapper: Error registering device: %s", err)
  792. return err
  793. }
  794. if err := devices.closeTransaction(); err != nil {
  795. devices.unregisterDevice(hash)
  796. devicemapper.DeleteDevice(devices.getPoolDevName(), deviceID)
  797. devices.markDeviceIDFree(deviceID)
  798. return err
  799. }
  800. return nil
  801. }
  802. func (devices *DeviceSet) loadMetadata(hash string) *devInfo {
  803. info := &devInfo{Hash: hash, devices: devices}
  804. jsonData, err := ioutil.ReadFile(devices.metadataFile(info))
  805. if err != nil {
  806. logrus.Debugf("devmapper: Failed to read %s with err: %v", devices.metadataFile(info), err)
  807. return nil
  808. }
  809. if err := json.Unmarshal(jsonData, &info); err != nil {
  810. logrus.Debugf("devmapper: Failed to unmarshal devInfo from %s with err: %v", devices.metadataFile(info), err)
  811. return nil
  812. }
  813. if info.DeviceID > maxDeviceID {
  814. logrus.Errorf("devmapper: Ignoring Invalid DeviceId=%d", info.DeviceID)
  815. return nil
  816. }
  817. return info
  818. }
  819. func getDeviceUUID(device string) (string, error) {
  820. out, err := exec.Command("blkid", "-s", "UUID", "-o", "value", device).Output()
  821. if err != nil {
  822. return "", fmt.Errorf("devmapper: Failed to find uuid for device %s:%v", device, err)
  823. }
  824. uuid := strings.TrimSuffix(string(out), "\n")
  825. uuid = strings.TrimSpace(uuid)
  826. logrus.Debugf("devmapper: UUID for device: %s is:%s", device, uuid)
  827. return uuid, nil
  828. }
  829. func (devices *DeviceSet) getBaseDeviceSize() uint64 {
  830. info, _ := devices.lookupDevice("")
  831. if info == nil {
  832. return 0
  833. }
  834. return info.Size
  835. }
  836. func (devices *DeviceSet) getBaseDeviceFS() string {
  837. return devices.BaseDeviceFilesystem
  838. }
  839. func (devices *DeviceSet) verifyBaseDeviceUUIDFS(baseInfo *devInfo) error {
  840. devices.Lock()
  841. defer devices.Unlock()
  842. if err := devices.activateDeviceIfNeeded(baseInfo, false); err != nil {
  843. return err
  844. }
  845. defer devices.deactivateDevice(baseInfo)
  846. uuid, err := getDeviceUUID(baseInfo.DevName())
  847. if err != nil {
  848. return err
  849. }
  850. if devices.BaseDeviceUUID != uuid {
  851. return fmt.Errorf("devmapper: Current Base Device UUID:%s does not match with stored UUID:%s. Possibly using a different thin pool than last invocation", uuid, devices.BaseDeviceUUID)
  852. }
  853. if devices.BaseDeviceFilesystem == "" {
  854. fsType, err := ProbeFsType(baseInfo.DevName())
  855. if err != nil {
  856. return err
  857. }
  858. if err := devices.saveBaseDeviceFilesystem(fsType); err != nil {
  859. return err
  860. }
  861. }
  862. // If user specified a filesystem using dm.fs option and current
  863. // file system of base image is not same, warn user that dm.fs
  864. // will be ignored.
  865. if devices.BaseDeviceFilesystem != devices.filesystem {
  866. logrus.Warnf("devmapper: Base device already exists and has filesystem %s on it. User specified filesystem %s will be ignored.", devices.BaseDeviceFilesystem, devices.filesystem)
  867. devices.filesystem = devices.BaseDeviceFilesystem
  868. }
  869. return nil
  870. }
  871. func (devices *DeviceSet) saveBaseDeviceFilesystem(fs string) error {
  872. devices.BaseDeviceFilesystem = fs
  873. return devices.saveDeviceSetMetaData()
  874. }
  875. func (devices *DeviceSet) saveBaseDeviceUUID(baseInfo *devInfo) error {
  876. devices.Lock()
  877. defer devices.Unlock()
  878. if err := devices.activateDeviceIfNeeded(baseInfo, false); err != nil {
  879. return err
  880. }
  881. defer devices.deactivateDevice(baseInfo)
  882. uuid, err := getDeviceUUID(baseInfo.DevName())
  883. if err != nil {
  884. return err
  885. }
  886. devices.BaseDeviceUUID = uuid
  887. return devices.saveDeviceSetMetaData()
  888. }
  889. func (devices *DeviceSet) createBaseImage() error {
  890. logrus.Debug("devmapper: Initializing base device-mapper thin volume")
  891. // Create initial device
  892. info, err := devices.createRegisterDevice("")
  893. if err != nil {
  894. return err
  895. }
  896. logrus.Debug("devmapper: Creating filesystem on base device-mapper thin volume")
  897. if err := devices.activateDeviceIfNeeded(info, false); err != nil {
  898. return err
  899. }
  900. if err := devices.createFilesystem(info); err != nil {
  901. return err
  902. }
  903. info.Initialized = true
  904. if err := devices.saveMetadata(info); err != nil {
  905. info.Initialized = false
  906. return err
  907. }
  908. if err := devices.saveBaseDeviceUUID(info); err != nil {
  909. return fmt.Errorf("devmapper: Could not query and save base device UUID:%v", err)
  910. }
  911. return nil
  912. }
  913. // Returns if thin pool device exists or not. If device exists, also makes
  914. // sure it is a thin pool device and not some other type of device.
  915. func (devices *DeviceSet) thinPoolExists(thinPoolDevice string) (bool, error) {
  916. logrus.Debugf("devmapper: Checking for existence of the pool %s", thinPoolDevice)
  917. info, err := devicemapper.GetInfo(thinPoolDevice)
  918. if err != nil {
  919. return false, fmt.Errorf("devmapper: GetInfo() on device %s failed: %v", thinPoolDevice, err)
  920. }
  921. // Device does not exist.
  922. if info.Exists == 0 {
  923. return false, nil
  924. }
  925. _, _, deviceType, _, err := devicemapper.GetStatus(thinPoolDevice)
  926. if err != nil {
  927. return false, fmt.Errorf("devmapper: GetStatus() on device %s failed: %v", thinPoolDevice, err)
  928. }
  929. if deviceType != "thin-pool" {
  930. return false, fmt.Errorf("devmapper: Device %s is not a thin pool", thinPoolDevice)
  931. }
  932. return true, nil
  933. }
  934. func (devices *DeviceSet) checkThinPool() error {
  935. _, transactionID, dataUsed, _, _, _, err := devices.poolStatus()
  936. if err != nil {
  937. return err
  938. }
  939. if dataUsed != 0 {
  940. return fmt.Errorf("devmapper: Unable to take ownership of thin-pool (%s) that already has used data blocks",
  941. devices.thinPoolDevice)
  942. }
  943. if transactionID != 0 {
  944. return fmt.Errorf("devmapper: Unable to take ownership of thin-pool (%s) with non-zero transaction ID",
  945. devices.thinPoolDevice)
  946. }
  947. return nil
  948. }
  949. // Base image is initialized properly. Either save UUID for first time (for
  950. // upgrade case or verify UUID.
  951. func (devices *DeviceSet) setupVerifyBaseImageUUIDFS(baseInfo *devInfo) error {
  952. // If BaseDeviceUUID is nil (upgrade case), save it and return success.
  953. if devices.BaseDeviceUUID == "" {
  954. if err := devices.saveBaseDeviceUUID(baseInfo); err != nil {
  955. return fmt.Errorf("devmapper: Could not query and save base device UUID:%v", err)
  956. }
  957. return nil
  958. }
  959. if err := devices.verifyBaseDeviceUUIDFS(baseInfo); err != nil {
  960. return fmt.Errorf("devmapper: Base Device UUID and Filesystem verification failed: %v", err)
  961. }
  962. return nil
  963. }
  964. func (devices *DeviceSet) checkGrowBaseDeviceFS(info *devInfo) error {
  965. if !userBaseSize {
  966. return nil
  967. }
  968. if devices.baseFsSize < devices.getBaseDeviceSize() {
  969. return fmt.Errorf("devmapper: Base device size cannot be smaller than %s", units.HumanSize(float64(devices.getBaseDeviceSize())))
  970. }
  971. if devices.baseFsSize == devices.getBaseDeviceSize() {
  972. return nil
  973. }
  974. info.lock.Lock()
  975. defer info.lock.Unlock()
  976. devices.Lock()
  977. defer devices.Unlock()
  978. info.Size = devices.baseFsSize
  979. if err := devices.saveMetadata(info); err != nil {
  980. // Try to remove unused device
  981. delete(devices.Devices, info.Hash)
  982. return err
  983. }
  984. return devices.growFS(info)
  985. }
  986. func (devices *DeviceSet) growFS(info *devInfo) error {
  987. if err := devices.activateDeviceIfNeeded(info, false); err != nil {
  988. return fmt.Errorf("Error activating devmapper device: %s", err)
  989. }
  990. defer devices.deactivateDevice(info)
  991. fsMountPoint := "/run/docker/mnt"
  992. if _, err := os.Stat(fsMountPoint); os.IsNotExist(err) {
  993. if err := os.MkdirAll(fsMountPoint, 0700); err != nil {
  994. return err
  995. }
  996. defer os.RemoveAll(fsMountPoint)
  997. }
  998. options := ""
  999. if devices.BaseDeviceFilesystem == "xfs" {
  1000. // XFS needs nouuid or it can't mount filesystems with the same fs
  1001. options = joinMountOptions(options, "nouuid")
  1002. }
  1003. options = joinMountOptions(options, devices.mountOptions)
  1004. if err := mount.Mount(info.DevName(), fsMountPoint, devices.BaseDeviceFilesystem, options); err != nil {
  1005. return fmt.Errorf("Error mounting '%s' on '%s': %s", info.DevName(), fsMountPoint, err)
  1006. }
  1007. defer syscall.Unmount(fsMountPoint, syscall.MNT_DETACH)
  1008. switch devices.BaseDeviceFilesystem {
  1009. case "ext4":
  1010. if out, err := exec.Command("resize2fs", info.DevName()).CombinedOutput(); err != nil {
  1011. return fmt.Errorf("Failed to grow rootfs:%v:%s", err, string(out))
  1012. }
  1013. case "xfs":
  1014. if out, err := exec.Command("xfs_growfs", info.DevName()).CombinedOutput(); err != nil {
  1015. return fmt.Errorf("Failed to grow rootfs:%v:%s", err, string(out))
  1016. }
  1017. default:
  1018. return fmt.Errorf("Unsupported filesystem type %s", devices.BaseDeviceFilesystem)
  1019. }
  1020. return nil
  1021. }
  1022. func (devices *DeviceSet) setupBaseImage() error {
  1023. oldInfo, _ := devices.lookupDeviceWithLock("")
  1024. // base image already exists. If it is initialized properly, do UUID
  1025. // verification and return. Otherwise remove image and set it up
  1026. // fresh.
  1027. if oldInfo != nil {
  1028. if oldInfo.Initialized && !oldInfo.Deleted {
  1029. if err := devices.setupVerifyBaseImageUUIDFS(oldInfo); err != nil {
  1030. return err
  1031. }
  1032. if err := devices.checkGrowBaseDeviceFS(oldInfo); err != nil {
  1033. return err
  1034. }
  1035. return nil
  1036. }
  1037. logrus.Debug("devmapper: Removing uninitialized base image")
  1038. // If previous base device is in deferred delete state,
  1039. // that needs to be cleaned up first. So don't try
  1040. // deferred deletion.
  1041. if err := devices.DeleteDevice("", true); err != nil {
  1042. return err
  1043. }
  1044. }
  1045. // If we are setting up base image for the first time, make sure
  1046. // thin pool is empty.
  1047. if devices.thinPoolDevice != "" && oldInfo == nil {
  1048. if err := devices.checkThinPool(); err != nil {
  1049. return err
  1050. }
  1051. }
  1052. // Create new base image device
  1053. if err := devices.createBaseImage(); err != nil {
  1054. return err
  1055. }
  1056. return nil
  1057. }
  1058. func setCloseOnExec(name string) {
  1059. if fileInfos, _ := ioutil.ReadDir("/proc/self/fd"); fileInfos != nil {
  1060. for _, i := range fileInfos {
  1061. link, _ := os.Readlink(filepath.Join("/proc/self/fd", i.Name()))
  1062. if link == name {
  1063. fd, err := strconv.Atoi(i.Name())
  1064. if err == nil {
  1065. syscall.CloseOnExec(fd)
  1066. }
  1067. }
  1068. }
  1069. }
  1070. }
  1071. // DMLog implements logging using DevMapperLogger interface.
  1072. func (devices *DeviceSet) DMLog(level int, file string, line int, dmError int, message string) {
  1073. // By default libdm sends us all the messages including debug ones.
  1074. // We need to filter out messages here and figure out which one
  1075. // should be printed.
  1076. if level > logLevel {
  1077. return
  1078. }
  1079. // FIXME(vbatts) push this back into ./pkg/devicemapper/
  1080. if level <= devicemapper.LogLevelErr {
  1081. logrus.Errorf("libdevmapper(%d): %s:%d (%d) %s", level, file, line, dmError, message)
  1082. } else if level <= devicemapper.LogLevelInfo {
  1083. logrus.Infof("libdevmapper(%d): %s:%d (%d) %s", level, file, line, dmError, message)
  1084. } else {
  1085. // FIXME(vbatts) push this back into ./pkg/devicemapper/
  1086. logrus.Debugf("libdevmapper(%d): %s:%d (%d) %s", level, file, line, dmError, message)
  1087. }
  1088. }
  1089. func major(device uint64) uint64 {
  1090. return (device >> 8) & 0xfff
  1091. }
  1092. func minor(device uint64) uint64 {
  1093. return (device & 0xff) | ((device >> 12) & 0xfff00)
  1094. }
  1095. // ResizePool increases the size of the pool.
  1096. func (devices *DeviceSet) ResizePool(size int64) error {
  1097. dirname := devices.loopbackDir()
  1098. datafilename := path.Join(dirname, "data")
  1099. if len(devices.dataDevice) > 0 {
  1100. datafilename = devices.dataDevice
  1101. }
  1102. metadatafilename := path.Join(dirname, "metadata")
  1103. if len(devices.metadataDevice) > 0 {
  1104. metadatafilename = devices.metadataDevice
  1105. }
  1106. datafile, err := os.OpenFile(datafilename, os.O_RDWR, 0)
  1107. if datafile == nil {
  1108. return err
  1109. }
  1110. defer datafile.Close()
  1111. fi, err := datafile.Stat()
  1112. if fi == nil {
  1113. return err
  1114. }
  1115. if fi.Size() > size {
  1116. return fmt.Errorf("devmapper: Can't shrink file")
  1117. }
  1118. dataloopback := loopback.FindLoopDeviceFor(datafile)
  1119. if dataloopback == nil {
  1120. return fmt.Errorf("devmapper: Unable to find loopback mount for: %s", datafilename)
  1121. }
  1122. defer dataloopback.Close()
  1123. metadatafile, err := os.OpenFile(metadatafilename, os.O_RDWR, 0)
  1124. if metadatafile == nil {
  1125. return err
  1126. }
  1127. defer metadatafile.Close()
  1128. metadataloopback := loopback.FindLoopDeviceFor(metadatafile)
  1129. if metadataloopback == nil {
  1130. return fmt.Errorf("devmapper: Unable to find loopback mount for: %s", metadatafilename)
  1131. }
  1132. defer metadataloopback.Close()
  1133. // Grow loopback file
  1134. if err := datafile.Truncate(size); err != nil {
  1135. return fmt.Errorf("devmapper: Unable to grow loopback file: %s", err)
  1136. }
  1137. // Reload size for loopback device
  1138. if err := loopback.SetCapacity(dataloopback); err != nil {
  1139. return fmt.Errorf("Unable to update loopback capacity: %s", err)
  1140. }
  1141. // Suspend the pool
  1142. if err := devicemapper.SuspendDevice(devices.getPoolName()); err != nil {
  1143. return fmt.Errorf("devmapper: Unable to suspend pool: %s", err)
  1144. }
  1145. // Reload with the new block sizes
  1146. if err := devicemapper.ReloadPool(devices.getPoolName(), dataloopback, metadataloopback, devices.thinpBlockSize); err != nil {
  1147. return fmt.Errorf("devmapper: Unable to reload pool: %s", err)
  1148. }
  1149. // Resume the pool
  1150. if err := devicemapper.ResumeDevice(devices.getPoolName()); err != nil {
  1151. return fmt.Errorf("devmapper: Unable to resume pool: %s", err)
  1152. }
  1153. return nil
  1154. }
  1155. func (devices *DeviceSet) loadTransactionMetaData() error {
  1156. jsonData, err := ioutil.ReadFile(devices.transactionMetaFile())
  1157. if err != nil {
  1158. // There is no active transaction. This will be the case
  1159. // during upgrade.
  1160. if os.IsNotExist(err) {
  1161. devices.OpenTransactionID = devices.TransactionID
  1162. return nil
  1163. }
  1164. return err
  1165. }
  1166. json.Unmarshal(jsonData, &devices.transaction)
  1167. return nil
  1168. }
  1169. func (devices *DeviceSet) saveTransactionMetaData() error {
  1170. jsonData, err := json.Marshal(&devices.transaction)
  1171. if err != nil {
  1172. return fmt.Errorf("devmapper: Error encoding metadata to json: %s", err)
  1173. }
  1174. return devices.writeMetaFile(jsonData, devices.transactionMetaFile())
  1175. }
  1176. func (devices *DeviceSet) removeTransactionMetaData() error {
  1177. return os.RemoveAll(devices.transactionMetaFile())
  1178. }
  1179. func (devices *DeviceSet) rollbackTransaction() error {
  1180. logrus.Debugf("devmapper: Rolling back open transaction: TransactionID=%d hash=%s device_id=%d", devices.OpenTransactionID, devices.DeviceIDHash, devices.DeviceID)
  1181. // A device id might have already been deleted before transaction
  1182. // closed. In that case this call will fail. Just leave a message
  1183. // in case of failure.
  1184. if err := devicemapper.DeleteDevice(devices.getPoolDevName(), devices.DeviceID); err != nil {
  1185. logrus.Errorf("devmapper: Unable to delete device: %s", err)
  1186. }
  1187. dinfo := &devInfo{Hash: devices.DeviceIDHash}
  1188. if err := devices.removeMetadata(dinfo); err != nil {
  1189. logrus.Errorf("devmapper: Unable to remove metadata: %s", err)
  1190. } else {
  1191. devices.markDeviceIDFree(devices.DeviceID)
  1192. }
  1193. if err := devices.removeTransactionMetaData(); err != nil {
  1194. logrus.Errorf("devmapper: Unable to remove transaction meta file %s: %s", devices.transactionMetaFile(), err)
  1195. }
  1196. return nil
  1197. }
  1198. func (devices *DeviceSet) processPendingTransaction() error {
  1199. if err := devices.loadTransactionMetaData(); err != nil {
  1200. return err
  1201. }
  1202. // If there was open transaction but pool transaction ID is same
  1203. // as open transaction ID, nothing to roll back.
  1204. if devices.TransactionID == devices.OpenTransactionID {
  1205. return nil
  1206. }
  1207. // If open transaction ID is less than pool transaction ID, something
  1208. // is wrong. Bail out.
  1209. if devices.OpenTransactionID < devices.TransactionID {
  1210. logrus.Errorf("devmapper: Open Transaction id %d is less than pool transaction id %d", devices.OpenTransactionID, devices.TransactionID)
  1211. return nil
  1212. }
  1213. // Pool transaction ID is not same as open transaction. There is
  1214. // a transaction which was not completed.
  1215. if err := devices.rollbackTransaction(); err != nil {
  1216. return fmt.Errorf("devmapper: Rolling back open transaction failed: %s", err)
  1217. }
  1218. devices.OpenTransactionID = devices.TransactionID
  1219. return nil
  1220. }
  1221. func (devices *DeviceSet) loadDeviceSetMetaData() error {
  1222. jsonData, err := ioutil.ReadFile(devices.deviceSetMetaFile())
  1223. if err != nil {
  1224. // For backward compatibility return success if file does
  1225. // not exist.
  1226. if os.IsNotExist(err) {
  1227. return nil
  1228. }
  1229. return err
  1230. }
  1231. return json.Unmarshal(jsonData, devices)
  1232. }
  1233. func (devices *DeviceSet) saveDeviceSetMetaData() error {
  1234. jsonData, err := json.Marshal(devices)
  1235. if err != nil {
  1236. return fmt.Errorf("devmapper: Error encoding metadata to json: %s", err)
  1237. }
  1238. return devices.writeMetaFile(jsonData, devices.deviceSetMetaFile())
  1239. }
  1240. func (devices *DeviceSet) openTransaction(hash string, DeviceID int) error {
  1241. devices.allocateTransactionID()
  1242. devices.DeviceIDHash = hash
  1243. devices.DeviceID = DeviceID
  1244. if err := devices.saveTransactionMetaData(); err != nil {
  1245. return fmt.Errorf("devmapper: Error saving transaction metadata: %s", err)
  1246. }
  1247. return nil
  1248. }
  1249. func (devices *DeviceSet) refreshTransaction(DeviceID int) error {
  1250. devices.DeviceID = DeviceID
  1251. if err := devices.saveTransactionMetaData(); err != nil {
  1252. return fmt.Errorf("devmapper: Error saving transaction metadata: %s", err)
  1253. }
  1254. return nil
  1255. }
  1256. func (devices *DeviceSet) closeTransaction() error {
  1257. if err := devices.updatePoolTransactionID(); err != nil {
  1258. logrus.Debug("devmapper: Failed to close Transaction")
  1259. return err
  1260. }
  1261. return nil
  1262. }
  1263. func determineDriverCapabilities(version string) error {
  1264. /*
  1265. * Driver version 4.27.0 and greater support deferred activation
  1266. * feature.
  1267. */
  1268. logrus.Debugf("devicemapper: driver version is %s", version)
  1269. versionSplit := strings.Split(version, ".")
  1270. major, err := strconv.Atoi(versionSplit[0])
  1271. if err != nil {
  1272. return graphdriver.ErrNotSupported
  1273. }
  1274. if major > 4 {
  1275. driverDeferredRemovalSupport = true
  1276. return nil
  1277. }
  1278. if major < 4 {
  1279. return nil
  1280. }
  1281. minor, err := strconv.Atoi(versionSplit[1])
  1282. if err != nil {
  1283. return graphdriver.ErrNotSupported
  1284. }
  1285. /*
  1286. * If major is 4 and minor is 27, then there is no need to
  1287. * check for patch level as it can not be less than 0.
  1288. */
  1289. if minor >= 27 {
  1290. driverDeferredRemovalSupport = true
  1291. return nil
  1292. }
  1293. return nil
  1294. }
  1295. // Determine the major and minor number of loopback device
  1296. func getDeviceMajorMinor(file *os.File) (uint64, uint64, error) {
  1297. stat, err := file.Stat()
  1298. if err != nil {
  1299. return 0, 0, err
  1300. }
  1301. dev := stat.Sys().(*syscall.Stat_t).Rdev
  1302. majorNum := major(dev)
  1303. minorNum := minor(dev)
  1304. logrus.Debugf("devmapper: Major:Minor for device: %s is:%v:%v", file.Name(), majorNum, minorNum)
  1305. return majorNum, minorNum, nil
  1306. }
  1307. // Given a file which is backing file of a loop back device, find the
  1308. // loopback device name and its major/minor number.
  1309. func getLoopFileDeviceMajMin(filename string) (string, uint64, uint64, error) {
  1310. file, err := os.Open(filename)
  1311. if err != nil {
  1312. logrus.Debugf("devmapper: Failed to open file %s", filename)
  1313. return "", 0, 0, err
  1314. }
  1315. defer file.Close()
  1316. loopbackDevice := loopback.FindLoopDeviceFor(file)
  1317. if loopbackDevice == nil {
  1318. return "", 0, 0, fmt.Errorf("devmapper: Unable to find loopback mount for: %s", filename)
  1319. }
  1320. defer loopbackDevice.Close()
  1321. Major, Minor, err := getDeviceMajorMinor(loopbackDevice)
  1322. if err != nil {
  1323. return "", 0, 0, err
  1324. }
  1325. return loopbackDevice.Name(), Major, Minor, nil
  1326. }
  1327. // Get the major/minor numbers of thin pool data and metadata devices
  1328. func (devices *DeviceSet) getThinPoolDataMetaMajMin() (uint64, uint64, uint64, uint64, error) {
  1329. var params, poolDataMajMin, poolMetadataMajMin string
  1330. _, _, _, params, err := devicemapper.GetTable(devices.getPoolName())
  1331. if err != nil {
  1332. return 0, 0, 0, 0, err
  1333. }
  1334. if _, err = fmt.Sscanf(params, "%s %s", &poolMetadataMajMin, &poolDataMajMin); err != nil {
  1335. return 0, 0, 0, 0, err
  1336. }
  1337. logrus.Debugf("devmapper: poolDataMajMin=%s poolMetaMajMin=%s\n", poolDataMajMin, poolMetadataMajMin)
  1338. poolDataMajMinorSplit := strings.Split(poolDataMajMin, ":")
  1339. poolDataMajor, err := strconv.ParseUint(poolDataMajMinorSplit[0], 10, 32)
  1340. if err != nil {
  1341. return 0, 0, 0, 0, err
  1342. }
  1343. poolDataMinor, err := strconv.ParseUint(poolDataMajMinorSplit[1], 10, 32)
  1344. if err != nil {
  1345. return 0, 0, 0, 0, err
  1346. }
  1347. poolMetadataMajMinorSplit := strings.Split(poolMetadataMajMin, ":")
  1348. poolMetadataMajor, err := strconv.ParseUint(poolMetadataMajMinorSplit[0], 10, 32)
  1349. if err != nil {
  1350. return 0, 0, 0, 0, err
  1351. }
  1352. poolMetadataMinor, err := strconv.ParseUint(poolMetadataMajMinorSplit[1], 10, 32)
  1353. if err != nil {
  1354. return 0, 0, 0, 0, err
  1355. }
  1356. return poolDataMajor, poolDataMinor, poolMetadataMajor, poolMetadataMinor, nil
  1357. }
  1358. func (devices *DeviceSet) loadThinPoolLoopBackInfo() error {
  1359. poolDataMajor, poolDataMinor, poolMetadataMajor, poolMetadataMinor, err := devices.getThinPoolDataMetaMajMin()
  1360. if err != nil {
  1361. return err
  1362. }
  1363. dirname := devices.loopbackDir()
  1364. // data device has not been passed in. So there should be a data file
  1365. // which is being mounted as loop device.
  1366. if devices.dataDevice == "" {
  1367. datafilename := path.Join(dirname, "data")
  1368. dataLoopDevice, dataMajor, dataMinor, err := getLoopFileDeviceMajMin(datafilename)
  1369. if err != nil {
  1370. return err
  1371. }
  1372. // Compare the two
  1373. if poolDataMajor == dataMajor && poolDataMinor == dataMinor {
  1374. devices.dataDevice = dataLoopDevice
  1375. devices.dataLoopFile = datafilename
  1376. }
  1377. }
  1378. // metadata device has not been passed in. So there should be a
  1379. // metadata file which is being mounted as loop device.
  1380. if devices.metadataDevice == "" {
  1381. metadatafilename := path.Join(dirname, "metadata")
  1382. metadataLoopDevice, metadataMajor, metadataMinor, err := getLoopFileDeviceMajMin(metadatafilename)
  1383. if err != nil {
  1384. return err
  1385. }
  1386. if poolMetadataMajor == metadataMajor && poolMetadataMinor == metadataMinor {
  1387. devices.metadataDevice = metadataLoopDevice
  1388. devices.metadataLoopFile = metadatafilename
  1389. }
  1390. }
  1391. return nil
  1392. }
  1393. func (devices *DeviceSet) enableDeferredRemovalDeletion() error {
  1394. // If user asked for deferred removal then check both libdm library
  1395. // and kernel driver support deferred removal otherwise error out.
  1396. if enableDeferredRemoval {
  1397. if !driverDeferredRemovalSupport {
  1398. return fmt.Errorf("devmapper: Deferred removal can not be enabled as kernel does not support it")
  1399. }
  1400. if !devicemapper.LibraryDeferredRemovalSupport {
  1401. return fmt.Errorf("devmapper: Deferred removal can not be enabled as libdm does not support it")
  1402. }
  1403. logrus.Debug("devmapper: Deferred removal support enabled.")
  1404. devices.deferredRemove = true
  1405. }
  1406. if enableDeferredDeletion {
  1407. if !devices.deferredRemove {
  1408. return fmt.Errorf("devmapper: Deferred deletion can not be enabled as deferred removal is not enabled. Enable deferred removal using --storage-opt dm.use_deferred_removal=true parameter")
  1409. }
  1410. logrus.Debug("devmapper: Deferred deletion support enabled.")
  1411. devices.deferredDelete = true
  1412. }
  1413. return nil
  1414. }
  1415. func (devices *DeviceSet) initDevmapper(doInit bool) (retErr error) {
  1416. // give ourselves to libdm as a log handler
  1417. devicemapper.LogInit(devices)
  1418. version, err := devicemapper.GetDriverVersion()
  1419. if err != nil {
  1420. // Can't even get driver version, assume not supported
  1421. return graphdriver.ErrNotSupported
  1422. }
  1423. if err := determineDriverCapabilities(version); err != nil {
  1424. return graphdriver.ErrNotSupported
  1425. }
  1426. if err := devices.enableDeferredRemovalDeletion(); err != nil {
  1427. return err
  1428. }
  1429. // https://github.com/docker/docker/issues/4036
  1430. if supported := devicemapper.UdevSetSyncSupport(true); !supported {
  1431. if dockerversion.IAmStatic == "true" {
  1432. logrus.Error("devmapper: Udev sync is not supported. This will lead to data loss and unexpected behavior. Install a dynamic binary to use devicemapper or select a different storage driver. For more information, see https://docs.docker.com/engine/reference/commandline/dockerd/#storage-driver-options")
  1433. } else {
  1434. logrus.Error("devmapper: Udev sync is not supported. This will lead to data loss and unexpected behavior. Install a more recent version of libdevmapper or select a different storage driver. For more information, see https://docs.docker.com/engine/reference/commandline/dockerd/#storage-driver-options")
  1435. }
  1436. if !devices.overrideUdevSyncCheck {
  1437. return graphdriver.ErrNotSupported
  1438. }
  1439. }
  1440. //create the root dir of the devmapper driver ownership to match this
  1441. //daemon's remapped root uid/gid so containers can start properly
  1442. uid, gid, err := idtools.GetRootUIDGID(devices.uidMaps, devices.gidMaps)
  1443. if err != nil {
  1444. return err
  1445. }
  1446. if err := idtools.MkdirAs(devices.root, 0700, uid, gid); err != nil && !os.IsExist(err) {
  1447. return err
  1448. }
  1449. if err := os.MkdirAll(devices.metadataDir(), 0700); err != nil && !os.IsExist(err) {
  1450. return err
  1451. }
  1452. prevSetupConfig, err := readLVMConfig(devices.root)
  1453. if err != nil {
  1454. return err
  1455. }
  1456. if !reflect.DeepEqual(devices.lvmSetupConfig, directLVMConfig{}) {
  1457. if devices.thinPoolDevice != "" {
  1458. return errors.New("cannot setup direct-lvm when `dm.thinpooldev` is also specified")
  1459. }
  1460. if !reflect.DeepEqual(prevSetupConfig, devices.lvmSetupConfig) {
  1461. if !reflect.DeepEqual(prevSetupConfig, directLVMConfig{}) {
  1462. return errors.New("changing direct-lvm config is not supported")
  1463. }
  1464. logrus.WithField("storage-driver", "devicemapper").WithField("direct-lvm-config", devices.lvmSetupConfig).Debugf("Setting up direct lvm mode")
  1465. if err := verifyBlockDevice(devices.lvmSetupConfig.Device, lvmSetupConfigForce); err != nil {
  1466. return err
  1467. }
  1468. if err := setupDirectLVM(devices.lvmSetupConfig); err != nil {
  1469. return err
  1470. }
  1471. if err := writeLVMConfig(devices.root, devices.lvmSetupConfig); err != nil {
  1472. return err
  1473. }
  1474. }
  1475. devices.thinPoolDevice = "docker-thinpool"
  1476. logrus.WithField("storage-driver", "devicemapper").Debugf("Setting dm.thinpooldev to %q", devices.thinPoolDevice)
  1477. }
  1478. // Set the device prefix from the device id and inode of the docker root dir
  1479. st, err := os.Stat(devices.root)
  1480. if err != nil {
  1481. return fmt.Errorf("devmapper: Error looking up dir %s: %s", devices.root, err)
  1482. }
  1483. sysSt := st.Sys().(*syscall.Stat_t)
  1484. // "reg-" stands for "regular file".
  1485. // In the future we might use "dev-" for "device file", etc.
  1486. // docker-maj,min[-inode] stands for:
  1487. // - Managed by docker
  1488. // - The target of this device is at major <maj> and minor <min>
  1489. // - If <inode> is defined, use that file inside the device as a loopback image. Otherwise use the device itself.
  1490. devices.devicePrefix = fmt.Sprintf("docker-%d:%d-%d", major(sysSt.Dev), minor(sysSt.Dev), sysSt.Ino)
  1491. logrus.Debugf("devmapper: Generated prefix: %s", devices.devicePrefix)
  1492. // Check for the existence of the thin-pool device
  1493. poolExists, err := devices.thinPoolExists(devices.getPoolName())
  1494. if err != nil {
  1495. return err
  1496. }
  1497. // It seems libdevmapper opens this without O_CLOEXEC, and go exec will not close files
  1498. // that are not Close-on-exec,
  1499. // so we add this badhack to make sure it closes itself
  1500. setCloseOnExec("/dev/mapper/control")
  1501. // Make sure the sparse images exist in <root>/devicemapper/data and
  1502. // <root>/devicemapper/metadata
  1503. createdLoopback := false
  1504. // If the pool doesn't exist, create it
  1505. if !poolExists && devices.thinPoolDevice == "" {
  1506. logrus.Debug("devmapper: Pool doesn't exist. Creating it.")
  1507. var (
  1508. dataFile *os.File
  1509. metadataFile *os.File
  1510. )
  1511. if devices.dataDevice == "" {
  1512. // Make sure the sparse images exist in <root>/devicemapper/data
  1513. hasData := devices.hasImage("data")
  1514. if !doInit && !hasData {
  1515. return errors.New("loopback data file not found")
  1516. }
  1517. if !hasData {
  1518. createdLoopback = true
  1519. }
  1520. data, err := devices.ensureImage("data", devices.dataLoopbackSize)
  1521. if err != nil {
  1522. logrus.Debugf("devmapper: Error device ensureImage (data): %s", err)
  1523. return err
  1524. }
  1525. dataFile, err = loopback.AttachLoopDevice(data)
  1526. if err != nil {
  1527. return err
  1528. }
  1529. devices.dataLoopFile = data
  1530. devices.dataDevice = dataFile.Name()
  1531. } else {
  1532. dataFile, err = os.OpenFile(devices.dataDevice, os.O_RDWR, 0600)
  1533. if err != nil {
  1534. return err
  1535. }
  1536. }
  1537. defer dataFile.Close()
  1538. if devices.metadataDevice == "" {
  1539. // Make sure the sparse images exist in <root>/devicemapper/metadata
  1540. hasMetadata := devices.hasImage("metadata")
  1541. if !doInit && !hasMetadata {
  1542. return errors.New("loopback metadata file not found")
  1543. }
  1544. if !hasMetadata {
  1545. createdLoopback = true
  1546. }
  1547. metadata, err := devices.ensureImage("metadata", devices.metaDataLoopbackSize)
  1548. if err != nil {
  1549. logrus.Debugf("devmapper: Error device ensureImage (metadata): %s", err)
  1550. return err
  1551. }
  1552. metadataFile, err = loopback.AttachLoopDevice(metadata)
  1553. if err != nil {
  1554. return err
  1555. }
  1556. devices.metadataLoopFile = metadata
  1557. devices.metadataDevice = metadataFile.Name()
  1558. } else {
  1559. metadataFile, err = os.OpenFile(devices.metadataDevice, os.O_RDWR, 0600)
  1560. if err != nil {
  1561. return err
  1562. }
  1563. }
  1564. defer metadataFile.Close()
  1565. if err := devicemapper.CreatePool(devices.getPoolName(), dataFile, metadataFile, devices.thinpBlockSize); err != nil {
  1566. return err
  1567. }
  1568. defer func() {
  1569. if retErr != nil {
  1570. err = devices.deactivatePool()
  1571. if err != nil {
  1572. logrus.Warnf("devmapper: Failed to deactivatePool: %v", err)
  1573. }
  1574. }
  1575. }()
  1576. }
  1577. // Pool already exists and caller did not pass us a pool. That means
  1578. // we probably created pool earlier and could not remove it as some
  1579. // containers were still using it. Detect some of the properties of
  1580. // pool, like is it using loop devices.
  1581. if poolExists && devices.thinPoolDevice == "" {
  1582. if err := devices.loadThinPoolLoopBackInfo(); err != nil {
  1583. logrus.Debugf("devmapper: Failed to load thin pool loopback device information:%v", err)
  1584. return err
  1585. }
  1586. }
  1587. // If we didn't just create the data or metadata image, we need to
  1588. // load the transaction id and migrate old metadata
  1589. if !createdLoopback {
  1590. if err := devices.initMetaData(); err != nil {
  1591. return err
  1592. }
  1593. }
  1594. if devices.thinPoolDevice == "" {
  1595. if devices.metadataLoopFile != "" || devices.dataLoopFile != "" {
  1596. logrus.Warn("devmapper: Usage of loopback devices is strongly discouraged for production use. Please use `--storage-opt dm.thinpooldev` or use `man docker` to refer to dm.thinpooldev section.")
  1597. }
  1598. }
  1599. // Right now this loads only NextDeviceID. If there is more metadata
  1600. // down the line, we might have to move it earlier.
  1601. if err := devices.loadDeviceSetMetaData(); err != nil {
  1602. return err
  1603. }
  1604. // Setup the base image
  1605. if doInit {
  1606. if err := devices.setupBaseImage(); err != nil {
  1607. logrus.Debugf("devmapper: Error device setupBaseImage: %s", err)
  1608. return err
  1609. }
  1610. }
  1611. return nil
  1612. }
  1613. // AddDevice adds a device and registers in the hash.
  1614. func (devices *DeviceSet) AddDevice(hash, baseHash string, storageOpt map[string]string) error {
  1615. logrus.Debugf("devmapper: AddDevice START(hash=%s basehash=%s)", hash, baseHash)
  1616. defer logrus.Debugf("devmapper: AddDevice END(hash=%s basehash=%s)", hash, baseHash)
  1617. // If a deleted device exists, return error.
  1618. baseInfo, err := devices.lookupDeviceWithLock(baseHash)
  1619. if err != nil {
  1620. return err
  1621. }
  1622. if baseInfo.Deleted {
  1623. return fmt.Errorf("devmapper: Base device %v has been marked for deferred deletion", baseInfo.Hash)
  1624. }
  1625. baseInfo.lock.Lock()
  1626. defer baseInfo.lock.Unlock()
  1627. devices.Lock()
  1628. defer devices.Unlock()
  1629. // Also include deleted devices in case hash of new device is
  1630. // same as one of the deleted devices.
  1631. if info, _ := devices.lookupDevice(hash); info != nil {
  1632. return fmt.Errorf("devmapper: device %s already exists. Deleted=%v", hash, info.Deleted)
  1633. }
  1634. size, err := devices.parseStorageOpt(storageOpt)
  1635. if err != nil {
  1636. return err
  1637. }
  1638. if size == 0 {
  1639. size = baseInfo.Size
  1640. }
  1641. if size < baseInfo.Size {
  1642. return fmt.Errorf("devmapper: Container size cannot be smaller than %s", units.HumanSize(float64(baseInfo.Size)))
  1643. }
  1644. if err := devices.takeSnapshot(hash, baseInfo, size); err != nil {
  1645. return err
  1646. }
  1647. // Grow the container rootfs.
  1648. if size > baseInfo.Size {
  1649. info, err := devices.lookupDevice(hash)
  1650. if err != nil {
  1651. return err
  1652. }
  1653. if err := devices.growFS(info); err != nil {
  1654. return err
  1655. }
  1656. }
  1657. return nil
  1658. }
  1659. func (devices *DeviceSet) parseStorageOpt(storageOpt map[string]string) (uint64, error) {
  1660. // Read size to change the block device size per container.
  1661. for key, val := range storageOpt {
  1662. key := strings.ToLower(key)
  1663. switch key {
  1664. case "size":
  1665. size, err := units.RAMInBytes(val)
  1666. if err != nil {
  1667. return 0, err
  1668. }
  1669. return uint64(size), nil
  1670. default:
  1671. return 0, fmt.Errorf("Unknown option %s", key)
  1672. }
  1673. }
  1674. return 0, nil
  1675. }
  1676. func (devices *DeviceSet) markForDeferredDeletion(info *devInfo) error {
  1677. // If device is already in deleted state, there is nothing to be done.
  1678. if info.Deleted {
  1679. return nil
  1680. }
  1681. logrus.Debugf("devmapper: Marking device %s for deferred deletion.", info.Hash)
  1682. info.Deleted = true
  1683. // save device metadata to reflect deleted state.
  1684. if err := devices.saveMetadata(info); err != nil {
  1685. info.Deleted = false
  1686. return err
  1687. }
  1688. devices.nrDeletedDevices++
  1689. return nil
  1690. }
  1691. // Should be called with devices.Lock() held.
  1692. func (devices *DeviceSet) deleteTransaction(info *devInfo, syncDelete bool) error {
  1693. if err := devices.openTransaction(info.Hash, info.DeviceID); err != nil {
  1694. logrus.Debugf("devmapper: Error opening transaction hash = %s deviceId = %d", "", info.DeviceID)
  1695. return err
  1696. }
  1697. defer devices.closeTransaction()
  1698. err := devicemapper.DeleteDevice(devices.getPoolDevName(), info.DeviceID)
  1699. if err != nil {
  1700. // If syncDelete is true, we want to return error. If deferred
  1701. // deletion is not enabled, we return an error. If error is
  1702. // something other then EBUSY, return an error.
  1703. if syncDelete || !devices.deferredDelete || err != devicemapper.ErrBusy {
  1704. logrus.Debugf("devmapper: Error deleting device: %s", err)
  1705. return err
  1706. }
  1707. }
  1708. if err == nil {
  1709. if err := devices.unregisterDevice(info.Hash); err != nil {
  1710. return err
  1711. }
  1712. // If device was already in deferred delete state that means
  1713. // deletion was being tried again later. Reduce the deleted
  1714. // device count.
  1715. if info.Deleted {
  1716. devices.nrDeletedDevices--
  1717. }
  1718. devices.markDeviceIDFree(info.DeviceID)
  1719. } else {
  1720. if err := devices.markForDeferredDeletion(info); err != nil {
  1721. return err
  1722. }
  1723. }
  1724. return nil
  1725. }
  1726. // Issue discard only if device open count is zero.
  1727. func (devices *DeviceSet) issueDiscard(info *devInfo) error {
  1728. logrus.Debugf("devmapper: issueDiscard START(device: %s).", info.Hash)
  1729. defer logrus.Debugf("devmapper: issueDiscard END(device: %s).", info.Hash)
  1730. // This is a workaround for the kernel not discarding block so
  1731. // on the thin pool when we remove a thinp device, so we do it
  1732. // manually.
  1733. // Even if device is deferred deleted, activate it and issue
  1734. // discards.
  1735. if err := devices.activateDeviceIfNeeded(info, true); err != nil {
  1736. return err
  1737. }
  1738. devinfo, err := devicemapper.GetInfo(info.Name())
  1739. if err != nil {
  1740. return err
  1741. }
  1742. if devinfo.OpenCount != 0 {
  1743. logrus.Debugf("devmapper: Device: %s is in use. OpenCount=%d. Not issuing discards.", info.Hash, devinfo.OpenCount)
  1744. return nil
  1745. }
  1746. if err := devicemapper.BlockDeviceDiscard(info.DevName()); err != nil {
  1747. logrus.Debugf("devmapper: Error discarding block on device: %s (ignoring)", err)
  1748. }
  1749. return nil
  1750. }
  1751. // Should be called with devices.Lock() held.
  1752. func (devices *DeviceSet) deleteDevice(info *devInfo, syncDelete bool) error {
  1753. if devices.doBlkDiscard {
  1754. devices.issueDiscard(info)
  1755. }
  1756. // Try to deactivate device in case it is active.
  1757. if err := devices.deactivateDevice(info); err != nil {
  1758. logrus.Debugf("devmapper: Error deactivating device: %s", err)
  1759. return err
  1760. }
  1761. if err := devices.deleteTransaction(info, syncDelete); err != nil {
  1762. return err
  1763. }
  1764. return nil
  1765. }
  1766. // DeleteDevice will return success if device has been marked for deferred
  1767. // removal. If one wants to override that and want DeleteDevice() to fail if
  1768. // device was busy and could not be deleted, set syncDelete=true.
  1769. func (devices *DeviceSet) DeleteDevice(hash string, syncDelete bool) error {
  1770. logrus.Debugf("devmapper: DeleteDevice START(hash=%v syncDelete=%v)", hash, syncDelete)
  1771. defer logrus.Debugf("devmapper: DeleteDevice END(hash=%v syncDelete=%v)", hash, syncDelete)
  1772. info, err := devices.lookupDeviceWithLock(hash)
  1773. if err != nil {
  1774. return err
  1775. }
  1776. info.lock.Lock()
  1777. defer info.lock.Unlock()
  1778. devices.Lock()
  1779. defer devices.Unlock()
  1780. return devices.deleteDevice(info, syncDelete)
  1781. }
  1782. func (devices *DeviceSet) deactivatePool() error {
  1783. logrus.Debug("devmapper: deactivatePool() START")
  1784. defer logrus.Debug("devmapper: deactivatePool() END")
  1785. devname := devices.getPoolDevName()
  1786. devinfo, err := devicemapper.GetInfo(devname)
  1787. if err != nil {
  1788. return err
  1789. }
  1790. if devinfo.Exists == 0 {
  1791. return nil
  1792. }
  1793. if err := devicemapper.RemoveDevice(devname); err != nil {
  1794. return err
  1795. }
  1796. if d, err := devicemapper.GetDeps(devname); err == nil {
  1797. logrus.Warnf("devmapper: device %s still has %d active dependents", devname, d.Count)
  1798. }
  1799. return nil
  1800. }
  1801. func (devices *DeviceSet) deactivateDevice(info *devInfo) error {
  1802. logrus.Debugf("devmapper: deactivateDevice START(%s)", info.Hash)
  1803. defer logrus.Debugf("devmapper: deactivateDevice END(%s)", info.Hash)
  1804. devinfo, err := devicemapper.GetInfo(info.Name())
  1805. if err != nil {
  1806. return err
  1807. }
  1808. if devinfo.Exists == 0 {
  1809. return nil
  1810. }
  1811. if devices.deferredRemove {
  1812. if err := devicemapper.RemoveDeviceDeferred(info.Name()); err != nil {
  1813. return err
  1814. }
  1815. } else {
  1816. if err := devices.removeDevice(info.Name()); err != nil {
  1817. return err
  1818. }
  1819. }
  1820. return nil
  1821. }
  1822. // Issues the underlying dm remove operation.
  1823. func (devices *DeviceSet) removeDevice(devname string) error {
  1824. var err error
  1825. logrus.Debugf("devmapper: removeDevice START(%s)", devname)
  1826. defer logrus.Debugf("devmapper: removeDevice END(%s)", devname)
  1827. for i := 0; i < 200; i++ {
  1828. err = devicemapper.RemoveDevice(devname)
  1829. if err == nil {
  1830. break
  1831. }
  1832. if err != devicemapper.ErrBusy {
  1833. return err
  1834. }
  1835. // If we see EBUSY it may be a transient error,
  1836. // sleep a bit a retry a few times.
  1837. devices.Unlock()
  1838. time.Sleep(100 * time.Millisecond)
  1839. devices.Lock()
  1840. }
  1841. return err
  1842. }
  1843. func (devices *DeviceSet) cancelDeferredRemovalIfNeeded(info *devInfo) error {
  1844. if !devices.deferredRemove {
  1845. return nil
  1846. }
  1847. logrus.Debugf("devmapper: cancelDeferredRemovalIfNeeded START(%s)", info.Name())
  1848. defer logrus.Debugf("devmapper: cancelDeferredRemovalIfNeeded END(%s)", info.Name())
  1849. devinfo, err := devicemapper.GetInfoWithDeferred(info.Name())
  1850. if err != nil {
  1851. return err
  1852. }
  1853. if devinfo != nil && devinfo.DeferredRemove == 0 {
  1854. return nil
  1855. }
  1856. // Cancel deferred remove
  1857. if err := devices.cancelDeferredRemoval(info); err != nil {
  1858. // If Error is ErrEnxio. Device is probably already gone. Continue.
  1859. if err != devicemapper.ErrEnxio {
  1860. return err
  1861. }
  1862. }
  1863. return nil
  1864. }
  1865. func (devices *DeviceSet) cancelDeferredRemoval(info *devInfo) error {
  1866. logrus.Debugf("devmapper: cancelDeferredRemoval START(%s)", info.Name())
  1867. defer logrus.Debugf("devmapper: cancelDeferredRemoval END(%s)", info.Name())
  1868. var err error
  1869. // Cancel deferred remove
  1870. for i := 0; i < 100; i++ {
  1871. err = devicemapper.CancelDeferredRemove(info.Name())
  1872. if err != nil {
  1873. if err == devicemapper.ErrBusy {
  1874. // If we see EBUSY it may be a transient error,
  1875. // sleep a bit a retry a few times.
  1876. devices.Unlock()
  1877. time.Sleep(100 * time.Millisecond)
  1878. devices.Lock()
  1879. continue
  1880. }
  1881. }
  1882. break
  1883. }
  1884. return err
  1885. }
  1886. // Shutdown shuts down the device by unmounting the root.
  1887. func (devices *DeviceSet) Shutdown(home string) error {
  1888. logrus.Debugf("devmapper: [deviceset %s] Shutdown()", devices.devicePrefix)
  1889. logrus.Debugf("devmapper: Shutting down DeviceSet: %s", devices.root)
  1890. defer logrus.Debugf("devmapper: [deviceset %s] Shutdown() END", devices.devicePrefix)
  1891. // Stop deletion worker. This should start delivering new events to
  1892. // ticker channel. That means no new instance of cleanupDeletedDevice()
  1893. // will run after this call. If one instance is already running at
  1894. // the time of the call, it must be holding devices.Lock() and
  1895. // we will block on this lock till cleanup function exits.
  1896. devices.deletionWorkerTicker.Stop()
  1897. devices.Lock()
  1898. // Save DeviceSet Metadata first. Docker kills all threads if they
  1899. // don't finish in certain time. It is possible that Shutdown()
  1900. // routine does not finish in time as we loop trying to deactivate
  1901. // some devices while these are busy. In that case shutdown() routine
  1902. // will be killed and we will not get a chance to save deviceset
  1903. // metadata. Hence save this early before trying to deactivate devices.
  1904. devices.saveDeviceSetMetaData()
  1905. // ignore the error since it's just a best effort to not try to unmount something that's mounted
  1906. mounts, _ := mount.GetMounts()
  1907. mounted := make(map[string]bool, len(mounts))
  1908. for _, mnt := range mounts {
  1909. mounted[mnt.Mountpoint] = true
  1910. }
  1911. if err := filepath.Walk(path.Join(home, "mnt"), func(p string, info os.FileInfo, err error) error {
  1912. if err != nil {
  1913. return err
  1914. }
  1915. if !info.IsDir() {
  1916. return nil
  1917. }
  1918. if mounted[p] {
  1919. // We use MNT_DETACH here in case it is still busy in some running
  1920. // container. This means it'll go away from the global scope directly,
  1921. // and the device will be released when that container dies.
  1922. if err := syscall.Unmount(p, syscall.MNT_DETACH); err != nil {
  1923. logrus.Debugf("devmapper: Shutdown unmounting %s, error: %s", p, err)
  1924. }
  1925. }
  1926. if devInfo, err := devices.lookupDevice(path.Base(p)); err != nil {
  1927. logrus.Debugf("devmapper: Shutdown lookup device %s, error: %s", path.Base(p), err)
  1928. } else {
  1929. if err := devices.deactivateDevice(devInfo); err != nil {
  1930. logrus.Debugf("devmapper: Shutdown deactivate %s , error: %s", devInfo.Hash, err)
  1931. }
  1932. }
  1933. return nil
  1934. }); err != nil && !os.IsNotExist(err) {
  1935. devices.Unlock()
  1936. return err
  1937. }
  1938. devices.Unlock()
  1939. info, _ := devices.lookupDeviceWithLock("")
  1940. if info != nil {
  1941. info.lock.Lock()
  1942. devices.Lock()
  1943. if err := devices.deactivateDevice(info); err != nil {
  1944. logrus.Debugf("devmapper: Shutdown deactivate base , error: %s", err)
  1945. }
  1946. devices.Unlock()
  1947. info.lock.Unlock()
  1948. }
  1949. devices.Lock()
  1950. if devices.thinPoolDevice == "" {
  1951. if err := devices.deactivatePool(); err != nil {
  1952. logrus.Debugf("devmapper: Shutdown deactivate pool , error: %s", err)
  1953. }
  1954. }
  1955. devices.Unlock()
  1956. return nil
  1957. }
  1958. // Recent XFS changes allow changing behavior of filesystem in case of errors.
  1959. // When thin pool gets full and XFS gets ENOSPC error, currently it tries
  1960. // IO infinitely and sometimes it can block the container process
  1961. // and process can't be killWith 0 value, XFS will not retry upon error
  1962. // and instead will shutdown filesystem.
  1963. func (devices *DeviceSet) xfsSetNospaceRetries(info *devInfo) error {
  1964. dmDevicePath, err := os.Readlink(info.DevName())
  1965. if err != nil {
  1966. return fmt.Errorf("devmapper: readlink failed for device %v:%v", info.DevName(), err)
  1967. }
  1968. dmDeviceName := path.Base(dmDevicePath)
  1969. filePath := "/sys/fs/xfs/" + dmDeviceName + "/error/metadata/ENOSPC/max_retries"
  1970. maxRetriesFile, err := os.OpenFile(filePath, os.O_WRONLY, 0)
  1971. if err != nil {
  1972. return fmt.Errorf("devmapper: user specified daemon option dm.xfs_nospace_max_retries but it does not seem to be supported on this system :%v", err)
  1973. }
  1974. defer maxRetriesFile.Close()
  1975. // Set max retries to 0
  1976. _, err = maxRetriesFile.WriteString(devices.xfsNospaceRetries)
  1977. if err != nil {
  1978. return fmt.Errorf("devmapper: Failed to write string %v to file %v:%v", devices.xfsNospaceRetries, filePath, err)
  1979. }
  1980. return nil
  1981. }
  1982. // MountDevice mounts the device if not already mounted.
  1983. func (devices *DeviceSet) MountDevice(hash, path, mountLabel string) error {
  1984. info, err := devices.lookupDeviceWithLock(hash)
  1985. if err != nil {
  1986. return err
  1987. }
  1988. if info.Deleted {
  1989. return fmt.Errorf("devmapper: Can't mount device %v as it has been marked for deferred deletion", info.Hash)
  1990. }
  1991. info.lock.Lock()
  1992. defer info.lock.Unlock()
  1993. devices.Lock()
  1994. defer devices.Unlock()
  1995. if err := devices.activateDeviceIfNeeded(info, false); err != nil {
  1996. return fmt.Errorf("devmapper: Error activating devmapper device for '%s': %s", hash, err)
  1997. }
  1998. fstype, err := ProbeFsType(info.DevName())
  1999. if err != nil {
  2000. return err
  2001. }
  2002. options := ""
  2003. if fstype == "xfs" {
  2004. // XFS needs nouuid or it can't mount filesystems with the same fs
  2005. options = joinMountOptions(options, "nouuid")
  2006. }
  2007. options = joinMountOptions(options, devices.mountOptions)
  2008. options = joinMountOptions(options, label.FormatMountLabel("", mountLabel))
  2009. if err := mount.Mount(info.DevName(), path, fstype, options); err != nil {
  2010. return fmt.Errorf("devmapper: Error mounting '%s' on '%s': %s", info.DevName(), path, err)
  2011. }
  2012. if fstype == "xfs" && devices.xfsNospaceRetries != "" {
  2013. if err := devices.xfsSetNospaceRetries(info); err != nil {
  2014. syscall.Unmount(path, syscall.MNT_DETACH)
  2015. devices.deactivateDevice(info)
  2016. return err
  2017. }
  2018. }
  2019. return nil
  2020. }
  2021. // UnmountDevice unmounts the device and removes it from hash.
  2022. func (devices *DeviceSet) UnmountDevice(hash, mountPath string) error {
  2023. logrus.Debugf("devmapper: UnmountDevice START(hash=%s)", hash)
  2024. defer logrus.Debugf("devmapper: UnmountDevice END(hash=%s)", hash)
  2025. info, err := devices.lookupDeviceWithLock(hash)
  2026. if err != nil {
  2027. return err
  2028. }
  2029. info.lock.Lock()
  2030. defer info.lock.Unlock()
  2031. devices.Lock()
  2032. defer devices.Unlock()
  2033. logrus.Debugf("devmapper: Unmount(%s)", mountPath)
  2034. if err := syscall.Unmount(mountPath, syscall.MNT_DETACH); err != nil {
  2035. return err
  2036. }
  2037. logrus.Debug("devmapper: Unmount done")
  2038. return devices.deactivateDevice(info)
  2039. }
  2040. // HasDevice returns true if the device metadata exists.
  2041. func (devices *DeviceSet) HasDevice(hash string) bool {
  2042. info, _ := devices.lookupDeviceWithLock(hash)
  2043. return info != nil
  2044. }
  2045. // List returns a list of device ids.
  2046. func (devices *DeviceSet) List() []string {
  2047. devices.Lock()
  2048. defer devices.Unlock()
  2049. ids := make([]string, len(devices.Devices))
  2050. i := 0
  2051. for k := range devices.Devices {
  2052. ids[i] = k
  2053. i++
  2054. }
  2055. return ids
  2056. }
  2057. func (devices *DeviceSet) deviceStatus(devName string) (sizeInSectors, mappedSectors, highestMappedSector uint64, err error) {
  2058. var params string
  2059. _, sizeInSectors, _, params, err = devicemapper.GetStatus(devName)
  2060. if err != nil {
  2061. return
  2062. }
  2063. if _, err = fmt.Sscanf(params, "%d %d", &mappedSectors, &highestMappedSector); err == nil {
  2064. return
  2065. }
  2066. return
  2067. }
  2068. // GetDeviceStatus provides size, mapped sectors
  2069. func (devices *DeviceSet) GetDeviceStatus(hash string) (*DevStatus, error) {
  2070. info, err := devices.lookupDeviceWithLock(hash)
  2071. if err != nil {
  2072. return nil, err
  2073. }
  2074. info.lock.Lock()
  2075. defer info.lock.Unlock()
  2076. devices.Lock()
  2077. defer devices.Unlock()
  2078. status := &DevStatus{
  2079. DeviceID: info.DeviceID,
  2080. Size: info.Size,
  2081. TransactionID: info.TransactionID,
  2082. }
  2083. if err := devices.activateDeviceIfNeeded(info, false); err != nil {
  2084. return nil, fmt.Errorf("devmapper: Error activating devmapper device for '%s': %s", hash, err)
  2085. }
  2086. sizeInSectors, mappedSectors, highestMappedSector, err := devices.deviceStatus(info.DevName())
  2087. if err != nil {
  2088. return nil, err
  2089. }
  2090. status.SizeInSectors = sizeInSectors
  2091. status.MappedSectors = mappedSectors
  2092. status.HighestMappedSector = highestMappedSector
  2093. return status, nil
  2094. }
  2095. func (devices *DeviceSet) poolStatus() (totalSizeInSectors, transactionID, dataUsed, dataTotal, metadataUsed, metadataTotal uint64, err error) {
  2096. var params string
  2097. if _, totalSizeInSectors, _, params, err = devicemapper.GetStatus(devices.getPoolName()); err == nil {
  2098. _, err = fmt.Sscanf(params, "%d %d/%d %d/%d", &transactionID, &metadataUsed, &metadataTotal, &dataUsed, &dataTotal)
  2099. }
  2100. return
  2101. }
  2102. // DataDevicePath returns the path to the data storage for this deviceset,
  2103. // regardless of loopback or block device
  2104. func (devices *DeviceSet) DataDevicePath() string {
  2105. return devices.dataDevice
  2106. }
  2107. // MetadataDevicePath returns the path to the metadata storage for this deviceset,
  2108. // regardless of loopback or block device
  2109. func (devices *DeviceSet) MetadataDevicePath() string {
  2110. return devices.metadataDevice
  2111. }
  2112. func (devices *DeviceSet) getUnderlyingAvailableSpace(loopFile string) (uint64, error) {
  2113. buf := new(syscall.Statfs_t)
  2114. if err := syscall.Statfs(loopFile, buf); err != nil {
  2115. logrus.Warnf("devmapper: Couldn't stat loopfile filesystem %v: %v", loopFile, err)
  2116. return 0, err
  2117. }
  2118. return buf.Bfree * uint64(buf.Bsize), nil
  2119. }
  2120. func (devices *DeviceSet) isRealFile(loopFile string) (bool, error) {
  2121. if loopFile != "" {
  2122. fi, err := os.Stat(loopFile)
  2123. if err != nil {
  2124. logrus.Warnf("devmapper: Couldn't stat loopfile %v: %v", loopFile, err)
  2125. return false, err
  2126. }
  2127. return fi.Mode().IsRegular(), nil
  2128. }
  2129. return false, nil
  2130. }
  2131. // Status returns the current status of this deviceset
  2132. func (devices *DeviceSet) Status() *Status {
  2133. devices.Lock()
  2134. defer devices.Unlock()
  2135. status := &Status{}
  2136. status.PoolName = devices.getPoolName()
  2137. status.DataFile = devices.DataDevicePath()
  2138. status.DataLoopback = devices.dataLoopFile
  2139. status.MetadataFile = devices.MetadataDevicePath()
  2140. status.MetadataLoopback = devices.metadataLoopFile
  2141. status.UdevSyncSupported = devicemapper.UdevSyncSupported()
  2142. status.DeferredRemoveEnabled = devices.deferredRemove
  2143. status.DeferredDeleteEnabled = devices.deferredDelete
  2144. status.DeferredDeletedDeviceCount = devices.nrDeletedDevices
  2145. status.BaseDeviceSize = devices.getBaseDeviceSize()
  2146. status.BaseDeviceFS = devices.getBaseDeviceFS()
  2147. totalSizeInSectors, _, dataUsed, dataTotal, metadataUsed, metadataTotal, err := devices.poolStatus()
  2148. if err == nil {
  2149. // Convert from blocks to bytes
  2150. blockSizeInSectors := totalSizeInSectors / dataTotal
  2151. status.Data.Used = dataUsed * blockSizeInSectors * 512
  2152. status.Data.Total = dataTotal * blockSizeInSectors * 512
  2153. status.Data.Available = status.Data.Total - status.Data.Used
  2154. // metadata blocks are always 4k
  2155. status.Metadata.Used = metadataUsed * 4096
  2156. status.Metadata.Total = metadataTotal * 4096
  2157. status.Metadata.Available = status.Metadata.Total - status.Metadata.Used
  2158. status.SectorSize = blockSizeInSectors * 512
  2159. if check, _ := devices.isRealFile(devices.dataLoopFile); check {
  2160. actualSpace, err := devices.getUnderlyingAvailableSpace(devices.dataLoopFile)
  2161. if err == nil && actualSpace < status.Data.Available {
  2162. status.Data.Available = actualSpace
  2163. }
  2164. }
  2165. if check, _ := devices.isRealFile(devices.metadataLoopFile); check {
  2166. actualSpace, err := devices.getUnderlyingAvailableSpace(devices.metadataLoopFile)
  2167. if err == nil && actualSpace < status.Metadata.Available {
  2168. status.Metadata.Available = actualSpace
  2169. }
  2170. }
  2171. minFreeData := (dataTotal * uint64(devices.minFreeSpacePercent)) / 100
  2172. status.MinFreeSpace = minFreeData * blockSizeInSectors * 512
  2173. }
  2174. return status
  2175. }
  2176. // Status returns the current status of this deviceset
  2177. func (devices *DeviceSet) exportDeviceMetadata(hash string) (*deviceMetadata, error) {
  2178. info, err := devices.lookupDeviceWithLock(hash)
  2179. if err != nil {
  2180. return nil, err
  2181. }
  2182. info.lock.Lock()
  2183. defer info.lock.Unlock()
  2184. metadata := &deviceMetadata{info.DeviceID, info.Size, info.Name()}
  2185. return metadata, nil
  2186. }
  2187. // NewDeviceSet creates the device set based on the options provided.
  2188. func NewDeviceSet(root string, doInit bool, options []string, uidMaps, gidMaps []idtools.IDMap) (*DeviceSet, error) {
  2189. devicemapper.SetDevDir("/dev")
  2190. devices := &DeviceSet{
  2191. root: root,
  2192. metaData: metaData{Devices: make(map[string]*devInfo)},
  2193. dataLoopbackSize: defaultDataLoopbackSize,
  2194. metaDataLoopbackSize: defaultMetaDataLoopbackSize,
  2195. baseFsSize: defaultBaseFsSize,
  2196. overrideUdevSyncCheck: defaultUdevSyncOverride,
  2197. doBlkDiscard: true,
  2198. thinpBlockSize: defaultThinpBlockSize,
  2199. deviceIDMap: make([]byte, deviceIDMapSz),
  2200. deletionWorkerTicker: time.NewTicker(time.Second * 30),
  2201. uidMaps: uidMaps,
  2202. gidMaps: gidMaps,
  2203. minFreeSpacePercent: defaultMinFreeSpacePercent,
  2204. }
  2205. foundBlkDiscard := false
  2206. var lvmSetupConfig directLVMConfig
  2207. for _, option := range options {
  2208. key, val, err := parsers.ParseKeyValueOpt(option)
  2209. if err != nil {
  2210. return nil, err
  2211. }
  2212. key = strings.ToLower(key)
  2213. switch key {
  2214. case "dm.basesize":
  2215. size, err := units.RAMInBytes(val)
  2216. if err != nil {
  2217. return nil, err
  2218. }
  2219. userBaseSize = true
  2220. devices.baseFsSize = uint64(size)
  2221. case "dm.loopdatasize":
  2222. size, err := units.RAMInBytes(val)
  2223. if err != nil {
  2224. return nil, err
  2225. }
  2226. devices.dataLoopbackSize = size
  2227. case "dm.loopmetadatasize":
  2228. size, err := units.RAMInBytes(val)
  2229. if err != nil {
  2230. return nil, err
  2231. }
  2232. devices.metaDataLoopbackSize = size
  2233. case "dm.fs":
  2234. if val != "ext4" && val != "xfs" {
  2235. return nil, fmt.Errorf("devmapper: Unsupported filesystem %s\n", val)
  2236. }
  2237. devices.filesystem = val
  2238. case "dm.mkfsarg":
  2239. devices.mkfsArgs = append(devices.mkfsArgs, val)
  2240. case "dm.mountopt":
  2241. devices.mountOptions = joinMountOptions(devices.mountOptions, val)
  2242. case "dm.metadatadev":
  2243. devices.metadataDevice = val
  2244. case "dm.datadev":
  2245. devices.dataDevice = val
  2246. case "dm.thinpooldev":
  2247. devices.thinPoolDevice = strings.TrimPrefix(val, "/dev/mapper/")
  2248. case "dm.blkdiscard":
  2249. foundBlkDiscard = true
  2250. devices.doBlkDiscard, err = strconv.ParseBool(val)
  2251. if err != nil {
  2252. return nil, err
  2253. }
  2254. case "dm.blocksize":
  2255. size, err := units.RAMInBytes(val)
  2256. if err != nil {
  2257. return nil, err
  2258. }
  2259. // convert to 512b sectors
  2260. devices.thinpBlockSize = uint32(size) >> 9
  2261. case "dm.override_udev_sync_check":
  2262. devices.overrideUdevSyncCheck, err = strconv.ParseBool(val)
  2263. if err != nil {
  2264. return nil, err
  2265. }
  2266. case "dm.use_deferred_removal":
  2267. enableDeferredRemoval, err = strconv.ParseBool(val)
  2268. if err != nil {
  2269. return nil, err
  2270. }
  2271. case "dm.use_deferred_deletion":
  2272. enableDeferredDeletion, err = strconv.ParseBool(val)
  2273. if err != nil {
  2274. return nil, err
  2275. }
  2276. case "dm.min_free_space":
  2277. if !strings.HasSuffix(val, "%") {
  2278. return nil, fmt.Errorf("devmapper: Option dm.min_free_space requires %% suffix")
  2279. }
  2280. valstring := strings.TrimSuffix(val, "%")
  2281. minFreeSpacePercent, err := strconv.ParseUint(valstring, 10, 32)
  2282. if err != nil {
  2283. return nil, err
  2284. }
  2285. if minFreeSpacePercent >= 100 {
  2286. return nil, fmt.Errorf("devmapper: Invalid value %v for option dm.min_free_space", val)
  2287. }
  2288. devices.minFreeSpacePercent = uint32(minFreeSpacePercent)
  2289. case "dm.xfs_nospace_max_retries":
  2290. _, err := strconv.ParseUint(val, 10, 64)
  2291. if err != nil {
  2292. return nil, err
  2293. }
  2294. devices.xfsNospaceRetries = val
  2295. case "dm.directlvm_device":
  2296. lvmSetupConfig.Device = val
  2297. case "dm.directlvm_device_force":
  2298. lvmSetupConfigForce, err = strconv.ParseBool(val)
  2299. if err != nil {
  2300. return nil, err
  2301. }
  2302. case "dm.thinp_percent":
  2303. per, err := strconv.ParseUint(strings.TrimSuffix(val, "%"), 10, 32)
  2304. if err != nil {
  2305. return nil, errors.Wrapf(err, "could not parse `dm.thinp_percent=%s`", val)
  2306. }
  2307. if per >= 100 {
  2308. return nil, errors.New("dm.thinp_percent must be greater than 0 and less than 100")
  2309. }
  2310. lvmSetupConfig.ThinpPercent = per
  2311. case "dm.thinp_metapercent":
  2312. per, err := strconv.ParseUint(strings.TrimSuffix(val, "%"), 10, 32)
  2313. if err != nil {
  2314. return nil, errors.Wrapf(err, "could not parse `dm.thinp_metapercent=%s`", val)
  2315. }
  2316. if per >= 100 {
  2317. return nil, errors.New("dm.thinp_metapercent must be greater than 0 and less than 100")
  2318. }
  2319. lvmSetupConfig.ThinpMetaPercent = per
  2320. case "dm.thinp_autoextend_percent":
  2321. per, err := strconv.ParseUint(strings.TrimSuffix(val, "%"), 10, 32)
  2322. if err != nil {
  2323. return nil, errors.Wrapf(err, "could not parse `dm.thinp_autoextend_percent=%s`", val)
  2324. }
  2325. if per > 100 {
  2326. return nil, errors.New("dm.thinp_autoextend_percent must be greater than 0 and less than 100")
  2327. }
  2328. lvmSetupConfig.AutoExtendPercent = per
  2329. case "dm.thinp_autoextend_threshold":
  2330. per, err := strconv.ParseUint(strings.TrimSuffix(val, "%"), 10, 32)
  2331. if err != nil {
  2332. return nil, errors.Wrapf(err, "could not parse `dm.thinp_autoextend_threshold=%s`", val)
  2333. }
  2334. if per > 100 {
  2335. return nil, errors.New("dm.thinp_autoextend_threshold must be greater than 0 and less than 100")
  2336. }
  2337. lvmSetupConfig.AutoExtendThreshold = per
  2338. default:
  2339. return nil, fmt.Errorf("devmapper: Unknown option %s\n", key)
  2340. }
  2341. }
  2342. if err := validateLVMConfig(lvmSetupConfig); err != nil {
  2343. return nil, err
  2344. }
  2345. devices.lvmSetupConfig = lvmSetupConfig
  2346. // By default, don't do blk discard hack on raw devices, its rarely useful and is expensive
  2347. if !foundBlkDiscard && (devices.dataDevice != "" || devices.thinPoolDevice != "") {
  2348. devices.doBlkDiscard = false
  2349. }
  2350. if err := devices.initDevmapper(doInit); err != nil {
  2351. return nil, err
  2352. }
  2353. return devices, nil
  2354. }