container.go 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539
  1. package docker
  2. import (
  3. "encoding/json"
  4. "errors"
  5. "./fs"
  6. "github.com/kr/pty"
  7. "io"
  8. "io/ioutil"
  9. "log"
  10. "os"
  11. "os/exec"
  12. "path"
  13. "strconv"
  14. "syscall"
  15. "time"
  16. )
  17. var sysInitPath string
  18. func init() {
  19. sysInitPath = SelfPath()
  20. }
  21. type Container struct {
  22. Id string
  23. Root string
  24. Created time.Time
  25. Path string
  26. Args []string
  27. Config *Config
  28. Mountpoint *fs.Mountpoint
  29. State *State
  30. Image string
  31. network *NetworkInterface
  32. networkManager *NetworkManager
  33. NetworkSettings *NetworkSettings
  34. SysInitPath string
  35. lxcConfigPath string
  36. cmd *exec.Cmd
  37. stdout *writeBroadcaster
  38. stderr *writeBroadcaster
  39. stdin io.ReadCloser
  40. stdinPipe io.WriteCloser
  41. stdoutLog *os.File
  42. stderrLog *os.File
  43. }
  44. type Config struct {
  45. Hostname string
  46. User string
  47. Ram int64
  48. Ports []int
  49. Tty bool // Attach standard streams to a tty, including stdin if it is not closed.
  50. OpenStdin bool // Open stdin
  51. }
  52. type NetworkSettings struct {
  53. IpAddress string
  54. IpPrefixLen int
  55. Gateway string
  56. PortMapping map[string]string
  57. }
  58. func createContainer(id string, root string, command string, args []string, image *fs.Image, config *Config, netManager *NetworkManager) (*Container, error) {
  59. mountpoint, err := image.Mountpoint(path.Join(root, "rootfs"), path.Join(root, "rw"))
  60. if err != nil {
  61. return nil, err
  62. }
  63. container := &Container{
  64. Id: id,
  65. Root: root,
  66. Created: time.Now(),
  67. Path: command,
  68. Args: args,
  69. Config: config,
  70. Image: image.Id,
  71. Mountpoint: mountpoint,
  72. State: newState(),
  73. networkManager: netManager,
  74. NetworkSettings: &NetworkSettings{},
  75. SysInitPath: sysInitPath,
  76. lxcConfigPath: path.Join(root, "config.lxc"),
  77. stdout: newWriteBroadcaster(),
  78. stderr: newWriteBroadcaster(),
  79. }
  80. if err := os.Mkdir(root, 0700); err != nil {
  81. return nil, err
  82. }
  83. // Setup logging of stdout and stderr to disk
  84. if stdoutLog, err := os.OpenFile(path.Join(container.Root, id+"-stdout.log"), os.O_RDWR|os.O_APPEND|os.O_CREATE, 0600); err != nil {
  85. return nil, err
  86. } else {
  87. container.stdoutLog = stdoutLog
  88. }
  89. if stderrLog, err := os.OpenFile(path.Join(container.Root, id+"-stderr.log"), os.O_RDWR|os.O_APPEND|os.O_CREATE, 0600); err != nil {
  90. return nil, err
  91. } else {
  92. container.stderrLog = stderrLog
  93. }
  94. if container.Config.OpenStdin {
  95. container.stdin, container.stdinPipe = io.Pipe()
  96. } else {
  97. container.stdinPipe = NopWriteCloser(ioutil.Discard) // Silently drop stdin
  98. }
  99. container.stdout.AddWriter(NopWriteCloser(container.stdoutLog))
  100. container.stderr.AddWriter(NopWriteCloser(container.stderrLog))
  101. if err := container.save(); err != nil {
  102. return nil, err
  103. }
  104. return container, nil
  105. }
  106. func loadContainer(store *fs.Store, containerPath string, netManager *NetworkManager) (*Container, error) {
  107. data, err := ioutil.ReadFile(path.Join(containerPath, "config.json"))
  108. if err != nil {
  109. return nil, err
  110. }
  111. mountpoint, err := store.FetchMountpoint(
  112. path.Join(containerPath, "rootfs"),
  113. path.Join(containerPath, "rw"),
  114. )
  115. if err != nil {
  116. return nil, err
  117. } else if mountpoint == nil {
  118. return nil, errors.New("Couldn't load container: unregistered mountpoint.")
  119. }
  120. container := &Container{
  121. stdout: newWriteBroadcaster(),
  122. stderr: newWriteBroadcaster(),
  123. lxcConfigPath: path.Join(containerPath, "config.lxc"),
  124. networkManager: netManager,
  125. NetworkSettings: &NetworkSettings{},
  126. Mountpoint: mountpoint,
  127. }
  128. // Load container settings
  129. if err := json.Unmarshal(data, container); err != nil {
  130. return nil, err
  131. }
  132. // Setup logging of stdout and stderr to disk
  133. if stdoutLog, err := os.OpenFile(path.Join(container.Root, container.Id+"-stdout.log"), os.O_RDWR|os.O_APPEND|os.O_CREATE, 0600); err != nil {
  134. return nil, err
  135. } else {
  136. container.stdoutLog = stdoutLog
  137. }
  138. if stderrLog, err := os.OpenFile(path.Join(container.Root, container.Id+"-stderr.log"), os.O_RDWR|os.O_APPEND|os.O_CREATE, 0600); err != nil {
  139. return nil, err
  140. } else {
  141. container.stderrLog = stderrLog
  142. }
  143. container.stdout.AddWriter(NopWriteCloser(container.stdoutLog))
  144. container.stderr.AddWriter(NopWriteCloser(container.stderrLog))
  145. if container.Config.OpenStdin {
  146. container.stdin, container.stdinPipe = io.Pipe()
  147. } else {
  148. container.stdinPipe = NopWriteCloser(ioutil.Discard) // Silently drop stdin
  149. }
  150. container.State = newState()
  151. return container, nil
  152. }
  153. func (container *Container) Cmd() *exec.Cmd {
  154. return container.cmd
  155. }
  156. func (container *Container) When() time.Time {
  157. return container.Created
  158. }
  159. func (container *Container) loadUserData() (map[string]string, error) {
  160. jsonData, err := ioutil.ReadFile(path.Join(container.Root, "userdata.json"))
  161. if err != nil {
  162. if os.IsNotExist(err) {
  163. return make(map[string]string), nil
  164. }
  165. return nil, err
  166. }
  167. data := make(map[string]string)
  168. if err := json.Unmarshal(jsonData, &data); err != nil {
  169. return nil, err
  170. }
  171. return data, nil
  172. }
  173. func (container *Container) saveUserData(data map[string]string) error {
  174. jsonData, err := json.Marshal(data)
  175. if err != nil {
  176. return err
  177. }
  178. return ioutil.WriteFile(path.Join(container.Root, "userdata.json"), jsonData, 0700)
  179. }
  180. func (container *Container) SetUserData(key, value string) error {
  181. data, err := container.loadUserData()
  182. if err != nil {
  183. return err
  184. }
  185. data[key] = value
  186. return container.saveUserData(data)
  187. }
  188. func (container *Container) GetUserData(key string) string {
  189. data, err := container.loadUserData()
  190. if err != nil {
  191. return ""
  192. }
  193. if value, exists := data[key]; exists {
  194. return value
  195. }
  196. return ""
  197. }
  198. func (container *Container) save() (err error) {
  199. data, err := json.Marshal(container)
  200. if err != nil {
  201. return
  202. }
  203. return ioutil.WriteFile(path.Join(container.Root, "config.json"), data, 0666)
  204. }
  205. func (container *Container) generateLXCConfig() error {
  206. fo, err := os.Create(container.lxcConfigPath)
  207. if err != nil {
  208. return err
  209. }
  210. defer fo.Close()
  211. if err := LxcTemplateCompiled.Execute(fo, container); err != nil {
  212. return err
  213. }
  214. return nil
  215. }
  216. func (container *Container) startPty() error {
  217. stdout_master, stdout_slave, err := pty.Open()
  218. if err != nil {
  219. return err
  220. }
  221. container.cmd.Stdout = stdout_slave
  222. stderr_master, stderr_slave, err := pty.Open()
  223. if err != nil {
  224. return err
  225. }
  226. container.cmd.Stderr = stderr_slave
  227. // Copy the PTYs to our broadcasters
  228. go func() {
  229. defer container.stdout.Close()
  230. io.Copy(container.stdout, stdout_master)
  231. }()
  232. go func() {
  233. defer container.stderr.Close()
  234. io.Copy(container.stderr, stderr_master)
  235. }()
  236. // stdin
  237. var stdin_slave io.ReadCloser
  238. if container.Config.OpenStdin {
  239. stdin_master, stdin_slave, err := pty.Open()
  240. if err != nil {
  241. return err
  242. }
  243. container.cmd.Stdin = stdin_slave
  244. // FIXME: The following appears to be broken.
  245. // "cannot set terminal process group (-1): Inappropriate ioctl for device"
  246. // container.cmd.SysProcAttr = &syscall.SysProcAttr{Setctty: true, Setsid: true}
  247. go func() {
  248. defer container.stdin.Close()
  249. io.Copy(stdin_master, container.stdin)
  250. }()
  251. }
  252. if err := container.cmd.Start(); err != nil {
  253. return err
  254. }
  255. stdout_slave.Close()
  256. stderr_slave.Close()
  257. if stdin_slave != nil {
  258. stdin_slave.Close()
  259. }
  260. return nil
  261. }
  262. func (container *Container) start() error {
  263. container.cmd.Stdout = container.stdout
  264. container.cmd.Stderr = container.stderr
  265. if container.Config.OpenStdin {
  266. stdin, err := container.cmd.StdinPipe()
  267. if err != nil {
  268. return err
  269. }
  270. go func() {
  271. defer stdin.Close()
  272. io.Copy(stdin, container.stdin)
  273. }()
  274. }
  275. return container.cmd.Start()
  276. }
  277. func (container *Container) Start() error {
  278. if err := container.Mountpoint.EnsureMounted(); err != nil {
  279. return err
  280. }
  281. if err := container.allocateNetwork(); err != nil {
  282. return err
  283. }
  284. if err := container.generateLXCConfig(); err != nil {
  285. return err
  286. }
  287. params := []string{
  288. "-n", container.Id,
  289. "-f", container.lxcConfigPath,
  290. "--",
  291. "/sbin/init",
  292. }
  293. // Networking
  294. params = append(params, "-g", container.network.Gateway.String())
  295. // User
  296. if container.Config.User != "" {
  297. params = append(params, "-u", container.Config.User)
  298. }
  299. // Program
  300. params = append(params, "--", container.Path)
  301. params = append(params, container.Args...)
  302. container.cmd = exec.Command("/usr/bin/lxc-start", params...)
  303. var err error
  304. if container.Config.Tty {
  305. err = container.startPty()
  306. } else {
  307. err = container.start()
  308. }
  309. if err != nil {
  310. return err
  311. }
  312. container.State.setRunning(container.cmd.Process.Pid)
  313. container.save()
  314. go container.monitor()
  315. return nil
  316. }
  317. func (container *Container) Run() error {
  318. if err := container.Start(); err != nil {
  319. return err
  320. }
  321. container.Wait()
  322. return nil
  323. }
  324. func (container *Container) Output() (output []byte, err error) {
  325. pipe, err := container.StdoutPipe()
  326. if err != nil {
  327. return nil, err
  328. }
  329. defer pipe.Close()
  330. if err := container.Start(); err != nil {
  331. return nil, err
  332. }
  333. output, err = ioutil.ReadAll(pipe)
  334. container.Wait()
  335. return output, err
  336. }
  337. // StdinPipe() returns a pipe connected to the standard input of the container's
  338. // active process.
  339. //
  340. func (container *Container) StdinPipe() (io.WriteCloser, error) {
  341. return container.stdinPipe, nil
  342. }
  343. func (container *Container) StdoutPipe() (io.ReadCloser, error) {
  344. reader, writer := io.Pipe()
  345. container.stdout.AddWriter(writer)
  346. return newBufReader(reader), nil
  347. }
  348. func (container *Container) StdoutLog() io.Reader {
  349. r, err := os.Open(container.stdoutLog.Name())
  350. if err != nil {
  351. return nil
  352. }
  353. return r
  354. }
  355. func (container *Container) StderrPipe() (io.ReadCloser, error) {
  356. reader, writer := io.Pipe()
  357. container.stderr.AddWriter(writer)
  358. return newBufReader(reader), nil
  359. }
  360. func (container *Container) StderrLog() io.Reader {
  361. r, err := os.Open(container.stderrLog.Name())
  362. if err != nil {
  363. return nil
  364. }
  365. return r
  366. }
  367. func (container *Container) allocateNetwork() error {
  368. iface, err := container.networkManager.Allocate()
  369. if err != nil {
  370. return err
  371. }
  372. container.NetworkSettings.PortMapping = make(map[string]string)
  373. for _, port := range container.Config.Ports {
  374. if extPort, err := iface.AllocatePort(port); err != nil {
  375. iface.Release()
  376. return err
  377. } else {
  378. container.NetworkSettings.PortMapping[strconv.Itoa(port)] = strconv.Itoa(extPort)
  379. }
  380. }
  381. container.network = iface
  382. container.NetworkSettings.IpAddress = iface.IPNet.IP.String()
  383. container.NetworkSettings.IpPrefixLen, _ = iface.IPNet.Mask.Size()
  384. container.NetworkSettings.Gateway = iface.Gateway.String()
  385. return nil
  386. }
  387. func (container *Container) releaseNetwork() error {
  388. err := container.network.Release()
  389. container.network = nil
  390. container.NetworkSettings = &NetworkSettings{}
  391. return err
  392. }
  393. func (container *Container) monitor() {
  394. // Wait for the program to exit
  395. container.cmd.Wait()
  396. exitCode := container.cmd.ProcessState.Sys().(syscall.WaitStatus).ExitStatus()
  397. // Cleanup
  398. if err := container.releaseNetwork(); err != nil {
  399. log.Printf("%v: Failed to release network: %v", container.Id, err)
  400. }
  401. container.stdout.Close()
  402. container.stderr.Close()
  403. if err := container.Mountpoint.Umount(); err != nil {
  404. log.Printf("%v: Failed to umount filesystem: %v", container.Id, err)
  405. }
  406. // Re-create a brand new stdin pipe once the container exited
  407. if container.Config.OpenStdin {
  408. container.stdin, container.stdinPipe = io.Pipe()
  409. }
  410. // Report status back
  411. container.State.setStopped(exitCode)
  412. container.save()
  413. }
  414. func (container *Container) kill() error {
  415. if err := container.cmd.Process.Kill(); err != nil {
  416. return err
  417. }
  418. // Wait for the container to be actually stopped
  419. container.Wait()
  420. return nil
  421. }
  422. func (container *Container) Kill() error {
  423. if !container.State.Running {
  424. return nil
  425. }
  426. return container.kill()
  427. }
  428. func (container *Container) Stop() error {
  429. if !container.State.Running {
  430. return nil
  431. }
  432. // 1. Send a SIGTERM
  433. if output, err := exec.Command("/usr/bin/lxc-kill", "-n", container.Id, "15").CombinedOutput(); err != nil {
  434. log.Printf(string(output))
  435. log.Printf("Failed to send SIGTERM to the process, force killing")
  436. if err := container.Kill(); err != nil {
  437. return err
  438. }
  439. }
  440. // 2. Wait for the process to exit on its own
  441. if err := container.WaitTimeout(10 * time.Second); err != nil {
  442. log.Printf("Container %v failed to exit within 10 seconds of SIGTERM - using the force", container.Id)
  443. if err := container.Kill(); err != nil {
  444. return err
  445. }
  446. }
  447. return nil
  448. }
  449. func (container *Container) Restart() error {
  450. if err := container.Stop(); err != nil {
  451. return err
  452. }
  453. if err := container.Start(); err != nil {
  454. return err
  455. }
  456. return nil
  457. }
  458. // Wait blocks until the container stops running, then returns its exit code.
  459. func (container *Container) Wait() int {
  460. for container.State.Running {
  461. container.State.wait()
  462. }
  463. return container.State.ExitCode
  464. }
  465. func (container *Container) WaitTimeout(timeout time.Duration) error {
  466. done := make(chan bool)
  467. go func() {
  468. container.Wait()
  469. done <- true
  470. }()
  471. select {
  472. case <-time.After(timeout):
  473. return errors.New("Timed Out")
  474. case <-done:
  475. return nil
  476. }
  477. return nil
  478. }