docker_cli_volume_test.go 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452
  1. package main
  2. import (
  3. "fmt"
  4. "io/ioutil"
  5. "os"
  6. "os/exec"
  7. "path/filepath"
  8. "strings"
  9. "github.com/docker/docker/integration-cli/checker"
  10. icmd "github.com/docker/docker/pkg/testutil/cmd"
  11. "github.com/go-check/check"
  12. )
  13. func (s *DockerSuite) TestVolumeCLICreate(c *check.C) {
  14. dockerCmd(c, "volume", "create")
  15. _, _, err := dockerCmdWithError("volume", "create", "-d", "nosuchdriver")
  16. c.Assert(err, check.NotNil)
  17. // test using hidden --name option
  18. out, _ := dockerCmd(c, "volume", "create", "--name=test")
  19. name := strings.TrimSpace(out)
  20. c.Assert(name, check.Equals, "test")
  21. out, _ = dockerCmd(c, "volume", "create", "test2")
  22. name = strings.TrimSpace(out)
  23. c.Assert(name, check.Equals, "test2")
  24. }
  25. func (s *DockerSuite) TestVolumeCLIInspect(c *check.C) {
  26. c.Assert(
  27. exec.Command(dockerBinary, "volume", "inspect", "doesntexist").Run(),
  28. check.Not(check.IsNil),
  29. check.Commentf("volume inspect should error on non-existent volume"),
  30. )
  31. out, _ := dockerCmd(c, "volume", "create")
  32. name := strings.TrimSpace(out)
  33. out, _ = dockerCmd(c, "volume", "inspect", "--format={{ .Name }}", name)
  34. c.Assert(strings.TrimSpace(out), check.Equals, name)
  35. dockerCmd(c, "volume", "create", "test")
  36. out, _ = dockerCmd(c, "volume", "inspect", "--format={{ .Name }}", "test")
  37. c.Assert(strings.TrimSpace(out), check.Equals, "test")
  38. }
  39. func (s *DockerSuite) TestVolumeCLIInspectMulti(c *check.C) {
  40. dockerCmd(c, "volume", "create", "test1")
  41. dockerCmd(c, "volume", "create", "test2")
  42. dockerCmd(c, "volume", "create", "not-shown")
  43. result := dockerCmdWithResult("volume", "inspect", "--format={{ .Name }}", "test1", "test2", "doesntexist", "not-shown")
  44. c.Assert(result, icmd.Matches, icmd.Expected{
  45. ExitCode: 1,
  46. Err: "No such volume: doesntexist",
  47. })
  48. out := result.Stdout()
  49. outArr := strings.Split(strings.TrimSpace(out), "\n")
  50. c.Assert(len(outArr), check.Equals, 2, check.Commentf("\n%s", out))
  51. c.Assert(out, checker.Contains, "test1")
  52. c.Assert(out, checker.Contains, "test2")
  53. c.Assert(out, checker.Not(checker.Contains), "not-shown")
  54. }
  55. func (s *DockerSuite) TestVolumeCLILs(c *check.C) {
  56. prefix, _ := getPrefixAndSlashFromDaemonPlatform()
  57. dockerCmd(c, "volume", "create", "aaa")
  58. dockerCmd(c, "volume", "create", "test")
  59. dockerCmd(c, "volume", "create", "soo")
  60. dockerCmd(c, "run", "-v", "soo:"+prefix+"/foo", "busybox", "ls", "/")
  61. out, _ := dockerCmd(c, "volume", "ls")
  62. outArr := strings.Split(strings.TrimSpace(out), "\n")
  63. c.Assert(len(outArr), check.Equals, 4, check.Commentf("\n%s", out))
  64. assertVolList(c, out, []string{"aaa", "soo", "test"})
  65. }
  66. func (s *DockerSuite) TestVolumeLsFormat(c *check.C) {
  67. dockerCmd(c, "volume", "create", "aaa")
  68. dockerCmd(c, "volume", "create", "test")
  69. dockerCmd(c, "volume", "create", "soo")
  70. out, _ := dockerCmd(c, "volume", "ls", "--format", "{{.Name}}")
  71. lines := strings.Split(strings.TrimSpace(string(out)), "\n")
  72. expected := []string{"aaa", "soo", "test"}
  73. var names []string
  74. names = append(names, lines...)
  75. c.Assert(expected, checker.DeepEquals, names, check.Commentf("Expected array with truncated names: %v, got: %v", expected, names))
  76. }
  77. func (s *DockerSuite) TestVolumeLsFormatDefaultFormat(c *check.C) {
  78. dockerCmd(c, "volume", "create", "aaa")
  79. dockerCmd(c, "volume", "create", "test")
  80. dockerCmd(c, "volume", "create", "soo")
  81. config := `{
  82. "volumesFormat": "{{ .Name }} default"
  83. }`
  84. d, err := ioutil.TempDir("", "integration-cli-")
  85. c.Assert(err, checker.IsNil)
  86. defer os.RemoveAll(d)
  87. err = ioutil.WriteFile(filepath.Join(d, "config.json"), []byte(config), 0644)
  88. c.Assert(err, checker.IsNil)
  89. out, _ := dockerCmd(c, "--config", d, "volume", "ls")
  90. lines := strings.Split(strings.TrimSpace(string(out)), "\n")
  91. expected := []string{"aaa default", "soo default", "test default"}
  92. var names []string
  93. names = append(names, lines...)
  94. c.Assert(expected, checker.DeepEquals, names, check.Commentf("Expected array with truncated names: %v, got: %v", expected, names))
  95. }
  96. // assertVolList checks volume retrieved with ls command
  97. // equals to expected volume list
  98. // note: out should be `volume ls [option]` result
  99. func assertVolList(c *check.C, out string, expectVols []string) {
  100. lines := strings.Split(out, "\n")
  101. var volList []string
  102. for _, line := range lines[1 : len(lines)-1] {
  103. volFields := strings.Fields(line)
  104. // wrap all volume name in volList
  105. volList = append(volList, volFields[1])
  106. }
  107. // volume ls should contains all expected volumes
  108. c.Assert(volList, checker.DeepEquals, expectVols)
  109. }
  110. func (s *DockerSuite) TestVolumeCLILsFilterDangling(c *check.C) {
  111. prefix, _ := getPrefixAndSlashFromDaemonPlatform()
  112. dockerCmd(c, "volume", "create", "testnotinuse1")
  113. dockerCmd(c, "volume", "create", "testisinuse1")
  114. dockerCmd(c, "volume", "create", "testisinuse2")
  115. // Make sure both "created" (but not started), and started
  116. // containers are included in reference counting
  117. dockerCmd(c, "run", "--name", "volume-test1", "-v", "testisinuse1:"+prefix+"/foo", "busybox", "true")
  118. dockerCmd(c, "create", "--name", "volume-test2", "-v", "testisinuse2:"+prefix+"/foo", "busybox", "true")
  119. out, _ := dockerCmd(c, "volume", "ls")
  120. // No filter, all volumes should show
  121. c.Assert(out, checker.Contains, "testnotinuse1\n", check.Commentf("expected volume 'testnotinuse1' in output"))
  122. c.Assert(out, checker.Contains, "testisinuse1\n", check.Commentf("expected volume 'testisinuse1' in output"))
  123. c.Assert(out, checker.Contains, "testisinuse2\n", check.Commentf("expected volume 'testisinuse2' in output"))
  124. out, _ = dockerCmd(c, "volume", "ls", "--filter", "dangling=false")
  125. // Explicitly disabling dangling
  126. c.Assert(out, check.Not(checker.Contains), "testnotinuse1\n", check.Commentf("expected volume 'testnotinuse1' in output"))
  127. c.Assert(out, checker.Contains, "testisinuse1\n", check.Commentf("expected volume 'testisinuse1' in output"))
  128. c.Assert(out, checker.Contains, "testisinuse2\n", check.Commentf("expected volume 'testisinuse2' in output"))
  129. out, _ = dockerCmd(c, "volume", "ls", "--filter", "dangling=true")
  130. // Filter "dangling" volumes; only "dangling" (unused) volumes should be in the output
  131. c.Assert(out, checker.Contains, "testnotinuse1\n", check.Commentf("expected volume 'testnotinuse1' in output"))
  132. c.Assert(out, check.Not(checker.Contains), "testisinuse1\n", check.Commentf("volume 'testisinuse1' in output, but not expected"))
  133. c.Assert(out, check.Not(checker.Contains), "testisinuse2\n", check.Commentf("volume 'testisinuse2' in output, but not expected"))
  134. out, _ = dockerCmd(c, "volume", "ls", "--filter", "dangling=1")
  135. // Filter "dangling" volumes; only "dangling" (unused) volumes should be in the output, dangling also accept 1
  136. c.Assert(out, checker.Contains, "testnotinuse1\n", check.Commentf("expected volume 'testnotinuse1' in output"))
  137. c.Assert(out, check.Not(checker.Contains), "testisinuse1\n", check.Commentf("volume 'testisinuse1' in output, but not expected"))
  138. c.Assert(out, check.Not(checker.Contains), "testisinuse2\n", check.Commentf("volume 'testisinuse2' in output, but not expected"))
  139. out, _ = dockerCmd(c, "volume", "ls", "--filter", "dangling=0")
  140. // dangling=0 is same as dangling=false case
  141. c.Assert(out, check.Not(checker.Contains), "testnotinuse1\n", check.Commentf("expected volume 'testnotinuse1' in output"))
  142. c.Assert(out, checker.Contains, "testisinuse1\n", check.Commentf("expected volume 'testisinuse1' in output"))
  143. c.Assert(out, checker.Contains, "testisinuse2\n", check.Commentf("expected volume 'testisinuse2' in output"))
  144. out, _ = dockerCmd(c, "volume", "ls", "--filter", "name=testisin")
  145. c.Assert(out, check.Not(checker.Contains), "testnotinuse1\n", check.Commentf("expected volume 'testnotinuse1' in output"))
  146. c.Assert(out, checker.Contains, "testisinuse1\n", check.Commentf("execpeted volume 'testisinuse1' in output"))
  147. c.Assert(out, checker.Contains, "testisinuse2\n", check.Commentf("expected volume 'testisinuse2' in output"))
  148. }
  149. func (s *DockerSuite) TestVolumeCLILsErrorWithInvalidFilterName(c *check.C) {
  150. out, _, err := dockerCmdWithError("volume", "ls", "-f", "FOO=123")
  151. c.Assert(err, checker.NotNil)
  152. c.Assert(out, checker.Contains, "Invalid filter")
  153. }
  154. func (s *DockerSuite) TestVolumeCLILsWithIncorrectFilterValue(c *check.C) {
  155. out, _, err := dockerCmdWithError("volume", "ls", "-f", "dangling=invalid")
  156. c.Assert(err, check.NotNil)
  157. c.Assert(out, checker.Contains, "Invalid filter")
  158. }
  159. func (s *DockerSuite) TestVolumeCLIRm(c *check.C) {
  160. prefix, _ := getPrefixAndSlashFromDaemonPlatform()
  161. out, _ := dockerCmd(c, "volume", "create")
  162. id := strings.TrimSpace(out)
  163. dockerCmd(c, "volume", "create", "test")
  164. dockerCmd(c, "volume", "rm", id)
  165. dockerCmd(c, "volume", "rm", "test")
  166. out, _ = dockerCmd(c, "volume", "ls")
  167. outArr := strings.Split(strings.TrimSpace(out), "\n")
  168. c.Assert(len(outArr), check.Equals, 1, check.Commentf("%s\n", out))
  169. volumeID := "testing"
  170. dockerCmd(c, "run", "-v", volumeID+":"+prefix+"/foo", "--name=test", "busybox", "sh", "-c", "echo hello > /foo/bar")
  171. out, _, err := runCommandWithOutput(exec.Command(dockerBinary, "volume", "rm", "testing"))
  172. c.Assert(
  173. err,
  174. check.Not(check.IsNil),
  175. check.Commentf("Should not be able to remove volume that is in use by a container\n%s", out))
  176. out, _ = dockerCmd(c, "run", "--volumes-from=test", "--name=test2", "busybox", "sh", "-c", "cat /foo/bar")
  177. c.Assert(strings.TrimSpace(out), check.Equals, "hello")
  178. dockerCmd(c, "rm", "-fv", "test2")
  179. dockerCmd(c, "volume", "inspect", volumeID)
  180. dockerCmd(c, "rm", "-f", "test")
  181. out, _ = dockerCmd(c, "run", "--name=test2", "-v", volumeID+":"+prefix+"/foo", "busybox", "sh", "-c", "cat /foo/bar")
  182. c.Assert(strings.TrimSpace(out), check.Equals, "hello", check.Commentf("volume data was removed"))
  183. dockerCmd(c, "rm", "test2")
  184. dockerCmd(c, "volume", "rm", volumeID)
  185. c.Assert(
  186. exec.Command("volume", "rm", "doesntexist").Run(),
  187. check.Not(check.IsNil),
  188. check.Commentf("volume rm should fail with non-existent volume"),
  189. )
  190. }
  191. // FIXME(vdemeester) should be a unit test in cli/command/volume package
  192. func (s *DockerSuite) TestVolumeCLINoArgs(c *check.C) {
  193. out, _ := dockerCmd(c, "volume")
  194. // no args should produce the cmd usage output
  195. usage := "Usage: docker volume COMMAND"
  196. c.Assert(out, checker.Contains, usage)
  197. // invalid arg should error and show the command usage on stderr
  198. icmd.RunCommand(dockerBinary, "volume", "somearg").Assert(c, icmd.Expected{
  199. ExitCode: 1,
  200. Error: "exit status 1",
  201. Err: usage,
  202. })
  203. // invalid flag should error and show the flag error and cmd usage
  204. result := icmd.RunCommand(dockerBinary, "volume", "--no-such-flag")
  205. result.Assert(c, icmd.Expected{
  206. ExitCode: 125,
  207. Error: "exit status 125",
  208. Err: usage,
  209. })
  210. c.Assert(result.Stderr(), checker.Contains, "unknown flag: --no-such-flag")
  211. }
  212. func (s *DockerSuite) TestVolumeCLIInspectTmplError(c *check.C) {
  213. out, _ := dockerCmd(c, "volume", "create")
  214. name := strings.TrimSpace(out)
  215. out, exitCode, err := dockerCmdWithError("volume", "inspect", "--format='{{ .FooBar }}'", name)
  216. c.Assert(err, checker.NotNil, check.Commentf("Output: %s", out))
  217. c.Assert(exitCode, checker.Equals, 1, check.Commentf("Output: %s", out))
  218. c.Assert(out, checker.Contains, "Template parsing error")
  219. }
  220. func (s *DockerSuite) TestVolumeCLICreateWithOpts(c *check.C) {
  221. testRequires(c, DaemonIsLinux)
  222. dockerCmd(c, "volume", "create", "-d", "local", "test", "--opt=type=tmpfs", "--opt=device=tmpfs", "--opt=o=size=1m,uid=1000")
  223. out, _ := dockerCmd(c, "run", "-v", "test:/foo", "busybox", "mount")
  224. mounts := strings.Split(out, "\n")
  225. var found bool
  226. for _, m := range mounts {
  227. if strings.Contains(m, "/foo") {
  228. found = true
  229. info := strings.Fields(m)
  230. // tmpfs on <path> type tmpfs (rw,relatime,size=1024k,uid=1000)
  231. c.Assert(info[0], checker.Equals, "tmpfs")
  232. c.Assert(info[2], checker.Equals, "/foo")
  233. c.Assert(info[4], checker.Equals, "tmpfs")
  234. c.Assert(info[5], checker.Contains, "uid=1000")
  235. c.Assert(info[5], checker.Contains, "size=1024k")
  236. break
  237. }
  238. }
  239. c.Assert(found, checker.Equals, true)
  240. }
  241. func (s *DockerSuite) TestVolumeCLICreateLabel(c *check.C) {
  242. testVol := "testvolcreatelabel"
  243. testLabel := "foo"
  244. testValue := "bar"
  245. out, _, err := dockerCmdWithError("volume", "create", "--label", testLabel+"="+testValue, testVol)
  246. c.Assert(err, check.IsNil)
  247. out, _ = dockerCmd(c, "volume", "inspect", "--format={{ .Labels."+testLabel+" }}", testVol)
  248. c.Assert(strings.TrimSpace(out), check.Equals, testValue)
  249. }
  250. func (s *DockerSuite) TestVolumeCLICreateLabelMultiple(c *check.C) {
  251. testVol := "testvolcreatelabel"
  252. testLabels := map[string]string{
  253. "foo": "bar",
  254. "baz": "foo",
  255. }
  256. args := []string{
  257. "volume",
  258. "create",
  259. testVol,
  260. }
  261. for k, v := range testLabels {
  262. args = append(args, "--label", k+"="+v)
  263. }
  264. out, _, err := dockerCmdWithError(args...)
  265. c.Assert(err, check.IsNil)
  266. for k, v := range testLabels {
  267. out, _ = dockerCmd(c, "volume", "inspect", "--format={{ .Labels."+k+" }}", testVol)
  268. c.Assert(strings.TrimSpace(out), check.Equals, v)
  269. }
  270. }
  271. func (s *DockerSuite) TestVolumeCLILsFilterLabels(c *check.C) {
  272. testVol1 := "testvolcreatelabel-1"
  273. out, _, err := dockerCmdWithError("volume", "create", "--label", "foo=bar1", testVol1)
  274. c.Assert(err, check.IsNil)
  275. testVol2 := "testvolcreatelabel-2"
  276. out, _, err = dockerCmdWithError("volume", "create", "--label", "foo=bar2", testVol2)
  277. c.Assert(err, check.IsNil)
  278. out, _ = dockerCmd(c, "volume", "ls", "--filter", "label=foo")
  279. // filter with label=key
  280. c.Assert(out, checker.Contains, "testvolcreatelabel-1\n", check.Commentf("expected volume 'testvolcreatelabel-1' in output"))
  281. c.Assert(out, checker.Contains, "testvolcreatelabel-2\n", check.Commentf("expected volume 'testvolcreatelabel-2' in output"))
  282. out, _ = dockerCmd(c, "volume", "ls", "--filter", "label=foo=bar1")
  283. // filter with label=key=value
  284. c.Assert(out, checker.Contains, "testvolcreatelabel-1\n", check.Commentf("expected volume 'testvolcreatelabel-1' in output"))
  285. c.Assert(out, check.Not(checker.Contains), "testvolcreatelabel-2\n", check.Commentf("expected volume 'testvolcreatelabel-2 in output"))
  286. out, _ = dockerCmd(c, "volume", "ls", "--filter", "label=non-exist")
  287. outArr := strings.Split(strings.TrimSpace(out), "\n")
  288. c.Assert(len(outArr), check.Equals, 1, check.Commentf("\n%s", out))
  289. out, _ = dockerCmd(c, "volume", "ls", "--filter", "label=foo=non-exist")
  290. outArr = strings.Split(strings.TrimSpace(out), "\n")
  291. c.Assert(len(outArr), check.Equals, 1, check.Commentf("\n%s", out))
  292. }
  293. func (s *DockerSuite) TestVolumeCLILsFilterDrivers(c *check.C) {
  294. // using default volume driver local to create volumes
  295. testVol1 := "testvol-1"
  296. out, _, err := dockerCmdWithError("volume", "create", testVol1)
  297. c.Assert(err, check.IsNil)
  298. testVol2 := "testvol-2"
  299. out, _, err = dockerCmdWithError("volume", "create", testVol2)
  300. c.Assert(err, check.IsNil)
  301. // filter with driver=local
  302. out, _ = dockerCmd(c, "volume", "ls", "--filter", "driver=local")
  303. c.Assert(out, checker.Contains, "testvol-1\n", check.Commentf("expected volume 'testvol-1' in output"))
  304. c.Assert(out, checker.Contains, "testvol-2\n", check.Commentf("expected volume 'testvol-2' in output"))
  305. // filter with driver=invaliddriver
  306. out, _ = dockerCmd(c, "volume", "ls", "--filter", "driver=invaliddriver")
  307. outArr := strings.Split(strings.TrimSpace(out), "\n")
  308. c.Assert(len(outArr), check.Equals, 1, check.Commentf("\n%s", out))
  309. // filter with driver=loca
  310. out, _ = dockerCmd(c, "volume", "ls", "--filter", "driver=loca")
  311. outArr = strings.Split(strings.TrimSpace(out), "\n")
  312. c.Assert(len(outArr), check.Equals, 1, check.Commentf("\n%s", out))
  313. // filter with driver=
  314. out, _ = dockerCmd(c, "volume", "ls", "--filter", "driver=")
  315. outArr = strings.Split(strings.TrimSpace(out), "\n")
  316. c.Assert(len(outArr), check.Equals, 1, check.Commentf("\n%s", out))
  317. }
  318. func (s *DockerSuite) TestVolumeCLIRmForceUsage(c *check.C) {
  319. out, _ := dockerCmd(c, "volume", "create")
  320. id := strings.TrimSpace(out)
  321. dockerCmd(c, "volume", "rm", "-f", id)
  322. dockerCmd(c, "volume", "rm", "--force", "nonexist")
  323. out, _ = dockerCmd(c, "volume", "ls")
  324. outArr := strings.Split(strings.TrimSpace(out), "\n")
  325. c.Assert(len(outArr), check.Equals, 1, check.Commentf("%s\n", out))
  326. }
  327. func (s *DockerSuite) TestVolumeCLIRmForce(c *check.C) {
  328. testRequires(c, SameHostDaemon, DaemonIsLinux)
  329. name := "test"
  330. out, _ := dockerCmd(c, "volume", "create", name)
  331. id := strings.TrimSpace(out)
  332. c.Assert(id, checker.Equals, name)
  333. out, _ = dockerCmd(c, "volume", "inspect", "--format", "{{.Mountpoint}}", name)
  334. c.Assert(strings.TrimSpace(out), checker.Not(checker.Equals), "")
  335. // Mountpoint is in the form of "/var/lib/docker/volumes/.../_data", removing `/_data`
  336. path := strings.TrimSuffix(strings.TrimSpace(out), "/_data")
  337. out, _, err := runCommandWithOutput(exec.Command("rm", "-rf", path))
  338. c.Assert(err, check.IsNil)
  339. dockerCmd(c, "volume", "rm", "-f", "test")
  340. out, _ = dockerCmd(c, "volume", "ls")
  341. c.Assert(out, checker.Not(checker.Contains), name)
  342. dockerCmd(c, "volume", "create", "test")
  343. out, _ = dockerCmd(c, "volume", "ls")
  344. c.Assert(out, checker.Contains, name)
  345. }
  346. func (s *DockerSuite) TestVolumeCliInspectWithVolumeOpts(c *check.C) {
  347. testRequires(c, DaemonIsLinux)
  348. // Without options
  349. name := "test1"
  350. dockerCmd(c, "volume", "create", "-d", "local", name)
  351. out, _ := dockerCmd(c, "volume", "inspect", "--format={{ .Options }}", name)
  352. c.Assert(strings.TrimSpace(out), checker.Contains, "map[]")
  353. // With options
  354. name = "test2"
  355. k1, v1 := "type", "tmpfs"
  356. k2, v2 := "device", "tmpfs"
  357. k3, v3 := "o", "size=1m,uid=1000"
  358. dockerCmd(c, "volume", "create", "-d", "local", name, "--opt", fmt.Sprintf("%s=%s", k1, v1), "--opt", fmt.Sprintf("%s=%s", k2, v2), "--opt", fmt.Sprintf("%s=%s", k3, v3))
  359. out, _ = dockerCmd(c, "volume", "inspect", "--format={{ .Options }}", name)
  360. c.Assert(strings.TrimSpace(out), checker.Contains, fmt.Sprintf("%s:%s", k1, v1))
  361. c.Assert(strings.TrimSpace(out), checker.Contains, fmt.Sprintf("%s:%s", k2, v2))
  362. c.Assert(strings.TrimSpace(out), checker.Contains, fmt.Sprintf("%s:%s", k3, v3))
  363. }