poly1305_arm.s 9.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379
  1. // Copyright 2015 The Go Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style
  3. // license that can be found in the LICENSE file.
  4. // This code was translated into a form compatible with 5a from the public
  5. // domain source by Andrew Moon: github.com/floodyberry/poly1305-opt/blob/master/app/extensions/poly1305.
  6. // +build arm,!gccgo,!appengine
  7. DATA poly1305_init_constants_armv6<>+0x00(SB)/4, $0x3ffffff
  8. DATA poly1305_init_constants_armv6<>+0x04(SB)/4, $0x3ffff03
  9. DATA poly1305_init_constants_armv6<>+0x08(SB)/4, $0x3ffc0ff
  10. DATA poly1305_init_constants_armv6<>+0x0c(SB)/4, $0x3f03fff
  11. DATA poly1305_init_constants_armv6<>+0x10(SB)/4, $0x00fffff
  12. GLOBL poly1305_init_constants_armv6<>(SB), 8, $20
  13. // Warning: the linker may use R11 to synthesize certain instructions. Please
  14. // take care and verify that no synthetic instructions use it.
  15. TEXT poly1305_init_ext_armv6<>(SB),4,$-4
  16. MOVM.DB.W [R4-R11], (R13)
  17. MOVM.IA.W (R1), [R2-R5]
  18. MOVW $poly1305_init_constants_armv6<>(SB), R7
  19. MOVW R2, R8
  20. MOVW R2>>26, R9
  21. MOVW R3>>20, g
  22. MOVW R4>>14, R11
  23. MOVW R5>>8, R12
  24. ORR R3<<6, R9, R9
  25. ORR R4<<12, g, g
  26. ORR R5<<18, R11, R11
  27. MOVM.IA (R7), [R2-R6]
  28. AND R8, R2, R2
  29. AND R9, R3, R3
  30. AND g, R4, R4
  31. AND R11, R5, R5
  32. AND R12, R6, R6
  33. MOVM.IA.W [R2-R6], (R0)
  34. EOR R2, R2, R2
  35. EOR R3, R3, R3
  36. EOR R4, R4, R4
  37. EOR R5, R5, R5
  38. EOR R6, R6, R6
  39. MOVM.IA.W [R2-R6], (R0)
  40. MOVM.IA.W (R1), [R2-R5]
  41. MOVM.IA [R2-R6], (R0)
  42. MOVM.IA.W (R13), [R4-R11]
  43. RET
  44. #define MOVW_UNALIGNED(Rsrc, Rdst, Rtmp, offset) \
  45. MOVBU (offset+0)(Rsrc), Rtmp; \
  46. MOVBU Rtmp, (offset+0)(Rdst); \
  47. MOVBU (offset+1)(Rsrc), Rtmp; \
  48. MOVBU Rtmp, (offset+1)(Rdst); \
  49. MOVBU (offset+2)(Rsrc), Rtmp; \
  50. MOVBU Rtmp, (offset+2)(Rdst); \
  51. MOVBU (offset+3)(Rsrc), Rtmp; \
  52. MOVBU Rtmp, (offset+3)(Rdst)
  53. TEXT poly1305_blocks_armv6<>(SB),4,$-4
  54. MOVM.DB.W [R4, R5, R6, R7, R8, R9, g, R11, R14], (R13)
  55. SUB $128, R13
  56. MOVW R0, 36(R13)
  57. MOVW R1, 40(R13)
  58. MOVW R2, 44(R13)
  59. MOVW R1, R14
  60. MOVW R2, R12
  61. MOVW 56(R0), R8
  62. WORD $0xe1180008 // TST R8, R8 not working see issue 5921
  63. EOR R6, R6, R6
  64. MOVW.EQ $(1<<24), R6
  65. MOVW R6, 32(R13)
  66. ADD $64, R13, g
  67. MOVM.IA (R0), [R0-R9]
  68. MOVM.IA [R0-R4], (g)
  69. CMP $16, R12
  70. BLO poly1305_blocks_armv6_done
  71. poly1305_blocks_armv6_mainloop:
  72. WORD $0xe31e0003 // TST R14, #3 not working see issue 5921
  73. BEQ poly1305_blocks_armv6_mainloop_aligned
  74. ADD $48, R13, g
  75. MOVW_UNALIGNED(R14, g, R0, 0)
  76. MOVW_UNALIGNED(R14, g, R0, 4)
  77. MOVW_UNALIGNED(R14, g, R0, 8)
  78. MOVW_UNALIGNED(R14, g, R0, 12)
  79. MOVM.IA (g), [R0-R3]
  80. ADD $16, R14
  81. B poly1305_blocks_armv6_mainloop_loaded
  82. poly1305_blocks_armv6_mainloop_aligned:
  83. MOVM.IA.W (R14), [R0-R3]
  84. poly1305_blocks_armv6_mainloop_loaded:
  85. MOVW R0>>26, g
  86. MOVW R1>>20, R11
  87. MOVW R2>>14, R12
  88. MOVW R14, 40(R13)
  89. MOVW R3>>8, R4
  90. ORR R1<<6, g, g
  91. ORR R2<<12, R11, R11
  92. ORR R3<<18, R12, R12
  93. BIC $0xfc000000, R0, R0
  94. BIC $0xfc000000, g, g
  95. MOVW 32(R13), R3
  96. BIC $0xfc000000, R11, R11
  97. BIC $0xfc000000, R12, R12
  98. ADD R0, R5, R5
  99. ADD g, R6, R6
  100. ORR R3, R4, R4
  101. ADD R11, R7, R7
  102. ADD $64, R13, R14
  103. ADD R12, R8, R8
  104. ADD R4, R9, R9
  105. MOVM.IA (R14), [R0-R4]
  106. MULLU R4, R5, (R11, g)
  107. MULLU R3, R5, (R14, R12)
  108. MULALU R3, R6, (R11, g)
  109. MULALU R2, R6, (R14, R12)
  110. MULALU R2, R7, (R11, g)
  111. MULALU R1, R7, (R14, R12)
  112. ADD R4<<2, R4, R4
  113. ADD R3<<2, R3, R3
  114. MULALU R1, R8, (R11, g)
  115. MULALU R0, R8, (R14, R12)
  116. MULALU R0, R9, (R11, g)
  117. MULALU R4, R9, (R14, R12)
  118. MOVW g, 24(R13)
  119. MOVW R11, 28(R13)
  120. MOVW R12, 16(R13)
  121. MOVW R14, 20(R13)
  122. MULLU R2, R5, (R11, g)
  123. MULLU R1, R5, (R14, R12)
  124. MULALU R1, R6, (R11, g)
  125. MULALU R0, R6, (R14, R12)
  126. MULALU R0, R7, (R11, g)
  127. MULALU R4, R7, (R14, R12)
  128. ADD R2<<2, R2, R2
  129. ADD R1<<2, R1, R1
  130. MULALU R4, R8, (R11, g)
  131. MULALU R3, R8, (R14, R12)
  132. MULALU R3, R9, (R11, g)
  133. MULALU R2, R9, (R14, R12)
  134. MOVW g, 8(R13)
  135. MOVW R11, 12(R13)
  136. MOVW R12, 0(R13)
  137. MOVW R14, w+4(SP)
  138. MULLU R0, R5, (R11, g)
  139. MULALU R4, R6, (R11, g)
  140. MULALU R3, R7, (R11, g)
  141. MULALU R2, R8, (R11, g)
  142. MULALU R1, R9, (R11, g)
  143. MOVM.IA (R13), [R0-R7]
  144. MOVW g>>26, R12
  145. MOVW R4>>26, R14
  146. ORR R11<<6, R12, R12
  147. ORR R5<<6, R14, R14
  148. BIC $0xfc000000, g, g
  149. BIC $0xfc000000, R4, R4
  150. ADD.S R12, R0, R0
  151. ADC $0, R1, R1
  152. ADD.S R14, R6, R6
  153. ADC $0, R7, R7
  154. MOVW R0>>26, R12
  155. MOVW R6>>26, R14
  156. ORR R1<<6, R12, R12
  157. ORR R7<<6, R14, R14
  158. BIC $0xfc000000, R0, R0
  159. BIC $0xfc000000, R6, R6
  160. ADD R14<<2, R14, R14
  161. ADD.S R12, R2, R2
  162. ADC $0, R3, R3
  163. ADD R14, g, g
  164. MOVW R2>>26, R12
  165. MOVW g>>26, R14
  166. ORR R3<<6, R12, R12
  167. BIC $0xfc000000, g, R5
  168. BIC $0xfc000000, R2, R7
  169. ADD R12, R4, R4
  170. ADD R14, R0, R0
  171. MOVW R4>>26, R12
  172. BIC $0xfc000000, R4, R8
  173. ADD R12, R6, R9
  174. MOVW w+44(SP), R12
  175. MOVW w+40(SP), R14
  176. MOVW R0, R6
  177. CMP $32, R12
  178. SUB $16, R12, R12
  179. MOVW R12, 44(R13)
  180. BHS poly1305_blocks_armv6_mainloop
  181. poly1305_blocks_armv6_done:
  182. MOVW 36(R13), R12
  183. MOVW R5, 20(R12)
  184. MOVW R6, 24(R12)
  185. MOVW R7, 28(R12)
  186. MOVW R8, 32(R12)
  187. MOVW R9, 36(R12)
  188. ADD $128, R13, R13
  189. MOVM.IA.W (R13), [R4, R5, R6, R7, R8, R9, g, R11, R14]
  190. RET
  191. #define MOVHUP_UNALIGNED(Rsrc, Rdst, Rtmp) \
  192. MOVBU.P 1(Rsrc), Rtmp; \
  193. MOVBU.P Rtmp, 1(Rdst); \
  194. MOVBU.P 1(Rsrc), Rtmp; \
  195. MOVBU.P Rtmp, 1(Rdst)
  196. #define MOVWP_UNALIGNED(Rsrc, Rdst, Rtmp) \
  197. MOVHUP_UNALIGNED(Rsrc, Rdst, Rtmp); \
  198. MOVHUP_UNALIGNED(Rsrc, Rdst, Rtmp)
  199. TEXT poly1305_finish_ext_armv6<>(SB),4,$-4
  200. MOVM.DB.W [R4, R5, R6, R7, R8, R9, g, R11, R14], (R13)
  201. SUB $16, R13, R13
  202. MOVW R0, R5
  203. MOVW R1, R6
  204. MOVW R2, R7
  205. MOVW R3, R8
  206. AND.S R2, R2, R2
  207. BEQ poly1305_finish_ext_armv6_noremaining
  208. EOR R0, R0
  209. MOVW R13, R9
  210. MOVW R0, 0(R13)
  211. MOVW R0, 4(R13)
  212. MOVW R0, 8(R13)
  213. MOVW R0, 12(R13)
  214. WORD $0xe3110003 // TST R1, #3 not working see issue 5921
  215. BEQ poly1305_finish_ext_armv6_aligned
  216. WORD $0xe3120008 // TST R2, #8 not working see issue 5921
  217. BEQ poly1305_finish_ext_armv6_skip8
  218. MOVWP_UNALIGNED(R1, R9, g)
  219. MOVWP_UNALIGNED(R1, R9, g)
  220. poly1305_finish_ext_armv6_skip8:
  221. WORD $0xe3120004 // TST $4, R2 not working see issue 5921
  222. BEQ poly1305_finish_ext_armv6_skip4
  223. MOVWP_UNALIGNED(R1, R9, g)
  224. poly1305_finish_ext_armv6_skip4:
  225. WORD $0xe3120002 // TST $2, R2 not working see issue 5921
  226. BEQ poly1305_finish_ext_armv6_skip2
  227. MOVHUP_UNALIGNED(R1, R9, g)
  228. B poly1305_finish_ext_armv6_skip2
  229. poly1305_finish_ext_armv6_aligned:
  230. WORD $0xe3120008 // TST R2, #8 not working see issue 5921
  231. BEQ poly1305_finish_ext_armv6_skip8_aligned
  232. MOVM.IA.W (R1), [g-R11]
  233. MOVM.IA.W [g-R11], (R9)
  234. poly1305_finish_ext_armv6_skip8_aligned:
  235. WORD $0xe3120004 // TST $4, R2 not working see issue 5921
  236. BEQ poly1305_finish_ext_armv6_skip4_aligned
  237. MOVW.P 4(R1), g
  238. MOVW.P g, 4(R9)
  239. poly1305_finish_ext_armv6_skip4_aligned:
  240. WORD $0xe3120002 // TST $2, R2 not working see issue 5921
  241. BEQ poly1305_finish_ext_armv6_skip2
  242. MOVHU.P 2(R1), g
  243. MOVH.P g, 2(R9)
  244. poly1305_finish_ext_armv6_skip2:
  245. WORD $0xe3120001 // TST $1, R2 not working see issue 5921
  246. BEQ poly1305_finish_ext_armv6_skip1
  247. MOVBU.P 1(R1), g
  248. MOVBU.P g, 1(R9)
  249. poly1305_finish_ext_armv6_skip1:
  250. MOVW $1, R11
  251. MOVBU R11, 0(R9)
  252. MOVW R11, 56(R5)
  253. MOVW R5, R0
  254. MOVW R13, R1
  255. MOVW $16, R2
  256. BL poly1305_blocks_armv6<>(SB)
  257. poly1305_finish_ext_armv6_noremaining:
  258. MOVW 20(R5), R0
  259. MOVW 24(R5), R1
  260. MOVW 28(R5), R2
  261. MOVW 32(R5), R3
  262. MOVW 36(R5), R4
  263. MOVW R4>>26, R12
  264. BIC $0xfc000000, R4, R4
  265. ADD R12<<2, R12, R12
  266. ADD R12, R0, R0
  267. MOVW R0>>26, R12
  268. BIC $0xfc000000, R0, R0
  269. ADD R12, R1, R1
  270. MOVW R1>>26, R12
  271. BIC $0xfc000000, R1, R1
  272. ADD R12, R2, R2
  273. MOVW R2>>26, R12
  274. BIC $0xfc000000, R2, R2
  275. ADD R12, R3, R3
  276. MOVW R3>>26, R12
  277. BIC $0xfc000000, R3, R3
  278. ADD R12, R4, R4
  279. ADD $5, R0, R6
  280. MOVW R6>>26, R12
  281. BIC $0xfc000000, R6, R6
  282. ADD R12, R1, R7
  283. MOVW R7>>26, R12
  284. BIC $0xfc000000, R7, R7
  285. ADD R12, R2, g
  286. MOVW g>>26, R12
  287. BIC $0xfc000000, g, g
  288. ADD R12, R3, R11
  289. MOVW $-(1<<26), R12
  290. ADD R11>>26, R12, R12
  291. BIC $0xfc000000, R11, R11
  292. ADD R12, R4, R14
  293. MOVW R14>>31, R12
  294. SUB $1, R12
  295. AND R12, R6, R6
  296. AND R12, R7, R7
  297. AND R12, g, g
  298. AND R12, R11, R11
  299. AND R12, R14, R14
  300. MVN R12, R12
  301. AND R12, R0, R0
  302. AND R12, R1, R1
  303. AND R12, R2, R2
  304. AND R12, R3, R3
  305. AND R12, R4, R4
  306. ORR R6, R0, R0
  307. ORR R7, R1, R1
  308. ORR g, R2, R2
  309. ORR R11, R3, R3
  310. ORR R14, R4, R4
  311. ORR R1<<26, R0, R0
  312. MOVW R1>>6, R1
  313. ORR R2<<20, R1, R1
  314. MOVW R2>>12, R2
  315. ORR R3<<14, R2, R2
  316. MOVW R3>>18, R3
  317. ORR R4<<8, R3, R3
  318. MOVW 40(R5), R6
  319. MOVW 44(R5), R7
  320. MOVW 48(R5), g
  321. MOVW 52(R5), R11
  322. ADD.S R6, R0, R0
  323. ADC.S R7, R1, R1
  324. ADC.S g, R2, R2
  325. ADC.S R11, R3, R3
  326. MOVM.IA [R0-R3], (R8)
  327. MOVW R5, R12
  328. EOR R0, R0, R0
  329. EOR R1, R1, R1
  330. EOR R2, R2, R2
  331. EOR R3, R3, R3
  332. EOR R4, R4, R4
  333. EOR R5, R5, R5
  334. EOR R6, R6, R6
  335. EOR R7, R7, R7
  336. MOVM.IA.W [R0-R7], (R12)
  337. MOVM.IA [R0-R7], (R12)
  338. ADD $16, R13, R13
  339. MOVM.IA.W (R13), [R4, R5, R6, R7, R8, R9, g, R11, R14]
  340. RET
  341. // func poly1305_auth_armv6(out *[16]byte, m *byte, mlen uint32, key *[32]key)
  342. TEXT ·poly1305_auth_armv6(SB),0,$280-16
  343. MOVW out+0(FP), R4
  344. MOVW m+4(FP), R5
  345. MOVW mlen+8(FP), R6
  346. MOVW key+12(FP), R7
  347. MOVW R13, R8
  348. BIC $63, R13
  349. SUB $64, R13, R13
  350. MOVW R13, R0
  351. MOVW R7, R1
  352. BL poly1305_init_ext_armv6<>(SB)
  353. BIC.S $15, R6, R2
  354. BEQ poly1305_auth_armv6_noblocks
  355. MOVW R13, R0
  356. MOVW R5, R1
  357. ADD R2, R5, R5
  358. SUB R2, R6, R6
  359. BL poly1305_blocks_armv6<>(SB)
  360. poly1305_auth_armv6_noblocks:
  361. MOVW R13, R0
  362. MOVW R5, R1
  363. MOVW R6, R2
  364. MOVW R4, R3
  365. BL poly1305_finish_ext_armv6<>(SB)
  366. MOVW R8, R13
  367. RET