1 // Copyright 2018 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 //go:build !purego
6
7 #include "textflag.h"
8
9 #define B0 V0
10 #define B1 V1
11 #define B2 V2
12 #define B3 V3
13 #define B4 V4
14 #define B5 V5
15 #define B6 V6
16 #define B7 V7
17
18 #define ACC0 V8
19 #define ACC1 V9
20 #define ACCM V10
21
22 #define T0 V11
23 #define T1 V12
24 #define T2 V13
25 #define T3 V14
26
27 #define POLY V15
28 #define ZERO V16
29 #define INC V17
30 #define CTR V18
31
32 #define K0 V19
33 #define K1 V20
34 #define K2 V21
35 #define K3 V22
36 #define K4 V23
37 #define K5 V24
38 #define K6 V25
39 #define K7 V26
40 #define K8 V27
41 #define K9 V28
42 #define K10 V29
43 #define K11 V30
44 #define KLAST V31
45
46 #define reduce() \
47 VEOR ACC0.B16, ACCM.B16, ACCM.B16 \
48 VEOR ACC1.B16, ACCM.B16, ACCM.B16 \
49 VEXT $8, ZERO.B16, ACCM.B16, T0.B16 \
50 VEXT $8, ACCM.B16, ZERO.B16, ACCM.B16 \
51 VEOR ACCM.B16, ACC0.B16, ACC0.B16 \
52 VEOR T0.B16, ACC1.B16, ACC1.B16 \
53 VPMULL POLY.D1, ACC0.D1, T0.Q1 \
54 VEXT $8, ACC0.B16, ACC0.B16, ACC0.B16 \
55 VEOR T0.B16, ACC0.B16, ACC0.B16 \
56 VPMULL POLY.D1, ACC0.D1, T0.Q1 \
57 VEOR T0.B16, ACC1.B16, ACC1.B16 \
58 VEXT $8, ACC1.B16, ACC1.B16, ACC1.B16 \
59 VEOR ACC1.B16, ACC0.B16, ACC0.B16 \
60
61 // reduce3 performs reduce using EOR3 if it is available.
62 #define reduce3() \
63 VEOR3 ACC0.B16, ACC1.B16, ACCM.B16, ACCM.B16 \
64 VEXT $8, ZERO.B16, ACCM.B16, T0.B16 \
65 VEXT $8, ACCM.B16, ZERO.B16, ACCM.B16 \
66 VEOR ACCM.B16, ACC0.B16, ACC0.B16 \
67 VEOR T0.B16, ACC1.B16, ACC1.B16 \
68 VPMULL POLY.D1, ACC0.D1, T0.Q1 \
69 VEXT $8, ACC0.B16, ACC0.B16, ACC0.B16 \
70 VEOR T0.B16, ACC0.B16, ACC0.B16 \
71 VPMULL POLY.D1, ACC0.D1, T0.Q1 \
72 VEOR T0.B16, ACC1.B16, ACC1.B16 \
73 VEXT $8, ACC1.B16, ACC1.B16, ACC1.B16 \
74 VEOR ACC1.B16, ACC0.B16, ACC0.B16 \
75
76 // func gcmAesFinish(productTable *[256]byte, tagMask, T *[16]byte, pLen, dLen uint64)
77 TEXT ·gcmAesFinish(SB),NOSPLIT,$0
78 #define pTbl R0
79 #define tMsk R1
80 #define tPtr R2
81 #define plen R3
82 #define dlen R4
83
84 MOVD $0xC2, R1
85 LSL $56, R1
86 MOVD $1, R0
87 VMOV R1, POLY.D[0]
88 VMOV R0, POLY.D[1]
89 VEOR ZERO.B16, ZERO.B16, ZERO.B16
90
91 MOVD productTable+0(FP), pTbl
92 MOVD tagMask+8(FP), tMsk
93 MOVD T+16(FP), tPtr
94 MOVD pLen+24(FP), plen
95 MOVD dLen+32(FP), dlen
96
97 VLD1 (tPtr), [ACC0.B16]
98 VLD1 (tMsk), [B1.B16]
99
100 LSL $3, plen
101 LSL $3, dlen
102
103 VMOV dlen, B0.D[0]
104 VMOV plen, B0.D[1]
105
106 ADD $14*16, pTbl
107 VLD1.P (pTbl), [T1.B16, T2.B16]
108
109 VEOR ACC0.B16, B0.B16, B0.B16
110
111 VEXT $8, B0.B16, B0.B16, T0.B16
112 VEOR B0.B16, T0.B16, T0.B16
113 VPMULL B0.D1, T1.D1, ACC1.Q1
114 VPMULL2 B0.D2, T1.D2, ACC0.Q1
115 VPMULL T0.D1, T2.D1, ACCM.Q1
116
117 reduce()
118
119 VREV64 ACC0.B16, ACC0.B16
120 VEOR B1.B16, ACC0.B16, ACC0.B16
121
122 VST1 [ACC0.B16], (tPtr)
123 RET
124 #undef pTbl
125 #undef tMsk
126 #undef tPtr
127 #undef plen
128 #undef dlen
129
130 // func gcmAesInit(productTable *[256]byte, ks []uint32)
131 TEXT ·gcmAesInit(SB),NOSPLIT,$0
132 #define pTbl R0
133 #define KS R1
134 #define NR R2
135 #define I R3
136 MOVD productTable+0(FP), pTbl
137 MOVD ks_base+8(FP), KS
138 MOVD ks_len+16(FP), NR
139
140 MOVD $0xC2, I
141 LSL $56, I
142 VMOV I, POLY.D[0]
143 MOVD $1, I
144 VMOV I, POLY.D[1]
145 VEOR ZERO.B16, ZERO.B16, ZERO.B16
146
147 // Encrypt block 0 with the AES key to generate the hash key H
148 VLD1.P 64(KS), [T0.B16, T1.B16, T2.B16, T3.B16]
149 VEOR B0.B16, B0.B16, B0.B16
150 AESE T0.B16, B0.B16
151 AESMC B0.B16, B0.B16
152 AESE T1.B16, B0.B16
153 AESMC B0.B16, B0.B16
154 AESE T2.B16, B0.B16
155 AESMC B0.B16, B0.B16
156 AESE T3.B16, B0.B16
157 AESMC B0.B16, B0.B16
158 VLD1.P 64(KS), [T0.B16, T1.B16, T2.B16, T3.B16]
159 AESE T0.B16, B0.B16
160 AESMC B0.B16, B0.B16
161 AESE T1.B16, B0.B16
162 AESMC B0.B16, B0.B16
163 AESE T2.B16, B0.B16
164 AESMC B0.B16, B0.B16
165 AESE T3.B16, B0.B16
166 AESMC B0.B16, B0.B16
167 TBZ $4, NR, initEncFinish
168 VLD1.P 32(KS), [T0.B16, T1.B16]
169 AESE T0.B16, B0.B16
170 AESMC B0.B16, B0.B16
171 AESE T1.B16, B0.B16
172 AESMC B0.B16, B0.B16
173 TBZ $3, NR, initEncFinish
174 VLD1.P 32(KS), [T0.B16, T1.B16]
175 AESE T0.B16, B0.B16
176 AESMC B0.B16, B0.B16
177 AESE T1.B16, B0.B16
178 AESMC B0.B16, B0.B16
179 initEncFinish:
180 VLD1 (KS), [T0.B16, T1.B16, T2.B16]
181 AESE T0.B16, B0.B16
182 AESMC B0.B16, B0.B16
183 AESE T1.B16, B0.B16
184 VEOR T2.B16, B0.B16, B0.B16
185
186 VREV64 B0.B16, B0.B16
187
188 // Multiply by 2 modulo P
189 VMOV B0.D[0], I
190 ASR $63, I
191 VMOV I, T1.D[0]
192 VMOV I, T1.D[1]
193 VAND POLY.B16, T1.B16, T1.B16
194 VUSHR $63, B0.D2, T2.D2
195 VEXT $8, ZERO.B16, T2.B16, T2.B16
196 VSHL $1, B0.D2, B0.D2
197 VEOR T1.B16, B0.B16, B0.B16
198 VEOR T2.B16, B0.B16, B0.B16 // Can avoid this when VSLI is available
199
200 // Karatsuba pre-computation
201 VEXT $8, B0.B16, B0.B16, B1.B16
202 VEOR B0.B16, B1.B16, B1.B16
203
204 ADD $14*16, pTbl
205 VST1 [B0.B16, B1.B16], (pTbl)
206 SUB $2*16, pTbl
207
208 VMOV B0.B16, B2.B16
209 VMOV B1.B16, B3.B16
210
211 MOVD $7, I
212
213 initLoop:
214 // Compute powers of H
215 SUBS $1, I
216
217 VPMULL B0.D1, B2.D1, T1.Q1
218 VPMULL2 B0.D2, B2.D2, T0.Q1
219 VPMULL B1.D1, B3.D1, T2.Q1
220 VEOR T0.B16, T2.B16, T2.B16
221 VEOR T1.B16, T2.B16, T2.B16
222 VEXT $8, ZERO.B16, T2.B16, T3.B16
223 VEXT $8, T2.B16, ZERO.B16, T2.B16
224 VEOR T2.B16, T0.B16, T0.B16
225 VEOR T3.B16, T1.B16, T1.B16
226 VPMULL POLY.D1, T0.D1, T2.Q1
227 VEXT $8, T0.B16, T0.B16, T0.B16
228 VEOR T2.B16, T0.B16, T0.B16
229 VPMULL POLY.D1, T0.D1, T2.Q1
230 VEXT $8, T0.B16, T0.B16, T0.B16
231 VEOR T2.B16, T0.B16, T0.B16
232 VEOR T1.B16, T0.B16, B2.B16
233 VMOV B2.B16, B3.B16
234 VEXT $8, B2.B16, B2.B16, B2.B16
235 VEOR B2.B16, B3.B16, B3.B16
236
237 VST1 [B2.B16, B3.B16], (pTbl)
238 SUB $2*16, pTbl
239
240 BNE initLoop
241 RET
242 #undef I
243 #undef NR
244 #undef KS
245 #undef pTbl
246
247 // func gcmAesData(productTable *[256]byte, data []byte, T *[16]byte)
248 TEXT ·gcmAesData(SB),NOSPLIT,$0
249 #define pTbl R0
250 #define aut R1
251 #define tPtr R2
252 #define autLen R3
253 #define H0 R4
254 #define pTblSave R5
255
256 #define mulRound(X) \
257 VLD1.P 32(pTbl), [T1.B16, T2.B16] \
258 VREV64 X.B16, X.B16 \
259 VEXT $8, X.B16, X.B16, T0.B16 \
260 VEOR X.B16, T0.B16, T0.B16 \
261 VPMULL X.D1, T1.D1, T3.Q1 \
262 VEOR T3.B16, ACC1.B16, ACC1.B16 \
263 VPMULL2 X.D2, T1.D2, T3.Q1 \
264 VEOR T3.B16, ACC0.B16, ACC0.B16 \
265 VPMULL T0.D1, T2.D1, T3.Q1 \
266 VEOR T3.B16, ACCM.B16, ACCM.B16
267
268 MOVD productTable+0(FP), pTbl
269 MOVD data_base+8(FP), aut
270 MOVD data_len+16(FP), autLen
271 MOVD T+32(FP), tPtr
272
273 VEOR ACC0.B16, ACC0.B16, ACC0.B16
274 CBZ autLen, dataBail
275
276 MOVD $0xC2, H0
277 LSL $56, H0
278 VMOV H0, POLY.D[0]
279 MOVD $1, H0
280 VMOV H0, POLY.D[1]
281 VEOR ZERO.B16, ZERO.B16, ZERO.B16
282 MOVD pTbl, pTblSave
283
284 CMP $13, autLen
285 BEQ dataTLS
286 CMP $128, autLen
287 BLT startSinglesLoop
288 B octetsLoop
289
290 dataTLS:
291 ADD $14*16, pTbl
292 VLD1.P (pTbl), [T1.B16, T2.B16]
293 VEOR B0.B16, B0.B16, B0.B16
294
295 MOVD (aut), H0
296 VMOV H0, B0.D[0]
297 MOVW 8(aut), H0
298 VMOV H0, B0.S[2]
299 MOVB 12(aut), H0
300 VMOV H0, B0.B[12]
301
302 MOVD $0, autLen
303 B dataMul
304
305 octetsLoop:
306 CMP $128, autLen
307 BLT startSinglesLoop
308 SUB $128, autLen
309
310 VLD1.P 32(aut), [B0.B16, B1.B16]
311
312 VLD1.P 32(pTbl), [T1.B16, T2.B16]
313 VREV64 B0.B16, B0.B16
314 VEOR ACC0.B16, B0.B16, B0.B16
315 VEXT $8, B0.B16, B0.B16, T0.B16
316 VEOR B0.B16, T0.B16, T0.B16
317 VPMULL B0.D1, T1.D1, ACC1.Q1
318 VPMULL2 B0.D2, T1.D2, ACC0.Q1
319 VPMULL T0.D1, T2.D1, ACCM.Q1
320
321 mulRound(B1)
322 VLD1.P 32(aut), [B2.B16, B3.B16]
323 mulRound(B2)
324 mulRound(B3)
325 VLD1.P 32(aut), [B4.B16, B5.B16]
326 mulRound(B4)
327 mulRound(B5)
328 VLD1.P 32(aut), [B6.B16, B7.B16]
329 mulRound(B6)
330 mulRound(B7)
331
332 MOVD pTblSave, pTbl
333 reduce()
334 B octetsLoop
335
336 startSinglesLoop:
337
338 ADD $14*16, pTbl
339 VLD1.P (pTbl), [T1.B16, T2.B16]
340
341 singlesLoop:
342
343 CMP $16, autLen
344 BLT dataEnd
345 SUB $16, autLen
346
347 VLD1.P 16(aut), [B0.B16]
348 dataMul:
349 VREV64 B0.B16, B0.B16
350 VEOR ACC0.B16, B0.B16, B0.B16
351
352 VEXT $8, B0.B16, B0.B16, T0.B16
353 VEOR B0.B16, T0.B16, T0.B16
354 VPMULL B0.D1, T1.D1, ACC1.Q1
355 VPMULL2 B0.D2, T1.D2, ACC0.Q1
356 VPMULL T0.D1, T2.D1, ACCM.Q1
357
358 reduce()
359
360 B singlesLoop
361
362 dataEnd:
363
364 CBZ autLen, dataBail
365 VEOR B0.B16, B0.B16, B0.B16
366 ADD autLen, aut
367
368 dataLoadLoop:
369 MOVB.W -1(aut), H0
370 VEXT $15, B0.B16, ZERO.B16, B0.B16
371 VMOV H0, B0.B[0]
372 SUBS $1, autLen
373 BNE dataLoadLoop
374 B dataMul
375
376 dataBail:
377 VST1 [ACC0.B16], (tPtr)
378 RET
379
380 #undef pTbl
381 #undef aut
382 #undef tPtr
383 #undef autLen
384 #undef H0
385 #undef pTblSave
386
387 // func gcmAesEnc(productTable *[256]byte, dst, src []byte, ctr, T *[16]byte, ks []uint32)
388 TEXT ·gcmAesEnc(SB),NOSPLIT,$0
389 #define pTbl R0
390 #define dstPtr R1
391 #define ctrPtr R2
392 #define srcPtr R3
393 #define ks R4
394 #define tPtr R5
395 #define srcPtrLen R6
396 #define NR R10
397 #define H0 R11
398 #define H1 R12
399 #define curK R13
400 #define pTblSave R14
401
402 #define aesrndx8(K) \
403 AESE K.B16, B0.B16 \
404 AESMC B0.B16, B0.B16 \
405 AESE K.B16, B1.B16 \
406 AESMC B1.B16, B1.B16 \
407 AESE K.B16, B2.B16 \
408 AESMC B2.B16, B2.B16 \
409 AESE K.B16, B3.B16 \
410 AESMC B3.B16, B3.B16 \
411 AESE K.B16, B4.B16 \
412 AESMC B4.B16, B4.B16 \
413 AESE K.B16, B5.B16 \
414 AESMC B5.B16, B5.B16 \
415 AESE K.B16, B6.B16 \
416 AESMC B6.B16, B6.B16 \
417 AESE K.B16, B7.B16 \
418 AESMC B7.B16, B7.B16
419
420 #define aesrndlastx8(K) \
421 AESE K.B16, B0.B16 \
422 AESE K.B16, B1.B16 \
423 AESE K.B16, B2.B16 \
424 AESE K.B16, B3.B16 \
425 AESE K.B16, B4.B16 \
426 AESE K.B16, B5.B16 \
427 AESE K.B16, B6.B16 \
428 AESE K.B16, B7.B16
429
430 // tailLoad reads the srcPtrLen bytes at srcPtr into the low bytes of
431 // X, zero-padded, loading 8, 4, 2, and 1 bytes at a time to avoid
432 // reading past the end of the source buffer. It also builds in T3 a
433 // mask of the bytes within the source length. It clobbers H0 and H1.
434 #define tailLoad(X) \
435 VEOR X.B16, X.B16, X.B16 \
436 VEOR T3.B16, T3.B16, T3.B16 \
437 MOVD $-1, H1 \
438 ADD srcPtrLen, srcPtr \
439 TBZ $3, srcPtrLen, tailLoad4 \
440 MOVD.W -8(srcPtr), H0 \
441 VMOV H0, X.D[0] \
442 VMOV H1, T3.D[0] \
443 tailLoad4: \
444 TBZ $2, srcPtrLen, tailLoad2 \
445 MOVW.W -4(srcPtr), H0 \
446 VEXT $12, X.B16, ZERO.B16, X.B16 \
447 VEXT $12, T3.B16, ZERO.B16, T3.B16 \
448 VMOV H0, X.S[0] \
449 VMOV H1, T3.S[0] \
450 tailLoad2: \
451 TBZ $1, srcPtrLen, tailLoad1 \
452 MOVH.W -2(srcPtr), H0 \
453 VEXT $14, X.B16, ZERO.B16, X.B16 \
454 VEXT $14, T3.B16, ZERO.B16, T3.B16 \
455 VMOV H0, X.H[0] \
456 VMOV H1, T3.H[0] \
457 tailLoad1: \
458 TBZ $0, srcPtrLen, tailLoad0 \
459 MOVB.W -1(srcPtr), H0 \
460 VEXT $15, X.B16, ZERO.B16, X.B16 \
461 VEXT $15, T3.B16, ZERO.B16, T3.B16 \
462 VMOV H0, X.B[0] \
463 VMOV H1, T3.B[0] \
464 tailLoad0:
465
466 // tailStore writes the low srcPtrLen bytes of X to dstPtr, storing 8,
467 // 4, 2, and 1 bytes at a time to avoid writing past the end of the
468 // destination buffer. It clobbers X and H0.
469 #define tailStore(X) \
470 TBZ $3, srcPtrLen, tailStore4 \
471 VMOV X.D[0], H0 \
472 MOVD.P H0, 8(dstPtr) \
473 VEXT $8, ZERO.B16, X.B16, X.B16 \
474 tailStore4: \
475 TBZ $2, srcPtrLen, tailStore2 \
476 VMOV X.S[0], H0 \
477 MOVW.P H0, 4(dstPtr) \
478 VEXT $4, ZERO.B16, X.B16, X.B16 \
479 tailStore2: \
480 TBZ $1, srcPtrLen, tailStore1 \
481 VMOV X.H[0], H0 \
482 MOVH.P H0, 2(dstPtr) \
483 VEXT $2, ZERO.B16, X.B16, X.B16 \
484 tailStore1: \
485 TBZ $0, srcPtrLen, tailStore0 \
486 VMOV X.B[0], H0 \
487 MOVB.P H0, 1(dstPtr) \
488 tailStore0:
489
490 MOVD productTable+0(FP), pTbl
491 MOVD dst+8(FP), dstPtr
492 MOVD src_base+32(FP), srcPtr
493 MOVD src_len+40(FP), srcPtrLen
494 MOVD ctr+56(FP), ctrPtr
495 MOVD T+64(FP), tPtr
496 MOVD ks_base+72(FP), ks
497 MOVD ks_len+80(FP), NR
498
499 MOVD $0xC2, H1
500 LSL $56, H1
501 MOVD $1, H0
502 VMOV H1, POLY.D[0]
503 VMOV H0, POLY.D[1]
504 VEOR ZERO.B16, ZERO.B16, ZERO.B16
505 // Compute NR from len(ks)
506 MOVD pTbl, pTblSave
507 // Current tag, after AAD
508 VLD1 (tPtr), [ACC0.B16]
509 VEOR ACC1.B16, ACC1.B16, ACC1.B16
510 VEOR ACCM.B16, ACCM.B16, ACCM.B16
511 // Prepare initial counter, and the increment vector
512 VLD1 (ctrPtr), [CTR.B16]
513 VEOR INC.B16, INC.B16, INC.B16
514 MOVD $1, H0
515 VMOV H0, INC.S[3]
516 VREV32 CTR.B16, CTR.B16
517 VADD CTR.S4, INC.S4, CTR.S4
518 // Skip to <8 blocks loop
519 CMP $128, srcPtrLen
520
521 MOVD ks, H0
522 // For AES-128 round keys are stored in: K0 .. K10, KLAST
523 VLD1.P 64(H0), [K0.B16, K1.B16, K2.B16, K3.B16]
524 VLD1.P 64(H0), [K4.B16, K5.B16, K6.B16, K7.B16]
525 VLD1.P 48(H0), [K8.B16, K9.B16, K10.B16]
526 VMOV K10.B16, KLAST.B16
527
528 BLT startSingles
529 // There are at least 8 blocks to encrypt
530 TBZ $4, NR, octetsLoop
531
532 // For AES-192 round keys occupy: K0 .. K7, K10, K11, K8, K9, KLAST
533 VMOV K8.B16, K10.B16
534 VMOV K9.B16, K11.B16
535 VMOV KLAST.B16, K8.B16
536 VLD1.P 16(H0), [K9.B16]
537 VLD1.P 16(H0), [KLAST.B16]
538 TBZ $3, NR, octetsLoop
539 // For AES-256 round keys occupy: K0 .. K7, K10, K11, mem, mem, K8, K9, KLAST
540 VMOV KLAST.B16, K8.B16
541 VLD1.P 16(H0), [K9.B16]
542 VLD1.P 16(H0), [KLAST.B16]
543 ADD $10*16, ks, H0
544 MOVD H0, curK
545
546 octetsLoop:
547 SUB $128, srcPtrLen
548
549 VMOV CTR.B16, B0.B16
550 VADD B0.S4, INC.S4, B1.S4
551 VREV32 B0.B16, B0.B16
552 VADD B1.S4, INC.S4, B2.S4
553 VREV32 B1.B16, B1.B16
554 VADD B2.S4, INC.S4, B3.S4
555 VREV32 B2.B16, B2.B16
556 VADD B3.S4, INC.S4, B4.S4
557 VREV32 B3.B16, B3.B16
558 VADD B4.S4, INC.S4, B5.S4
559 VREV32 B4.B16, B4.B16
560 VADD B5.S4, INC.S4, B6.S4
561 VREV32 B5.B16, B5.B16
562 VADD B6.S4, INC.S4, B7.S4
563 VREV32 B6.B16, B6.B16
564 VADD B7.S4, INC.S4, CTR.S4
565 VREV32 B7.B16, B7.B16
566
567 aesrndx8(K0)
568 aesrndx8(K1)
569 aesrndx8(K2)
570 aesrndx8(K3)
571 aesrndx8(K4)
572 aesrndx8(K5)
573 aesrndx8(K6)
574 aesrndx8(K7)
575 TBZ $4, NR, octetsFinish
576 aesrndx8(K10)
577 aesrndx8(K11)
578 TBZ $3, NR, octetsFinish
579 VLD1.P 32(curK), [T1.B16, T2.B16]
580 aesrndx8(T1)
581 aesrndx8(T2)
582 MOVD H0, curK
583 octetsFinish:
584 aesrndx8(K8)
585 aesrndlastx8(K9)
586
587 // This path should not use EOR3 even if it is available, because it
588 // would contend on limited pipeline resources and regress on Neoverse V2.
589 VEOR KLAST.B16, B0.B16, B0.B16
590 VEOR KLAST.B16, B1.B16, B1.B16
591 VEOR KLAST.B16, B2.B16, B2.B16
592 VEOR KLAST.B16, B3.B16, B3.B16
593 VEOR KLAST.B16, B4.B16, B4.B16
594 VEOR KLAST.B16, B5.B16, B5.B16
595 VEOR KLAST.B16, B6.B16, B6.B16
596 VEOR KLAST.B16, B7.B16, B7.B16
597
598 VLD1.P 32(srcPtr), [T1.B16, T2.B16]
599 VEOR B0.B16, T1.B16, B0.B16
600 VEOR B1.B16, T2.B16, B1.B16
601 VST1.P [B0.B16, B1.B16], 32(dstPtr)
602 VLD1.P 32(srcPtr), [T1.B16, T2.B16]
603 VEOR B2.B16, T1.B16, B2.B16
604 VEOR B3.B16, T2.B16, B3.B16
605 VST1.P [B2.B16, B3.B16], 32(dstPtr)
606 VLD1.P 32(srcPtr), [T1.B16, T2.B16]
607 VEOR B4.B16, T1.B16, B4.B16
608 VEOR B5.B16, T2.B16, B5.B16
609 VST1.P [B4.B16, B5.B16], 32(dstPtr)
610 VLD1.P 32(srcPtr), [T1.B16, T2.B16]
611 VEOR B6.B16, T1.B16, B6.B16
612 VEOR B7.B16, T2.B16, B7.B16
613 VST1.P [B6.B16, B7.B16], 32(dstPtr)
614
615 VLD1.P 32(pTbl), [T1.B16, T2.B16]
616 VREV64 B0.B16, B0.B16
617 VEOR ACC0.B16, B0.B16, B0.B16
618 VEXT $8, B0.B16, B0.B16, T0.B16
619 VEOR B0.B16, T0.B16, T0.B16
620 VPMULL B0.D1, T1.D1, ACC1.Q1
621 VPMULL2 B0.D2, T1.D2, ACC0.Q1
622 VPMULL T0.D1, T2.D1, ACCM.Q1
623
624 mulRound(B1)
625 mulRound(B2)
626 mulRound(B3)
627 mulRound(B4)
628 mulRound(B5)
629 mulRound(B6)
630 mulRound(B7)
631 MOVD pTblSave, pTbl
632 reduce()
633
634 CMP $128, srcPtrLen
635 BGE octetsLoop
636
637 startSingles:
638 CBZ srcPtrLen, done
639 ADD $14*16, pTbl
640 // Preload H and its Karatsuba precomp
641 VLD1.P (pTbl), [T1.B16, T2.B16]
642 // Preload AES round keys
643 ADD $128, ks
644 VLD1.P 48(ks), [K8.B16, K9.B16, K10.B16]
645 VMOV K10.B16, KLAST.B16
646 TBZ $4, NR, singlesLoop
647 VLD1.P 32(ks), [B1.B16, B2.B16]
648 VMOV B2.B16, KLAST.B16
649 TBZ $3, NR, singlesLoop
650 VLD1.P 32(ks), [B3.B16, B4.B16]
651 VMOV B4.B16, KLAST.B16
652
653 singlesLoop:
654 CMP $16, srcPtrLen
655 BLT tail
656 SUB $16, srcPtrLen
657
658 VLD1.P 16(srcPtr), [T0.B16]
659 VEOR KLAST.B16, T0.B16, T0.B16
660
661 VREV32 CTR.B16, B0.B16
662 VADD CTR.S4, INC.S4, CTR.S4
663
664 AESE K0.B16, B0.B16
665 AESMC B0.B16, B0.B16
666 AESE K1.B16, B0.B16
667 AESMC B0.B16, B0.B16
668 AESE K2.B16, B0.B16
669 AESMC B0.B16, B0.B16
670 AESE K3.B16, B0.B16
671 AESMC B0.B16, B0.B16
672 AESE K4.B16, B0.B16
673 AESMC B0.B16, B0.B16
674 AESE K5.B16, B0.B16
675 AESMC B0.B16, B0.B16
676 AESE K6.B16, B0.B16
677 AESMC B0.B16, B0.B16
678 AESE K7.B16, B0.B16
679 AESMC B0.B16, B0.B16
680 AESE K8.B16, B0.B16
681 AESMC B0.B16, B0.B16
682 AESE K9.B16, B0.B16
683 TBZ $4, NR, singlesLast
684 AESMC B0.B16, B0.B16
685 AESE K10.B16, B0.B16
686 AESMC B0.B16, B0.B16
687 AESE B1.B16, B0.B16
688 TBZ $3, NR, singlesLast
689 AESMC B0.B16, B0.B16
690 AESE B2.B16, B0.B16
691 AESMC B0.B16, B0.B16
692 AESE B3.B16, B0.B16
693 singlesLast:
694 VEOR T0.B16, B0.B16, B0.B16
695
696 VST1.P [B0.B16], 16(dstPtr)
697 encReduce:
698 VREV64 B0.B16, B0.B16
699 VEOR ACC0.B16, B0.B16, B0.B16
700
701 VEXT $8, B0.B16, B0.B16, T0.B16
702 VEOR B0.B16, T0.B16, T0.B16
703 VPMULL B0.D1, T1.D1, ACC1.Q1
704 VPMULL2 B0.D2, T1.D2, ACC0.Q1
705 VPMULL T0.D1, T2.D1, ACCM.Q1
706
707 reduce()
708
709 B singlesLoop
710 tail:
711 CBZ srcPtrLen, done
712
713 tailLoad(T0)
714
715 VEOR KLAST.B16, T0.B16, T0.B16
716 VREV32 CTR.B16, B0.B16
717
718 AESE K0.B16, B0.B16
719 AESMC B0.B16, B0.B16
720 AESE K1.B16, B0.B16
721 AESMC B0.B16, B0.B16
722 AESE K2.B16, B0.B16
723 AESMC B0.B16, B0.B16
724 AESE K3.B16, B0.B16
725 AESMC B0.B16, B0.B16
726 AESE K4.B16, B0.B16
727 AESMC B0.B16, B0.B16
728 AESE K5.B16, B0.B16
729 AESMC B0.B16, B0.B16
730 AESE K6.B16, B0.B16
731 AESMC B0.B16, B0.B16
732 AESE K7.B16, B0.B16
733 AESMC B0.B16, B0.B16
734 AESE K8.B16, B0.B16
735 AESMC B0.B16, B0.B16
736 AESE K9.B16, B0.B16
737 TBZ $4, NR, tailLast
738 AESMC B0.B16, B0.B16
739 AESE K10.B16, B0.B16
740 AESMC B0.B16, B0.B16
741 AESE B1.B16, B0.B16
742 TBZ $3, NR, tailLast
743 AESMC B0.B16, B0.B16
744 AESE B2.B16, B0.B16
745 AESMC B0.B16, B0.B16
746 AESE B3.B16, B0.B16
747
748 tailLast:
749 VEOR T0.B16, B0.B16, B0.B16
750 VAND T3.B16, B0.B16, B0.B16
751
752 // Store from a copy, since tailStore clobbers its argument and
753 // B0 is the GHASH input of encReduce.
754 VMOV B0.B16, T0.B16
755 tailStore(T0)
756 MOVD ZR, srcPtrLen
757
758 B encReduce
759
760 done:
761 VST1 [ACC0.B16], (tPtr)
762 RET
763
764 // func gcmAesDec(productTable *[256]byte, dst, src []byte, ctr, T *[16]byte, ks []uint32)
765 TEXT ·gcmAesDec(SB),NOSPLIT,$0
766 MOVD productTable+0(FP), pTbl
767 MOVD dst+8(FP), dstPtr
768 MOVD src_base+32(FP), srcPtr
769 MOVD src_len+40(FP), srcPtrLen
770 MOVD ctr+56(FP), ctrPtr
771 MOVD T+64(FP), tPtr
772 MOVD ks_base+72(FP), ks
773 MOVD ks_len+80(FP), NR
774
775 MOVD $0xC2, H1
776 LSL $56, H1
777 MOVD $1, H0
778 VMOV H1, POLY.D[0]
779 VMOV H0, POLY.D[1]
780 VEOR ZERO.B16, ZERO.B16, ZERO.B16
781 // Compute NR from len(ks)
782 MOVD pTbl, pTblSave
783 // Current tag, after AAD
784 VLD1 (tPtr), [ACC0.B16]
785 VEOR ACC1.B16, ACC1.B16, ACC1.B16
786 VEOR ACCM.B16, ACCM.B16, ACCM.B16
787 // Prepare initial counter, and the increment vector
788 VLD1 (ctrPtr), [CTR.B16]
789 VEOR INC.B16, INC.B16, INC.B16
790 MOVD $1, H0
791 VMOV H0, INC.S[3]
792 VREV32 CTR.B16, CTR.B16
793 VADD CTR.S4, INC.S4, CTR.S4
794
795 MOVD ks, H0
796 // For AES-128 round keys are stored in: K0 .. K10, KLAST
797 VLD1.P 64(H0), [K0.B16, K1.B16, K2.B16, K3.B16]
798 VLD1.P 64(H0), [K4.B16, K5.B16, K6.B16, K7.B16]
799 VLD1.P 48(H0), [K8.B16, K9.B16, K10.B16]
800 VMOV K10.B16, KLAST.B16
801
802 // Skip to <8 blocks loop
803 CMP $128, srcPtrLen
804 BLT startSingles
805 // There are at least 8 blocks to encrypt
806 MOVBU ·supportsAESGCMEOR3(SB), H1
807 TBZ $4, NR, octetsLoop
808
809 // For AES-192 round keys occupy: K0 .. K7, K10, K11, K8, K9, KLAST
810 VMOV K8.B16, K10.B16
811 VMOV K9.B16, K11.B16
812 VMOV KLAST.B16, K8.B16
813 VLD1.P 16(H0), [K9.B16]
814 VLD1.P 16(H0), [KLAST.B16]
815 TBZ $3, NR, octetsLoop
816 // For AES-256 round keys occupy: K0 .. K7, K10, K11, mem, mem, K8, K9, KLAST
817 VMOV KLAST.B16, K8.B16
818 VLD1.P 16(H0), [K9.B16]
819 VLD1.P 16(H0), [KLAST.B16]
820 ADD $10*16, ks, H0
821 MOVD H0, curK
822
823 octetsLoop:
824 SUB $128, srcPtrLen
825
826 VMOV CTR.B16, B0.B16
827 VADD B0.S4, INC.S4, B1.S4
828 VREV32 B0.B16, B0.B16
829 VADD B1.S4, INC.S4, B2.S4
830 VREV32 B1.B16, B1.B16
831 VADD B2.S4, INC.S4, B3.S4
832 VREV32 B2.B16, B2.B16
833 VADD B3.S4, INC.S4, B4.S4
834 VREV32 B3.B16, B3.B16
835 VADD B4.S4, INC.S4, B5.S4
836 VREV32 B4.B16, B4.B16
837 VADD B5.S4, INC.S4, B6.S4
838 VREV32 B5.B16, B5.B16
839 VADD B6.S4, INC.S4, B7.S4
840 VREV32 B6.B16, B6.B16
841 VADD B7.S4, INC.S4, CTR.S4
842 VREV32 B7.B16, B7.B16
843
844 aesrndx8(K0)
845 aesrndx8(K1)
846 aesrndx8(K2)
847 aesrndx8(K3)
848 aesrndx8(K4)
849 aesrndx8(K5)
850 aesrndx8(K6)
851 aesrndx8(K7)
852 TBZ $4, NR, octetsFinish
853 aesrndx8(K10)
854 aesrndx8(K11)
855 TBZ $3, NR, octetsFinish
856 VLD1.P 32(curK), [T1.B16, T2.B16]
857 aesrndx8(T1)
858 aesrndx8(T2)
859 MOVD H0, curK
860 octetsFinish:
861 aesrndx8(K8)
862 aesrndlastx8(K9)
863
864 CBNZ H1, octetsEOR3
865
866 VEOR KLAST.B16, B0.B16, T1.B16
867 VEOR KLAST.B16, B1.B16, T2.B16
868 VEOR KLAST.B16, B2.B16, B2.B16
869 VEOR KLAST.B16, B3.B16, B3.B16
870 VEOR KLAST.B16, B4.B16, B4.B16
871 VEOR KLAST.B16, B5.B16, B5.B16
872 VEOR KLAST.B16, B6.B16, B6.B16
873 VEOR KLAST.B16, B7.B16, B7.B16
874
875 VLD1.P 32(srcPtr), [B0.B16, B1.B16]
876 VEOR B0.B16, T1.B16, T1.B16
877 VEOR B1.B16, T2.B16, T2.B16
878 VST1.P [T1.B16, T2.B16], 32(dstPtr)
879
880 VLD1.P 32(pTbl), [T1.B16, T2.B16]
881 VREV64 B0.B16, B0.B16
882 VEOR ACC0.B16, B0.B16, B0.B16
883 VEXT $8, B0.B16, B0.B16, T0.B16
884 VEOR B0.B16, T0.B16, T0.B16
885 VPMULL B0.D1, T1.D1, ACC1.Q1
886 VPMULL2 B0.D2, T1.D2, ACC0.Q1
887 VPMULL T0.D1, T2.D1, ACCM.Q1
888 mulRound(B1)
889
890 VLD1.P 32(srcPtr), [B0.B16, B1.B16]
891 VEOR B2.B16, B0.B16, T1.B16
892 VEOR B3.B16, B1.B16, T2.B16
893 VST1.P [T1.B16, T2.B16], 32(dstPtr)
894 mulRound(B0)
895 mulRound(B1)
896
897 VLD1.P 32(srcPtr), [B0.B16, B1.B16]
898 VEOR B4.B16, B0.B16, T1.B16
899 VEOR B5.B16, B1.B16, T2.B16
900 VST1.P [T1.B16, T2.B16], 32(dstPtr)
901 mulRound(B0)
902 mulRound(B1)
903
904 VLD1.P 32(srcPtr), [B0.B16, B1.B16]
905 VEOR B6.B16, B0.B16, T1.B16
906 VEOR B7.B16, B1.B16, T2.B16
907 VST1.P [T1.B16, T2.B16], 32(dstPtr)
908 mulRound(B0)
909 mulRound(B1)
910
911 MOVD pTblSave, pTbl
912 reduce()
913
914 CMP $128, srcPtrLen
915 BGE octetsLoop
916 B startSingles
917
918 octetsEOR3:
919 VEOR KLAST.B16, B0.B16, T1.B16
920 VEOR KLAST.B16, B1.B16, T2.B16
921
922 VLD1.P 32(srcPtr), [B0.B16, B1.B16]
923 VEOR B0.B16, T1.B16, T1.B16
924 VEOR B1.B16, T2.B16, T2.B16
925 VST1.P [T1.B16, T2.B16], 32(dstPtr)
926
927 VLD1.P 32(pTbl), [T1.B16, T2.B16]
928 VREV64 B0.B16, B0.B16
929 VEOR ACC0.B16, B0.B16, B0.B16
930 VEXT $8, B0.B16, B0.B16, T0.B16
931 VEOR B0.B16, T0.B16, T0.B16
932 VPMULL B0.D1, T1.D1, ACC1.Q1
933 VPMULL2 B0.D2, T1.D2, ACC0.Q1
934 VPMULL T0.D1, T2.D1, ACCM.Q1
935 mulRound(B1)
936
937 VLD1.P 32(srcPtr), [B0.B16, B1.B16]
938 VEOR3 KLAST.B16, B0.B16, B2.B16, T1.B16
939 VEOR3 KLAST.B16, B1.B16, B3.B16, T2.B16
940 VST1.P [T1.B16, T2.B16], 32(dstPtr)
941 mulRound(B0)
942 mulRound(B1)
943
944 VLD1.P 32(srcPtr), [B0.B16, B1.B16]
945 VEOR3 KLAST.B16, B0.B16, B4.B16, T1.B16
946 VEOR3 KLAST.B16, B1.B16, B5.B16, T2.B16
947 VST1.P [T1.B16, T2.B16], 32(dstPtr)
948 mulRound(B0)
949 mulRound(B1)
950
951 VLD1.P 32(srcPtr), [B0.B16, B1.B16]
952 VEOR3 KLAST.B16, B0.B16, B6.B16, T1.B16
953 VEOR3 KLAST.B16, B1.B16, B7.B16, T2.B16
954 VST1.P [T1.B16, T2.B16], 32(dstPtr)
955 mulRound(B0)
956 mulRound(B1)
957
958 MOVD pTblSave, pTbl
959 reduce3()
960
961 CMP $128, srcPtrLen
962 BGE octetsLoop
963
964 startSingles:
965 CBZ srcPtrLen, done
966 ADD $14*16, pTbl
967 // Preload H and its Karatsuba precomp
968 VLD1.P (pTbl), [T1.B16, T2.B16]
969 // Preload AES round keys
970 ADD $128, ks
971 VLD1.P 48(ks), [K8.B16, K9.B16, K10.B16]
972 VMOV K10.B16, KLAST.B16
973 TBZ $4, NR, singlesLoop
974 VLD1.P 32(ks), [B1.B16, B2.B16]
975 VMOV B2.B16, KLAST.B16
976 TBZ $3, NR, singlesLoop
977 VLD1.P 32(ks), [B3.B16, B4.B16]
978 VMOV B4.B16, KLAST.B16
979
980 singlesLoop:
981 CMP $16, srcPtrLen
982 BLT tail
983 SUB $16, srcPtrLen
984
985 VLD1.P 16(srcPtr), [T0.B16]
986 VREV64 T0.B16, B5.B16
987 VEOR KLAST.B16, T0.B16, T0.B16
988
989 VREV32 CTR.B16, B0.B16
990 VADD CTR.S4, INC.S4, CTR.S4
991
992 AESE K0.B16, B0.B16
993 AESMC B0.B16, B0.B16
994 AESE K1.B16, B0.B16
995 AESMC B0.B16, B0.B16
996 AESE K2.B16, B0.B16
997 AESMC B0.B16, B0.B16
998 AESE K3.B16, B0.B16
999 AESMC B0.B16, B0.B16
1000 AESE K4.B16, B0.B16
1001 AESMC B0.B16, B0.B16
1002 AESE K5.B16, B0.B16
1003 AESMC B0.B16, B0.B16
1004 AESE K6.B16, B0.B16
1005 AESMC B0.B16, B0.B16
1006 AESE K7.B16, B0.B16
1007 AESMC B0.B16, B0.B16
1008 AESE K8.B16, B0.B16
1009 AESMC B0.B16, B0.B16
1010 AESE K9.B16, B0.B16
1011 TBZ $4, NR, singlesLast
1012 AESMC B0.B16, B0.B16
1013 AESE K10.B16, B0.B16
1014 AESMC B0.B16, B0.B16
1015 AESE B1.B16, B0.B16
1016 TBZ $3, NR, singlesLast
1017 AESMC B0.B16, B0.B16
1018 AESE B2.B16, B0.B16
1019 AESMC B0.B16, B0.B16
1020 AESE B3.B16, B0.B16
1021 singlesLast:
1022 VEOR T0.B16, B0.B16, B0.B16
1023
1024 VST1.P [B0.B16], 16(dstPtr)
1025
1026 VEOR ACC0.B16, B5.B16, B5.B16
1027 VEXT $8, B5.B16, B5.B16, T0.B16
1028 VEOR B5.B16, T0.B16, T0.B16
1029 VPMULL B5.D1, T1.D1, ACC1.Q1
1030 VPMULL2 B5.D2, T1.D2, ACC0.Q1
1031 VPMULL T0.D1, T2.D1, ACCM.Q1
1032 reduce()
1033
1034 B singlesLoop
1035 tail:
1036 CBZ srcPtrLen, done
1037
1038 VREV32 CTR.B16, B0.B16
1039 VADD CTR.S4, INC.S4, CTR.S4
1040
1041 AESE K0.B16, B0.B16
1042 AESMC B0.B16, B0.B16
1043 AESE K1.B16, B0.B16
1044 AESMC B0.B16, B0.B16
1045 AESE K2.B16, B0.B16
1046 AESMC B0.B16, B0.B16
1047 AESE K3.B16, B0.B16
1048 AESMC B0.B16, B0.B16
1049 AESE K4.B16, B0.B16
1050 AESMC B0.B16, B0.B16
1051 AESE K5.B16, B0.B16
1052 AESMC B0.B16, B0.B16
1053 AESE K6.B16, B0.B16
1054 AESMC B0.B16, B0.B16
1055 AESE K7.B16, B0.B16
1056 AESMC B0.B16, B0.B16
1057 AESE K8.B16, B0.B16
1058 AESMC B0.B16, B0.B16
1059 AESE K9.B16, B0.B16
1060 TBZ $4, NR, tailLast
1061 AESMC B0.B16, B0.B16
1062 AESE K10.B16, B0.B16
1063 AESMC B0.B16, B0.B16
1064 AESE B1.B16, B0.B16
1065 TBZ $3, NR, tailLast
1066 AESMC B0.B16, B0.B16
1067 AESE B2.B16, B0.B16
1068 AESMC B0.B16, B0.B16
1069 AESE B3.B16, B0.B16
1070 tailLast:
1071 VEOR KLAST.B16, B0.B16, B0.B16
1072
1073 tailLoad(B5)
1074
1075 VEOR B5.B16, B0.B16, B0.B16
1076
1077 tailStore(B0)
1078
1079 VREV64 B5.B16, B5.B16
1080
1081 VEOR ACC0.B16, B5.B16, B5.B16
1082 VEXT $8, B5.B16, B5.B16, T0.B16
1083 VEOR B5.B16, T0.B16, T0.B16
1084 VPMULL B5.D1, T1.D1, ACC1.Q1
1085 VPMULL2 B5.D2, T1.D2, ACC0.Q1
1086 VPMULL T0.D1, T2.D1, ACCM.Q1
1087 reduce()
1088 done:
1089 VST1 [ACC0.B16], (tPtr)
1090
1091 RET
1092
View as plain text