1 // Copyright 2026 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 // AES for loong64 using LSX VSHUFB-based software S-box lookup.
6 // No hardware AES instructions exist on LoongArch (as of LA464/LA664).
7 //
8 // SubBytes: 256-byte S-box split into 8×32-byte chunks; 8 VSHUFB ops
9 // select the right byte per input nibble group.
10 // ShiftRows: implemented via a VSHUFB with a fixed permutation table.
11 // MixColumns: GF(2^8) multiply-by-2 via shift+conditional-XOR, then
12 // combine columns with VXORV.
13 // InvMixColumns: implemented via GFMULCONST, a nibble-split VSHUFB table
14 // lookup (separate lo/hi 16-entry tables per GF(2^8) constant). The
15 // 9/11/13/14 constant multiples are computed sequentially — the four
16 // GFMULCONST calls share the same midx1/midx2/tlo scratch registers,
17 // so they are NOT computed in parallel — then combined with
18 // rot1/rot2/rot3 (VSHUFB) and VXORV. This replaces an earlier
19 // XTIME-chain implementation (three chained GF(2^8) doublings), which
20 // was measured to be slower on AESCBCDecrypt1K/Decrypt benchmarks on
21 // this target despite a theoretically shorter critical path; see the
22 // GFMULCONST-table generation note below for how the lo/hi constant
23 // tables were produced and verified.
24 //
25 // expandKeyAsm: Phase 1/2 (deriving enc[] from the input key) is a
26 // scalar port of expandKeyGeneric — the key schedule has
27 // a true serial dependency chain and operates on 32-bit
28 // words smaller than the vector width, so it is not
29 // vectorized. Phase 3 (deriving dec[] from enc[]) has
30 // independent iterations over 16-byte word groups and is
31 // vectorized using INVMIXCOLUMNS directly (no SubBytes —
32 // the scalar td0/td1/td2/td3 tables already fold in and
33 // cancel out sbox0/sbox1, see aes_generic.go), with a
34 // byte-order swap (VSHUFB with byteSwap32) before/after
35 // to convert between word-storage order and the state
36 // byte order that INVMIXCOLUMNS's rot1/rot2/rot3 tables
37 // assume.
38
39 //go:build !purego
40
41 #include "textflag.h"
42
43 // AES S-box tables for loong64 scalar implementation.
44 // Stored as raw byte arrays; each DATA line encodes 8 consecutive
45 // sbox bytes as a little-endian uint64 (byte[0] in the lowest address).
46 //
47 // Formula: value = b7<<56 | b6<<48 | b5<<40 | b4<<32
48 // | b3<<24 | b2<<16 | b1<<8 | b0
49
50 // -----------------------------------------------------------------------
51 // sbox0 — AES encryption S-box (256 bytes)
52 // Source: FIPS-197 Figure 7 / const.go var sbox0
53 // -----------------------------------------------------------------------
54 DATA sbox0_0+0x00(SB)/8, $0xc56f6bf27b777c63 // sbox0[ 0.. 7]
55 DATA sbox0_0+0x08(SB)/8, $0x76abd7fe2b670130 // sbox0[ 8..15]
56 DATA sbox0_0+0x10(SB)/8, $0xf04759fa7dc982ca // sbox0[ 16..23]
57 DATA sbox0_0+0x18(SB)/8, $0xc072a49cafa2d4ad // sbox0[ 24..31]
58 DATA sbox0_0+0x20(SB)/8, $0xccf73f362693fdb7 // sbox0[ 32..39]
59 DATA sbox0_0+0x28(SB)/8, $0x1531d871f1e5a534 // sbox0[ 40..47]
60 DATA sbox0_0+0x30(SB)/8, $0x9a059618c323c704 // sbox0[ 48..55]
61 DATA sbox0_0+0x38(SB)/8, $0x75b227ebe2801207 // sbox0[ 56..63]
62 DATA sbox0_0+0x40(SB)/8, $0xa05a6e1b1a2c8309 // sbox0[ 64..71]
63 DATA sbox0_0+0x48(SB)/8, $0x842fe329b3d63b52 // sbox0[ 72..79]
64 DATA sbox0_0+0x50(SB)/8, $0x5bb1fc20ed00d153 // sbox0[ 80..87]
65 DATA sbox0_0+0x58(SB)/8, $0xcf584c4a39becb6a // sbox0[ 88..95]
66 DATA sbox0_0+0x60(SB)/8, $0x85334d43fbaaefd0 // sbox0[ 96..103]
67 DATA sbox0_0+0x68(SB)/8, $0xa89f3c507f02f945 // sbox0[104..111]
68 DATA sbox0_0+0x70(SB)/8, $0xf5389d928f40a351 // sbox0[112..119]
69 DATA sbox0_0+0x78(SB)/8, $0xd2f3ff1021dab6bc // sbox0[120..127]
70 DATA sbox0_0+0x80(SB)/8, $0x1744975fec130ccd // sbox0[128..135]
71 DATA sbox0_0+0x88(SB)/8, $0x73195d643d7ea7c4 // sbox0[136..143]
72 DATA sbox0_0+0x90(SB)/8, $0x88902a22dc4f8160 // sbox0[144..151]
73 DATA sbox0_0+0x98(SB)/8, $0xdb0b5ede14b8ee46 // sbox0[152..159]
74 DATA sbox0_0+0xa0(SB)/8, $0x5c2406490a3a32e0 // sbox0[160..167]
75 DATA sbox0_0+0xa8(SB)/8, $0x79e4959162acd3c2 // sbox0[168..175]
76 DATA sbox0_0+0xb0(SB)/8, $0xa94ed58d6d37c8e7 // sbox0[176..183]
77 DATA sbox0_0+0xb8(SB)/8, $0x08ae7a65eaf4566c // sbox0[184..191]
78 DATA sbox0_0+0xc0(SB)/8, $0xc6b4a61c2e2578ba // sbox0[192..199]
79 DATA sbox0_0+0xc8(SB)/8, $0x8a8bbd4b1f74dde8 // sbox0[200..207]
80 DATA sbox0_0+0xd0(SB)/8, $0x0ef6034866b53e70 // sbox0[208..215]
81 DATA sbox0_0+0xd8(SB)/8, $0x9e1dc186b9573561 // sbox0[216..223]
82 DATA sbox0_0+0xe0(SB)/8, $0x948ed9691198f8e1 // sbox0[224..231]
83 DATA sbox0_0+0xe8(SB)/8, $0xdf2855cee9871e9b // sbox0[232..239]
84 DATA sbox0_0+0xf0(SB)/8, $0x6842e6bf0d89a18c // sbox0[240..247]
85 DATA sbox0_0+0xf8(SB)/8, $0x16bb54b00f2d9941 // sbox0[248..255]
86 GLOBL sbox0_0(SB), (NOPTR+RODATA), $256
87
88 // -----------------------------------------------------------------------
89 // sbox1 — AES decryption (inverse) S-box (256 bytes)
90 // Source: FIPS-197 Figure 14 / const.go var sbox1
91 // -----------------------------------------------------------------------
92 DATA sbox1_0<>+0x00(SB)/8, $0x38a53630d56a0952 // sbox1[ 0.. 7]
93 DATA sbox1_0<>+0x08(SB)/8, $0xfbd7f3819ea340bf // sbox1[ 8..15]
94 DATA sbox1_0<>+0x10(SB)/8, $0x87ff2f9b8239e37c // sbox1[ 16..23]
95 DATA sbox1_0<>+0x18(SB)/8, $0xcbe9dec444438e34 // sbox1[ 24..31]
96 DATA sbox1_0<>+0x20(SB)/8, $0x3d23c2a632947b54 // sbox1[ 32..39]
97 DATA sbox1_0<>+0x28(SB)/8, $0x4ec3fa420b954cee // sbox1[ 40..47]
98 DATA sbox1_0<>+0x30(SB)/8, $0xb224d92866a12e08 // sbox1[ 48..55]
99 DATA sbox1_0<>+0x38(SB)/8, $0x25d18b6d49a25b76 // sbox1[ 56..63]
100 DATA sbox1_0<>+0x40(SB)/8, $0x1698688664f6f872 // sbox1[ 64..71]
101 DATA sbox1_0<>+0x48(SB)/8, $0x92b6655dcc5ca4d4 // sbox1[ 72..79]
102 DATA sbox1_0<>+0x50(SB)/8, $0xdab9edfd5048706c // sbox1[ 80..87]
103 DATA sbox1_0<>+0x58(SB)/8, $0x849d8da75746155e // sbox1[ 88..95]
104 DATA sbox1_0<>+0x60(SB)/8, $0x0ad3bc8c00abd890 // sbox1[ 96..103]
105 DATA sbox1_0<>+0x68(SB)/8, $0x0645b3b80558e4f7 // sbox1[104..111]
106 DATA sbox1_0<>+0x70(SB)/8, $0x020f3fca8f1e2cd0 // sbox1[112..119]
107 DATA sbox1_0<>+0x78(SB)/8, $0x6b8a130103bdafc1 // sbox1[120..127]
108 DATA sbox1_0<>+0x80(SB)/8, $0xeadc674f4111913a // sbox1[128..135]
109 DATA sbox1_0<>+0x88(SB)/8, $0x73e6b4f0cecff297 // sbox1[136..143]
110 DATA sbox1_0<>+0x90(SB)/8, $0x8535ade72274ac96 // sbox1[144..151]
111 DATA sbox1_0<>+0x98(SB)/8, $0x6edf751ce837f9e2 // sbox1[152..159]
112 DATA sbox1_0<>+0xa0(SB)/8, $0x89c5291d711af147 // sbox1[160..167]
113 DATA sbox1_0<>+0xa8(SB)/8, $0x1bbe18aa0e62b76f // sbox1[168..175]
114 DATA sbox1_0<>+0xb0(SB)/8, $0x2079d2c64b3e56fc // sbox1[176..183]
115 DATA sbox1_0<>+0xb8(SB)/8, $0xf45acd78fec0db9a // sbox1[184..191]
116 DATA sbox1_0<>+0xc0(SB)/8, $0x31c7078833a8dd1f // sbox1[192..199]
117 DATA sbox1_0<>+0xc8(SB)/8, $0x5fec8027591012b1 // sbox1[200..207]
118 DATA sbox1_0<>+0xd0(SB)/8, $0x0d4ab519a97f5160 // sbox1[208..215]
119 DATA sbox1_0<>+0xd8(SB)/8, $0xef9cc9939f7ae52d // sbox1[216..223]
120 DATA sbox1_0<>+0xe0(SB)/8, $0xb0f52aae4d3be0a0 // sbox1[224..231]
121 DATA sbox1_0<>+0xe8(SB)/8, $0x619953833cbbebc8 // sbox1[232..239]
122 DATA sbox1_0<>+0xf0(SB)/8, $0x26d677ba7e042b17 // sbox1[240..247]
123 DATA sbox1_0<>+0xf8(SB)/8, $0x7d0c2155631469e1 // sbox1[248..255]
124 GLOBL sbox1_0<>(SB), (NOPTR+RODATA), $256
125
126 // ShiftRows permutation for encryption:
127 // state byte positions after ShiftRows (row i rotated left by i):
128 // row0: 0,1,2,3 row1: 5,6,7,4 row2: 10,11,8,9 row3: 15,12,13,14
129 DATA shiftRows+0x00(SB)/8, $0x030e09040f0a0500
130 DATA shiftRows+0x08(SB)/8, $0x0b06010c07020d08
131 GLOBL shiftRows(SB), (NOPTR+RODATA), $16
132
133 // rot1: {1,2,3,0, 5,6,7,4, 9,10,11,8, 13,14,15,12}
134 DATA rot1+0x00(SB)/8, $0x0407060500030201
135 DATA rot1+0x08(SB)/8, $0x0c0f0e0d080b0a09
136 GLOBL rot1(SB), (NOPTR+RODATA), $16
137
138 // rot2: {2,3,0,1, 6,7,4,5, 10,11,8,9, 14,15,12,13}
139 DATA rot2+0x00(SB)/8, $0x0504070601000302
140 DATA rot2+0x08(SB)/8, $0x0d0c0f0e09080b0a
141 GLOBL rot2(SB), (NOPTR+RODATA), $16
142
143 // rot3: {3,0,1,2, 7,4,5,6, 11,8,9,10, 15,12,13,14}
144 DATA rot3+0x00(SB)/8, $0x0605040702010003
145 DATA rot3+0x08(SB)/8, $0x0e0d0c0f0a09080b
146 GLOBL rot3(SB), (NOPTR+RODATA), $16
147
148 // InvShiftRows permutation for decryption.
149 DATA invShiftRows<>+0x00(SB)/8, $0x0b0e0104070a0d00
150 DATA invShiftRows<>+0x08(SB)/8, $0x0306090c0f020508
151 GLOBL invShiftRows<>(SB), (NOPTR+RODATA), $16
152
153 DATA byteSwap32+0x00(SB)/8, $0x0405060700010203 // index 0-7: {3,2,1,0,7,6,5,4}
154 DATA byteSwap32+0x08(SB)/8, $0x0c0d0e0f08090a0b // index 8-15: {11,10,9,8,15,14,13,12}
155 GLOBL byteSwap32(SB), (NOPTR+RODATA), $16
156
157 // mul9lo[v] = mul(9, v), v = 0..15
158 DATA mul9lo<>+0x00(SB)/8, $0x3f362d241b120900
159 DATA mul9lo<>+0x08(SB)/8, $0x777e656c535a4148
160 GLOBL mul9lo<>(SB), (NOPTR+RODATA), $16
161
162 // mul9hi[v] = mul(9, v<<4), v = 0..15
163 DATA mul9hi<>+0x00(SB)/8, $0xdd4de676ab3b9000
164 DATA mul9hi<>+0x08(SB)/8, $0x31a10a9a47d77cec
165 GLOBL mul9hi<>(SB), (NOPTR+RODATA), $16
166
167 // mul11lo[v] = mul(11, v), v = 0..15
168 DATA mul11lo<>+0x00(SB)/8, $0x313a272c1d160b00
169 DATA mul11lo<>+0x08(SB)/8, $0x69627f74454e5358
170 GLOBL mul11lo<>(SB), (NOPTR+RODATA), $16
171
172 // mul11hi[v] = mul(11, v<<4), v = 0..15
173 DATA mul11hi<>+0x00(SB)/8, $0x3d8d46f6cb7bb000
174 DATA mul11hi<>+0x08(SB)/8, $0xca7ab1013c8c47f7
175 GLOBL mul11hi<>(SB), (NOPTR+RODATA), $16
176
177 // mul13lo[v] = mul(13, v), v = 0..15
178 DATA mul13lo<>+0x00(SB)/8, $0x232e3934171a0d00
179 DATA mul13lo<>+0x08(SB)/8, $0x4b46515c7f726568
180 GLOBL mul13lo<>(SB), (NOPTR+RODATA), $16
181
182 // mul13hi[v] = mul(13, v<<4), v = 0..15
183 DATA mul13hi<>+0x00(SB)/8, $0x06d6bd6d6bbbd000
184 DATA mul13hi<>+0x08(SB)/8, $0xdc0c67b7b1610ada
185 GLOBL mul13hi<>(SB), (NOPTR+RODATA), $16
186
187 // mul14lo[v] = mul(14, v), v = 0..15
188 DATA mul14lo<>+0x00(SB)/8, $0x2a243638121c0e00
189 DATA mul14lo<>+0x08(SB)/8, $0x5a544648626c7e70
190 GLOBL mul14lo<>(SB), (NOPTR+RODATA), $16
191
192 // mul14hi[v] = mul(14, v<<4), v = 0..15
193 DATA mul14hi<>+0x00(SB)/8, $0x96764dad3bdbe000
194 DATA mul14hi<>+0x08(SB)/8, $0xd7370cec7a9aa141
195 GLOBL mul14hi<>(SB), (NOPTR+RODATA), $16
196
197 #define SUBBYTES(src, sbox_ptr, out) \
198 VANDB $0xe0, src, V25; \ // V25 = src & 0xe0, keep only bit7/bit6/bit5
199 VSRAB $7, V25, V31; \ // V31 = byte-wise broadcast of bit7 (0xFF/0x00)
200 VSLLB $1, V25, V24; \ // shift bit6 into the sign position
201 VSRAB $7, V24, V30; \ // V30 = byte-wise broadcast of bit6
202 VSLLB $2, V25, V24; \ // shift bit5 into the sign position
203 VSRAB $7, V24, V29; \ // V29 = byte-wise broadcast of bit5
204 VANDB $0x1f, src, V26; \ // V26 = src & 0x1f, index for VSHUFB 32-way lookup
205 \
206 /* ---- 2. Chunk0/1 -> r01 (V16..V19,V28,V27 freed after use) ---- */ \
207 VMOVQ (sbox_ptr), V16; \
208 VMOVQ 16(sbox_ptr), V17; \
209 VMOVQ 32(sbox_ptr), V18; \
210 VMOVQ 48(sbox_ptr), V19; \
211 VSHUFB V26, V16, V17, V28; \ // V28 = chunk0_raw
212 VSHUFB V26, V18, V19, V27; \ // V27 = chunk1_raw
213 VBITSELV V29, V27, V28, V20; \ // V20 = r01 = M5 ? chunk1 : chunk0
214 \
215 /* ---- 3. Chunk2/3 -> r23 ---- */ \
216 VMOVQ 64(sbox_ptr), V16; \
217 VMOVQ 80(sbox_ptr), V17; \
218 VMOVQ 96(sbox_ptr), V18; \
219 VMOVQ 112(sbox_ptr), V19; \
220 VSHUFB V26, V16, V17, V28; \ // V28 = chunk2_raw
221 VSHUFB V26, V18, V19, V27; \ // V27 = chunk3_raw
222 VBITSELV V29, V27, V28, V21; \ // V21 = r23
223 \
224 /* ---- 4. Merge first group (chunk0..3) -> r0123 ---- */ \
225 VBITSELV V30, V21, V20, V22; \ // V22 = r0123 = M6 ? r23 : r01
226 \
227 /* ---- 5. Chunk4/5 -> r45 ---- */ \
228 VMOVQ 128(sbox_ptr), V16; \
229 VMOVQ 144(sbox_ptr), V17; \
230 VMOVQ 160(sbox_ptr), V18; \
231 VMOVQ 176(sbox_ptr), V19; \
232 VSHUFB V26, V16, V17, V28; \ // V28 = chunk4_raw
233 VSHUFB V26, V18, V19, V27; \ // V27 = chunk5_raw
234 VBITSELV V29, V27, V28, V20; \ // V20 reused = r45
235 \
236 /* ---- 6. Chunk6/7 -> r67 ---- */ \
237 VMOVQ 192(sbox_ptr), V16; \
238 VMOVQ 208(sbox_ptr), V17; \
239 VMOVQ 224(sbox_ptr), V18; \
240 VMOVQ 240(sbox_ptr), V19; \
241 VSHUFB V26, V16, V17, V28; \ // V28 = chunk6_raw
242 VSHUFB V26, V18, V19, V27; \ // V27 = chunk7_raw
243 VBITSELV V29, V27, V28, V21; \ // V21 reused = r67
244 \
245 /* ---- 7. Merge second group (chunk4..7) -> r4567 ---- */ \
246 VBITSELV V30, V21, V20, V23; \ // V23 = r4567 = M6 ? r67 : r45
247 \
248 /* ---- 8. Final top-level merge ---- */ \
249 VBITSELV V31, V23, V22, out // out = M7 ? r4567 : r0123
250
251 // -----------------------------------------------------------------------
252 // MixColumns helper: multiply each byte of Vn by 2 in GF(2^8).
253 // xtime(a) = (a << 1) ^ (0x1b if a & 0x80 else 0)
254 // -----------------------------------------------------------------------
255 #define XTIME(src, dst, tmp) \
256 VSLLB $1, src, dst; \ /* dst = src << 1 */
257 VSRAB $7, src, tmp; \ /* tmp[i] = 0xff if src[i]>=0x80, else 0x00 */
258 VANDB $0x1b, tmp, tmp; \ /* tmp[i] = 0x1b if src[i]>=0x80, else 0x00 */
259 VXORV tmp, dst, dst /* dst ^= tmp */
260
261 #define MIXCOLUMNS(src, out, r1, r2, r3, t1, t2, t3) \
262 VSHUFB r1, src, src, t1; \ /* t1 = rot1(a) */
263 VXORV src, t1, t2; \ /* t2 = a ^ rot1(a) */
264 VSHUFB r2, src, src, t3; \ /* t3 = rot2(a) */
265 VXORV t2, t3, out; \ /* out = a^rot1(a)^rot2(a) */
266 VSHUFB r3, src, src, t3; \ /* t3 = rot3(a) */
267 VXORV out, t3, out; \ /* out = t = a0^a1^a2^a3（each byte） */
268 XTIME(t2, t1, t3); \ /* t1 = xtime(a^rot1(a)) */
269 VXORV src, out, out; \ /* out = a ^ t */
270 VXORV out, t1, out /* out ^= xtime(...) */
271
272 // t = subw(t): S-box substitution for each byte of the 32-bit word t, results written back to t
273 #define SUBW_INLINE(t, sbox, tmp1, tmp2) \
274 SRLV $24, t, tmp1; \
275 AND $0xff, tmp1; \
276 ADDV sbox, tmp1, tmp1; \
277 MOVBU (tmp1), tmp1; \
278 SLLV $24, tmp1, tmp2; \
279 SRLV $16, t, tmp1; \
280 AND $0xff, tmp1; \
281 ADDV sbox, tmp1, tmp1; \
282 MOVBU (tmp1), tmp1; \
283 SLLV $16, tmp1, tmp1; \
284 OR tmp1, tmp2; \
285 SRLV $8, t, tmp1; \
286 AND $0xff, tmp1; \
287 ADDV sbox, tmp1, tmp1; \
288 MOVBU (tmp1), tmp1; \
289 SLLV $8, tmp1, tmp1; \
290 OR tmp1, tmp2; \
291 AND $0xff, t, tmp1; \
292 ADDV sbox, tmp1, tmp1; \
293 MOVBU (tmp1), tmp1; \
294 OR tmp1, tmp2; \
295 MOVV tmp2, t
296
297 TEXT ·encryptBlockAsm(SB), NOSPLIT, $0-32
298 MOVV nr+0(FP), R4
299 MOVV xk+8(FP), R5
300 MOVV dst+16(FP), R6
301 MOVV src+24(FP), R7
302
303 // Load 16-byte state
304 VMOVQ (R7), V0
305
306 // Load constant table addresses
307 MOVV $sbox0_0(SB), R9
308 MOVV $shiftRows(SB), R10
309 MOVV $rot1(SB), R11
310 MOVV $rot2(SB), R12
311 MOVV $rot3(SB), R13
312 MOVV $byteSwap32(SB), R14
313
314 // Load vector constants
315 VMOVQ (R10), V7 // shiftRows index
316 VMOVQ (R11), V4 // rot1 index
317 VMOVQ (R12), V5 // rot2 index
318 VMOVQ (R13), V6 // rot3 index
319 VMOVQ (R14), V9 // byteSwap32 index
320
321 // AddRoundKey (round 0)
322 VMOVQ (R5), V8
323 VSHUFB V9, V8, V8, V8
324 VXORV V8, V0, V0
325 ADDV $16, R5
326
327 // Middle rounds: nr - 1 iterations
328 ADDV $-1, R4, R8
329 Lenc_loop:
330 SUBBYTES(V0, R9, V0)
331
332 // ShiftRows
333 VSHUFB V7, V0, V0, V0
334
335 // MixColumns
336 MIXCOLUMNS(V0, V15, V4, V5, V6, V10, V11, V12)
337
338 // AddRoundKey
339 VMOVQ (R5), V8
340 VSHUFB V9, V8, V8, V8
341 VXORV V8, V15, V0
342 ADDV $16, R5
343
344 SUBV $1, R8
345 BNE R8, R0, Lenc_loop
346
347 // Final round: SubBytes + ShiftRows + AddRoundKey (no MixColumns)
348 SUBBYTES(V0, R9, V0)
349 VSHUFB V7, V0, V0, V0
350 VMOVQ (R5), V8
351 VSHUFB V9, V8, V8, V8
352 VXORV V8, V0, V0
353
354 // Store ciphertext
355 VMOVQ V0, (R6)
356 RET
357
358 // GFMULCONST(src, lo_tbl_ptr, hi_tbl_ptr, out, lo_idx, hi_idx, tlo)
359 // out = const * src (GF(2^8)), via nibble-split lookup.
360 // lo_tbl_ptr/hi_tbl_ptr: base address registers for this constant's 16B tables.
361 // lo_idx/hi_idx/tlo: scratch, freed immediately after the macro.
362 #define GFMULCONST(src, lo_tbl, hi_tbl, out, lo_idx, hi_idx, tlo) \
363 VANDB $0x0f, src, lo_idx; \ // lo_idx = src & 0x0f
364 VSRLB $4, src, hi_idx; \ // hi_idx = src >> 4 (unsigned, top nibble)
365 VMOVQ (lo_tbl), tlo; \ // load this constant's lo table (16B)
366 VSHUFB lo_idx, tlo, tlo, lo_idx;\ // lo_idx = lo_table[src&0xf] (single-source 16-way lookup)
367 VMOVQ (hi_tbl), tlo; \ // load this constant's hi table (16B), tlo reused
368 VSHUFB hi_idx, tlo, tlo, hi_idx;\ // hi_idx = hi_table[src>>4]
369 VXORV lo_idx, hi_idx, out // out = lo_part ^ hi_part
370
371 // INVMIXCOLUMNS(src, out, r1, r2, r3, t9, t11, t13, t14, midx1, midx2, tlo)
372 // r1/r2/r3: rotation index tables (already loaded, e.g. V4/V5/V6)
373 // t9/t11/t13/t14: four accumulator registers, each holds one const*src result
374 // midx1/midx2/tlo: shared scratch reused sequentially across the 4 GFMULCONST calls
375 #define INVMIXCOLUMNS(src, out, r1, r2, r3, t9, t11, t13, t14, midx1, midx2, tlo) \
376 MOVV $mul9lo<>(SB), R13; \
377 MOVV $mul9hi<>(SB), R14; \
378 GFMULCONST(src, R13, R14, t9, midx1, midx2, tlo); \ // t9 = 9*src
379 \
380 MOVV $mul11lo<>(SB), R13; \
381 MOVV $mul11hi<>(SB), R14; \
382 GFMULCONST(src, R13, R14, t11, midx1, midx2, tlo); \ // t11 = 11*src
383 \
384 MOVV $mul13lo<>(SB), R13; \
385 MOVV $mul13hi<>(SB), R14; \
386 GFMULCONST(src, R13, R14, t13, midx1, midx2, tlo); \ // t13 = 13*src
387 \
388 MOVV $mul14lo<>(SB), R13; \
389 MOVV $mul14hi<>(SB), R14; \
390 GFMULCONST(src, R13, R14, t14, midx1, midx2, tlo); \ // t14 = 14*src
391 \
392 /* out = 14*src ^ rot1(11*src) ^ rot2(13*src) ^ rot3(9*src) */ \
393 VSHUFB r1, t11, t11, t11; \ // t11 = rot1(11*src)
394 VSHUFB r2, t13, t13, t13; \ // t13 = rot2(13*src)
395 VSHUFB r3, t9, t9, t9; \ // t9 = rot3(9*src)
396 VXORV t11, t14, out; \ // out = 14*src ^ rot1(11*src)
397 VXORV t13, out, out; \ // out ^= rot2(13*src)
398 VXORV t9, out, out // out ^= rot3(9*src)
399
400 TEXT ·decryptBlockAsm(SB), NOSPLIT, $0-32
401 MOVV nr+0(FP), R4
402 MOVV xk+8(FP), R5
403 MOVV dst+16(FP), R6
404 MOVV src+24(FP), R7
405
406 // Load state
407 VMOVQ (R7), V0
408
409 // Load invShiftRows index vector
410 MOVV $invShiftRows<>(SB), R9
411 VMOVQ (R9), V7
412
413 // Load sbox1 base for InvSubBytes
414 MOVV $sbox1_0<>(SB), R10
415
416 // Load byte-swap table for round-key endianness
417 MOVV $byteSwap32(SB), R11
418 VMOVQ (R11), V2
419
420 // Load rotation indices for InvMixColumns
421 MOVV $rot1(SB), R12
422 VMOVQ (R12), V4
423 MOVV $rot2(SB), R12
424 VMOVQ (R12), V5
425 MOVV $rot3(SB), R12
426 VMOVQ (R12), V6
427
428 // Initial AddRoundKey (last round key for decryption)
429 VMOVQ (R5), V8
430 VSHUFB V2, V8, V8, V8 // byte-swap each 32-bit word
431 VXORV V8, V0, V0
432 ADDV $16, R5
433
434 // R8 = nr - 1 (middle rounds with InvMixColumns)
435 ADDV $-1, R4, R8
436
437 Ldec_loop:
438 SUBBYTES(V0, R10, V13)
439 VSHUFB V7, V13, V13, V0 // InvShiftRows
440
441 INVMIXCOLUMNS(V0, V0, V4, V5, V6, V9, V10, V11, V12, V13, V14, V16)
442
443 VMOVQ (R5), V8 // AddRoundKey(put in last)
444 VSHUFB V2, V8, V8, V8
445 VXORV V8, V0, V0
446 ADDV $16, R5
447
448 SUBV $1, R8
449 BNE R8, R0, Ldec_loop
450
451 // Final round: InvSubBytes + InvShiftRows + AddRoundKey (no InvMixColumns)
452 SUBBYTES(V0, R10, V13)
453 VSHUFB V7, V13, V13, V0
454 VMOVQ (R5), V8
455 VSHUFB V2, V8, V8, V8
456 VXORV V8, V0, V0
457
458 // Store result
459 VMOVQ V0, (R6)
460 RET
461
462 // -----------------------------------------------------------------------
463 // func expandKeyAsm(nr int, key *byte, enc *uint32, dec *uint32)
464 //
465 // Uses scalar S-box lookup (sbox0) for subw(); no VSHUFB needed here
466 // since key schedule processes 4 bytes at a time, not 16.
467 // -----------------------------------------------------------------------
468 TEXT ·expandKeyAsm(SB), NOSPLIT, $0-32
469 MOVV nr+0(FP), R4
470 MOVV key+8(FP), R5
471 MOVV enc+16(FP), R6
472 MOVV dec+24(FP), R7
473
474 MOVV $sbox0_0(SB), R8 // S-box for subw()
475 MOVV $·powx(SB), R14 // Rcon table
476
477 // roundKeysSize = (nr+1)*4
478 ADDV $1, R4, R9
479 SLLV $2, R9, R9 // R9 = (nr+1)*4
480
481 // nk = nr - 6 (AES-128→4, AES-192→6, AES-256→8)
482 ADDV $-6, R4, R10
483
484 // Precompute the "nk > 6" flag once, instead of re-deriving it
485 // (via SUBV $6, R10, R18 + BEQ/BLT) on every iteration where pos==4.
486 MOVV $0, R25 // R25 = 0 (false) by default
487 MOVV $6, R18
488 BGE R18, R10, Lnk6_done // nk <= 6 -> flag stays false
489 MOVV $1, R25 // nk > 6 -> flag = true
490
491 Lnk6_done:
492 // Hoist the "pos == 4" comparison constant out of the per-word loop.
493 MOVV $4, R20
494
495 MOVV R6, R17 // save enc base for Phase 3
496
497 // Phase 1: load initial key words (i = 0..nk-1)
498 MOVV $0, R11 // i = 0
499 Linit_loop:
500 BEQ R11, R10, Linit_done
501 SLLV $2, R11, R13
502 ADDV R5, R13, R13 // &key[i*4]
503 MOVWU (R13), R12 // load 4 bytes (little-endian)
504 REVB2W R12, R12 // → big-endian word
505 MOVW R12, (R6) // enc[i] = word
506 ADDV $4, R6
507 ADDV $1, R11
508 JMP Linit_loop
509
510 Linit_done:
511 // Phase 2: key expansion (i = nk..roundKeysSize-1)
512 // pos = i%nk (counter, reset to 0 when reaches nk)
513 // rcon_idx = i/nk - 1 (incremented each time pos wraps)
514 MOVV $0, R15 // pos = 0 (nk%nk = 0)
515 MOVV $0, R16 // rcon_idx = 0
516
517 Lexpand_loop:
518 BEQ R11, R9, Lexpand_enc_done
519
520 MOVWU -4(R6), R12 // t = enc[i-1]
521
522 BNE R15, R0, Lcheck_nk6 // pos != 0 → skip i%nk==0 branch
523
524 // i%nk == 0: t = subw(rotw(t)) ^ (powx[rcon_idx] << 24)
525 ROTR $24, R12, R12 // rotw: rotate-left-8 = rotate-right-24
526 SUBW_INLINE(R12, R8, R13, R19)
527 ADDV R14, R16, R13 // &powx[rcon_idx]
528 MOVBU (R13), R13
529 SLLV $24, R13, R13
530 XOR R13, R12 // t ^= rcon
531 ADDV $1, R16 // rcon_idx++
532 JMP Ldo_xor
533
534 Lcheck_nk6:
535 BNE R15, R20, Ldo_xor // pos != 4 -> skip (R20 preloaded with 4)
536 BEQ R25, R0, Ldo_xor // nk <= 6 -> skip (R25 preloaded flag)
537 // nk > 6 && pos == 4: t = subw(t)
538 SUBW_INLINE(R12, R8, R13, R19)
539
540 Ldo_xor:
541 SLLV $2, R10, R13
542 SUBV R13, R6, R13 // &enc[i-nk] (R6 - nk*4)
543 MOVWU (R13), R13 // enc[i-nk]
544 XOR R13, R12 // t ^= enc[i-nk]
545 MOVW R12, (R6) // enc[i] = t
546 ADDV $4, R6
547 ADDV $1, R11
548 ADDV $1, R15 // pos++
549 BNE R15, R10, Lexpand_loop
550 MOVV $0, R15 // pos reset
551 JMP Lexpand_loop
552
553 Lexpand_enc_done:
554 BEQ R7, R0, Lexpand_done // dec == nil → skip
555
556 // ---- Phase 3 (vectorized): derive dec[] from enc[] ----
557 // n = roundKeysSize (R9), enc base = R17 (saved earlier), dec base = R7
558 // Load rotation tables (rot1/rot2/rot3) needed by INVMIXCOLUMNS.
559 MOVV $rot1(SB), R26
560 MOVV $rot2(SB), R27
561 MOVV $rot3(SB), R28
562 MOVV $byteSwap32(SB), R24
563 VMOVQ (R26), V4
564 VMOVQ (R27), V5
565 VMOVQ (R28), V6
566 VMOVQ (R24), V2
567
568 MOVV $sbox0_0(SB), R29 // sbox0 table base (for SUBBYTES)
569
570 MOVV $0, R19 // i = 0
571 Ldec_outer_vec:
572 BEQ R19, R9, Lexpand_done
573 SUBV R19, R9, R20
574 ADDV $-4, R20, R20 // R20 = ei = n - i - 4
575 SLLV $2, R20, R25
576 ADDV R17, R25, R25 // &enc[ei] (R17 = saved enc base)
577 VMOVQ (R25), V0 // load 4 consecutive enc words as one 16-byte group
578
579 // boundary: first (i==0) and last (i+4==n) groups get no InvMixColumns
580 BEQ R19, R0, Ldec_copy_vec
581 ADDV $4, R19, R20
582 BEQ R20, R9, Ldec_copy_vec
583
584 VSHUFB V2, V0, V0, V0 // word-storage order -> state order (V2 = byteSwap32 index, same table used in decryptBlockAsm)
585 INVMIXCOLUMNS(V0, V0, V4, V5, V6, V9, V10, V11, V12, V13, V14, V16)
586 VSHUFB V2, V0, V0, V0 // state order -> word-storage order (so decryptBlockAsm's own byte-swap-before-use still works)
587
588 Ldec_copy_vec:
589 SLLV $2, R19, R25
590 ADDV R7, R25, R25 // &dec[i]
591 VMOVQ V0, (R25)
592
593 ADDV $4, R19
594 JMP Ldec_outer_vec
595
596 Lexpand_done:
597 RET
598
View as plain text