Text file
src/runtime/memmove_loong64.s
1 // Copyright 2022 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 #include "go_asm.h"
6 #include "textflag.h"
7
8 // See memmove Go doc for important implementation constraints.
9
10 // Register map
11 //
12 // to R4
13 // from R5
14 // n(aka count) R6
15 // to-end R7
16 // from-end R8
17 // data R11-R18
18 // tmp R9
19
20 // Algorithm:
21 //
22 // Memory alignment check is only performed for copy size greater
23 // than 64 bytes to minimize overhead.
24 //
25 // when copy size <= 64 bytes, jump to label tail, according to the
26 // copy size to select the appropriate case and copy directly.
27 // Based on the common memory access instructions of loong64, the
28 // currently implemented cases are:
29 // move_0, move_1, move_2, move_3, move_4, move_5through7, move_8,
30 // move_9through16, move_17through32, move_33through64
31 //
32 // when copy size > 64 bytes, use the destination-aligned copying,
33 // adopt the following strategy to copy in 3 parts:
34 // 1. Head: do the memory alignment
35 // 2. Body: a 64-byte loop structure
36 // 3. Tail: processing of the remaining part (<= 64 bytes)
37 //
38 // forward:
39 //
40 // Dst NewDst Dstend
41 // | |<----count after correction---->|
42 // |<-------------count before correction---------->|
43 // |<--8-(Dst&7)-->| |<---64 bytes--->|
44 // +------------------------------------------------+
45 // | Head | Body | Tail |
46 // +---------------+---------------+----------------+
47 // NewDst = Dst - (Dst & 7) + 8
48 // count = count - 8 + (Dst & 7)
49 // Src = Src - (Dst & 7) + 8
50 //
51 // backward:
52 //
53 // Dst NewDstend Dstend
54 // |<-----count after correction------>| |
55 // |<------------count before correction--------------->|
56 // |<---64 bytes--->| |<---Dstend&7--->|
57 // +----------------------------------------------------+
58 // | Tail | Body | Head |
59 // +----------------+------------------+----------------+
60 // NewDstend = Dstend - (Dstend & 7)
61 // count = count - (Dstend & 7)
62 // Srcend = Srcend - (Dstend & 7)
63
64 // func memmove(to, from unsafe.Pointer, n uintptr)
65 TEXT runtime·memmove<ABIInternal>(SB), NOSPLIT|NOFRAME, $0-24
66 BEQ R4, R5, move_0
67 BEQ R6, move_0
68
69 ADDV R4, R6, R7 // to-end pointer
70 ADDV R5, R6, R8 // from-end pointer
71
72 // copy size <= 64 bytes, copy directly, not check aligned
73 tail:
74 // < 2 bytes
75 SGTU $2, R6, R9
76 BNE R9, move_1
77
78 // < 3 bytes
79 SGTU $3, R6, R9
80 BNE R9, move_2
81
82 // < 4 bytes
83 SGTU $4, R6, R9
84 BNE R9, move_3
85
86 // < 5 bytes
87 SGTU $5, R6, R9
88 BNE R9, move_4
89
90 // >= 5 bytes and < 8 bytes
91 SGTU $8, R6, R9
92 BNE R9, move_5through7
93
94 // < 9 bytes
95 SGTU $9, R6, R9
96 BNE R9, move_8
97
98 // >= 9 bytes and < 17 bytes
99 SGTU $17, R6, R9
100 BNE R9, move_9through16
101
102 // >= 17 bytes and < 33 bytes
103 SGTU $33, R6, R9
104 BNE R9, move_17through32
105
106 // >= 33 bytes and < 65 bytes: dispatch to scalar or vector
107 // (LSX) handling depending on dst/src alignment.
108 SGTU $65, R6, R9
109 BNE R9, move_33through64_dispatch
110
111 // >= 65 bytes and < 256 bytes
112 SGTU $256, R6, R9
113 BNE R9, move_large
114
115 // >= 256
116 JMP lasx_move_large
117
118 move_0:
119 RET
120
121 move_1:
122 MOVB (R5), R11
123 MOVB R11, (R4)
124 RET
125 move_2:
126 MOVH (R5), R11
127 MOVH R11, (R4)
128 RET
129 move_3:
130 MOVH (R5), R11
131 MOVB -1(R8), R12
132 MOVH R11, (R4)
133 MOVB R12, -1(R7)
134 RET
135 move_4:
136 MOVW (R5), R11
137 MOVW R11, (R4)
138 RET
139 move_5through7:
140 MOVW (R5), R11
141 MOVW -4(R8), R12
142 MOVW R11, (R4)
143 MOVW R12, -4(R7)
144 RET
145 move_8:
146 MOVV (R5), R11
147 MOVV R11, (R4)
148 RET
149 move_9through16:
150 MOVV (R5), R11
151 MOVV -8(R8), R12
152 MOVV R11, (R4)
153 MOVV R12, -8(R7)
154 RET
155 move_17through32:
156 MOVV (R5), R11
157 MOVV 8(R5), R12
158 MOVV -16(R8), R13
159 MOVV -8(R8), R14
160 MOVV R11, (R4)
161 MOVV R12, 8(R4)
162 MOVV R13, -16(R7)
163 MOVV R14, -8(R7)
164 RET
165 move_33through64:
166 MOVV (R5), R11
167 MOVV 8(R5), R12
168 MOVV 16(R5), R13
169 MOVV 24(R5), R14
170 MOVV -32(R8), R15
171 MOVV -24(R8), R16
172 MOVV -16(R8), R17
173 MOVV -8(R8), R18
174 MOVV R11, (R4)
175 MOVV R12, 8(R4)
176 MOVV R13, 16(R4)
177 MOVV R14, 24(R4)
178 MOVV R15, -32(R7)
179 MOVV R16, -24(R7)
180 MOVV R17, -16(R7)
181 MOVV R18, -8(R7)
182 RET
183
184 // move_33through64_dispatch only runs for the 33~64 byte bucket.
185 // Aligned dst/src (the common case) falls straight through to the
186 // original scalar move_33through64 after 3 extra instructions
187 // (OR/AND/BEQ). Unaligned dst/src without LSX support falls through
188 // after 5 extra instructions (OR/AND/BEQ/MOVBU/BEQ). Unaligned
189 // dst/src with LSX available uses four 128-bit vector load/store
190 // pairs instead of the eight scalar MOVV load/store pairs used by
191 // move_33through64.
192 move_33through64_dispatch:
193 OR R4, R5, R9
194 AND $7, R9
195 BEQ R9, move_33through64
196 MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLSX(SB), R9
197 BEQ R9, move_33through64
198 JMP lsx_move_33through64
199
200 lsx_move_33through64:
201 VMOVQ (R5), V0
202 VMOVQ 16(R5), V1
203 VMOVQ -32(R8), V2
204 VMOVQ -16(R8), V3
205 VMOVQ V0, (R4)
206 VMOVQ V1, 16(R4)
207 VMOVQ V2, -32(R7)
208 VMOVQ V3, -16(R7)
209 RET
210
211 move_large:
212 // if (dst > src) && (dst < (src + count))
213 // regarded as memory overlap
214 // jump to backward
215 // else
216 // jump to forward
217 BGEU R5, R4, forward
218 ADDV R5, R6, R10
219 BLTU R4, R10, backward
220 forward:
221 AND $7, R4, R9 // dst & 7
222 BEQ R9, forward_move_64loop
223 forward_unaligned:
224 MOVV $8, R10
225 SUBV R9, R10 // head = 8 - (dst & 7)
226 MOVV (R5), R11
227 SUBV R10, R6 // newcount = count - (8 - (dst & 7))
228 ADDV R10, R5 // newsrc = src + (8 - (dst & 7))
229 MOVV (R5), R12
230 MOVV R11, (R4)
231 ADDV R10, R4 // newdst = dst + (8 - (dst & 7))
232 MOVV R12, (R4)
233 SUBV $8, R6
234 ADDV $8, R4
235 ADDV $8, R5
236 SGTU $65, R6, R9
237 BNE R9, move_33through64
238 forward_move_64loop:
239 SUBV $64, R6
240 SGTU $64, R6, R9
241 MOVV (R5), R11
242 MOVV 8(R5), R12
243 MOVV 16(R5), R13
244 MOVV 24(R5), R14
245 MOVV 32(R5), R15
246 MOVV 40(R5), R16
247 MOVV 48(R5), R17
248 MOVV 56(R5), R18
249 MOVV R11, (R4)
250 MOVV R12, 8(R4)
251 MOVV R13, 16(R4)
252 MOVV R14, 24(R4)
253 MOVV R15, 32(R4)
254 MOVV R16, 40(R4)
255 MOVV R17, 48(R4)
256 MOVV R18, 56(R4)
257 ADDV $64, R5
258 ADDV $64, R4
259 BEQ R9, forward_move_64loop
260 // 0 < remaining_length < 64
261 BNE R6, tail
262 RET
263
264 // The backward copy algorithm is the same as the forward
265 // copy, except for the direction.
266 backward:
267 AND $7, R7, R9 // dstend & 7
268 BEQ R9, backward_move_64loop
269 backward_unaligned:
270 MOVV -8(R8), R11
271 SUBV R9, R6 // newcount = count - (dstend & 7)
272 SUBV R9, R8 // newsrcend = srcend - (dstend & 7)
273 MOVV -8(R8), R12
274 MOVV R11, -8(R7)
275 SUBV R9, R7 // newdstend = dstend - (dstend & 7)
276 MOVV R12, -8(R7)
277 SUBV $8, R6
278 SUBV $8, R7
279 SUBV $8, R8
280 SGTU $65, R6, R9
281 BNE R9, move_33through64
282 backward_move_64loop:
283 SUBV $64, R6
284 SGTU $64, R6, R9
285 MOVV -8(R8), R11
286 MOVV -16(R8), R12
287 MOVV -24(R8), R13
288 MOVV -32(R8), R14
289 MOVV -40(R8), R15
290 MOVV -48(R8), R16
291 MOVV -56(R8), R17
292 MOVV -64(R8), R18
293 MOVV R11, -8(R7)
294 MOVV R12, -16(R7)
295 MOVV R13, -24(R7)
296 MOVV R14, -32(R7)
297 MOVV R15, -40(R7)
298 MOVV R16, -48(R7)
299 MOVV R17, -56(R7)
300 MOVV R18, -64(R7)
301 SUBV $64, R7
302 SUBV $64, R8
303 BEQ R9, backward_move_64loop
304 // 0 < remaining_length < 64
305 BNE R6, tail
306 RET
307
308 // use simd 128 instructions to implement memmove
309 // n >= 256 bytes, check 16-byte alignment
310 lsx_move_large:
311 MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLSX(SB), R9
312 BEQ R9, move_large
313
314 // if (dst > src) && (dst < (src + count))
315 // regarded as memory overlap
316 // jump to lsx_backward
317 // else
318 // jump to lsx_forward
319 BGEU R5, R4, lsx_forward
320 ADDV R5, R6, R10
321 BLTU R4, R10, lsx_backward
322 lsx_forward:
323 AND $15, R4, R9 // dst & 15
324 BEQ R9, lsx_forward_move_128
325 lsx_forward_unaligned:
326 MOVV $16, R10
327 SUBV R9, R10 // head = 16 - (dst & 15)
328 VMOVQ (R5), V0
329 SUBV R10, R6 // newcount = count - (16 - (dst & 15))
330 ADDV R10, R5 // newsrc = src + (16 - (dst & 15))
331 VMOVQ (R5), V1
332 VMOVQ V0, (R4)
333 ADDV R10, R4 // newdst = dst + (16 - (dst & 15))
334 VMOVQ V1, (R4)
335 SUBV $16, R6
336 ADDV $16, R4
337 ADDV $16, R5
338 lsx_forward_move_128:
339 SGTU $128, R6, R9
340 BNE R9, lsx_forward_move_32
341 lsx_forward_move_128loop:
342 SUBV $128, R6
343 SGTU $128, R6, R9
344 VMOVQ 0(R5), V0
345 VMOVQ 16(R5), V1
346 VMOVQ 32(R5), V2
347 VMOVQ 48(R5), V3
348 VMOVQ 64(R5), V4
349 VMOVQ 80(R5), V5
350 VMOVQ 96(R5), V6
351 VMOVQ 112(R5), V7
352 VMOVQ V0, 0(R4)
353 VMOVQ V1, 16(R4)
354 VMOVQ V2, 32(R4)
355 VMOVQ V3, 48(R4)
356 VMOVQ V4, 64(R4)
357 VMOVQ V5, 80(R4)
358 VMOVQ V6, 96(R4)
359 VMOVQ V7, 112(R4)
360 ADDV $128, R5
361 ADDV $128, R4
362 BEQ R9, lsx_forward_move_128loop
363 lsx_forward_move_32:
364 SGTU $32, R6, R9
365 BNE R9, lsx_forward_move_tail
366 lsx_forward_move_32loop:
367 SUBV $32, R6
368 SGTU $32, R6, R9
369 VMOVQ 0(R5), V0
370 VMOVQ 16(R5), V1
371 VMOVQ V0, 0(R4)
372 VMOVQ V1, 16(R4)
373 ADDV $32, R5
374 ADDV $32, R4
375 BEQ R9, lsx_forward_move_32loop
376 lsx_forward_move_tail:
377 // 0 < remaining_length < 64
378 BNE R6, tail
379 RET
380
381 lsx_backward:
382 AND $15, R7, R9 // dstend & 15
383 BEQ R9, lsx_backward_move_128
384 lsx_backward_unaligned:
385 VMOVQ -16(R8), V0
386 SUBV R9, R6 // newcount = count - (dstend & 15)
387 SUBV R9, R8 // newsrcend = srcend - (dstend & 15)
388 VMOVQ -16(R8), V1
389 VMOVQ V0, -16(R7)
390 SUBV R9, R7 // newdstend = dstend - (dstend & 15)
391 VMOVQ V1, -16(R7)
392 SUBV $16, R6
393 SUBV $16, R7
394 SUBV $16, R8
395 lsx_backward_move_128:
396 SGTU $128, R6, R9
397 BNE R9, lsx_backward_move_32
398 lsx_backward_move_128loop:
399 SUBV $128, R6
400 SGTU $128, R6, R9
401 VMOVQ -16(R8), V0
402 VMOVQ -32(R8), V1
403 VMOVQ -48(R8), V2
404 VMOVQ -64(R8), V3
405 VMOVQ -80(R8), V4
406 VMOVQ -96(R8), V5
407 VMOVQ -112(R8), V6
408 VMOVQ -128(R8), V7
409 VMOVQ V0, -16(R7)
410 VMOVQ V1, -32(R7)
411 VMOVQ V2, -48(R7)
412 VMOVQ V3, -64(R7)
413 VMOVQ V4, -80(R7)
414 VMOVQ V5, -96(R7)
415 VMOVQ V6, -112(R7)
416 VMOVQ V7, -128(R7)
417 SUBV $128, R8
418 SUBV $128, R7
419 BEQ R9, lsx_backward_move_128loop
420 lsx_backward_move_32:
421 SGTU $32, R6, R9
422 BNE R9, lsx_backward_move_tail
423 lsx_backward_move_32loop:
424 SUBV $32, R6
425 SGTU $32, R6, R9
426 VMOVQ -16(R8), V0
427 VMOVQ -32(R8), V1
428 VMOVQ V0, -16(R7)
429 VMOVQ V1, -32(R7)
430 SUBV $32, R8
431 SUBV $32, R7
432 BEQ R9, lsx_backward_move_32loop
433 lsx_backward_move_tail:
434 // 0 < remaining_length < 64
435 BNE R6, tail
436 RET
437
438 // use simd 256 instructions to implement memmove
439 // n >= 256 bytes, check 32-byte alignment
440 lasx_move_large:
441 MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLASX(SB), R9
442 BEQ R9, lsx_move_large
443
444 // if (dst > src) && (dst < (src + count))
445 // regarded as memory overlap
446 // jump to lasx_backward
447 // else
448 // jump to lasx_forward
449 BGEU R5, R4, lasx_forward
450 ADDV R5, R6, R10
451 BLTU R4, R10, lasx_backward
452 lasx_forward:
453 AND $31, R4, R9 // dst & 31
454 BEQ R9, lasx_forward_move_256
455 lasx_forward_unaligned:
456 MOVV $32, R10
457 SUBV R9, R10 // head = 32 - (dst & 31)
458 XVMOVQ (R5), X0
459 SUBV R10, R6 // newcount = count - (32 - (dst & 31))
460 ADDV R10, R5 // newsrc = src + (32 - (dst & 31))
461 XVMOVQ (R5), X1
462 XVMOVQ X0, (R4)
463 ADDV R10, R4 // newdst = dst + (32 - (dst & 31))
464 XVMOVQ X1, (R4)
465 SUBV $32, R6
466 ADDV $32, R4
467 ADDV $32, R5
468 lasx_forward_move_256:
469 SGTU $256, R6, R9
470 BNE R9, lasx_forward_move_64
471 lasx_forward_move_256loop:
472 SUBV $256, R6
473 SGTU $256, R6, R9
474 XVMOVQ 0(R5), X0
475 XVMOVQ 32(R5), X1
476 XVMOVQ 64(R5), X2
477 XVMOVQ 96(R5), X3
478 XVMOVQ 128(R5), X4
479 XVMOVQ 160(R5), X5
480 XVMOVQ 192(R5), X6
481 XVMOVQ 224(R5), X7
482 XVMOVQ X0, 0(R4)
483 XVMOVQ X1, 32(R4)
484 XVMOVQ X2, 64(R4)
485 XVMOVQ X3, 96(R4)
486 XVMOVQ X4, 128(R4)
487 XVMOVQ X5, 160(R4)
488 XVMOVQ X6, 192(R4)
489 XVMOVQ X7, 224(R4)
490 ADDV $256, R5
491 ADDV $256, R4
492 BEQ R9, lasx_forward_move_256loop
493 lasx_forward_move_64:
494 SGTU $64, R6, R9
495 BNE R9, lasx_forward_move_tail
496 lasx_forward_move_64loop:
497 SUBV $64, R6
498 SGTU $64, R6, R9
499 XVMOVQ (R5), X0
500 XVMOVQ 32(R5), X1
501 XVMOVQ X0, (R4)
502 XVMOVQ X1, 32(R4)
503 ADDV $64, R5
504 ADDV $64, R4
505 BEQ R9, lasx_forward_move_64loop
506 lasx_forward_move_tail:
507 // 0 < remaining_length < 64
508 BNE R6, tail
509 RET
510
511 lasx_backward:
512 AND $31, R7, R9 // dstend & 31
513 BEQ R9, lasx_backward_move_256
514 lasx_backward_unaligned:
515 XVMOVQ -32(R8), X0
516 SUBV R9, R6 // newcount = count - (dstend & 31)
517 SUBV R9, R8 // newsrcend = srcend - (dstend & 31)
518 XVMOVQ -32(R8), X1
519 XVMOVQ X0, -32(R7)
520 SUBV R9, R7 // newdstend = dstend - (dstend & 31)
521 XVMOVQ X1, -32(R7)
522 SUBV $32, R6
523 SUBV $32, R7
524 SUBV $32, R8
525 lasx_backward_move_256:
526 SGTU $256, R6, R9
527 BNE R9, lasx_backward_move_64
528 lasx_backward_move_256loop:
529 SUBV $256, R6
530 SGTU $256, R6, R9
531 XVMOVQ -32(R8), X0
532 XVMOVQ -64(R8), X1
533 XVMOVQ -96(R8), X2
534 XVMOVQ -128(R8), X3
535 XVMOVQ -160(R8), X4
536 XVMOVQ -192(R8), X5
537 XVMOVQ -224(R8), X6
538 XVMOVQ -256(R8), X7
539 XVMOVQ X0, -32(R7)
540 XVMOVQ X1, -64(R7)
541 XVMOVQ X2, -96(R7)
542 XVMOVQ X3, -128(R7)
543 XVMOVQ X4, -160(R7)
544 XVMOVQ X5, -192(R7)
545 XVMOVQ X6, -224(R7)
546 XVMOVQ X7, -256(R7)
547 SUBV $256, R8
548 SUBV $256, R7
549 BEQ R9, lasx_backward_move_256loop
550 lasx_backward_move_64:
551 SGTU $64, R6, R9
552 BNE R9, lasx_backward_move_tail
553 lasx_backward_move_64loop:
554 SUBV $64, R6
555 SGTU $64, R6, R9
556 XVMOVQ -32(R8), X0
557 XVMOVQ -64(R8), X1
558 XVMOVQ X0, -32(R7)
559 XVMOVQ X1, -64(R7)
560 SUBV $64, R8
561 SUBV $64, R7
562 BEQ R9, lasx_backward_move_64loop
563 lasx_backward_move_tail:
564 // 0 < remaining_length < 64
565 BNE R6, tail
566 RET
567
View as plain text