Source file test/codegen/simd_arm64.go

     1  // asmcheck
     2  
     3  // Copyright 2026 The Go Authors. All rights reserved.
     4  // Use of this source code is governed by a BSD-style
     5  // license that can be found in the LICENSE file.
     6  
     7  // These tests check ARM64 SIMD code generation and peephole optimizations.
     8  
     9  //go:build goexperiment.simd && arm64
    10  
    11  package codegen
    12  
    13  import (
    14  	"simd/archsimd"
    15  )
    16  
    17  //go:noinline
    18  func forceSpill() {}
    19  
    20  func spillAroundCall(a archsimd.Int8x16) archsimd.Int8x16 {
    21  	forceSpill()
    22  	// arm64:`FMOVQ` `FMOVQ`
    23  	return a
    24  }
    25  
    26  var (
    27  	sinkU8  archsimd.Uint8x16
    28  	sinkI8  archsimd.Int8x16
    29  	sinkU16 archsimd.Uint16x8
    30  	sinkU32 archsimd.Uint32x4
    31  	sinkU64 archsimd.Uint64x2
    32  	sinkF32 archsimd.Float32x4
    33  	sinkF64 archsimd.Float64x2
    34  )
    35  
    36  func broadcastConstImmFold(k int) {
    37  	switch k {
    38  	case 0:
    39  		// arm64:`VMOVI [$]0,` -`VDUP`
    40  		sinkU8 = archsimd.BroadcastUint8x16(0)
    41  	case 1:
    42  		// arm64:`VMOVI [$]1,` -`VDUP`
    43  		sinkU8 = archsimd.BroadcastUint8x16(1)
    44  	case 127:
    45  		// arm64:`VMOVI [$]127,` -`VDUP`
    46  		sinkI8 = archsimd.BroadcastInt8x16(127)
    47  	case 128:
    48  		// arm64:`VMOVI [$]128,` -`VDUP`
    49  		sinkU8 = archsimd.BroadcastUint8x16(128)
    50  	case -128:
    51  		// arm64:`VMOVI [$]128,` -`VDUP`
    52  		sinkI8 = archsimd.BroadcastInt8x16(-128)
    53  	case 255:
    54  		// arm64:`VMOVI [$]255,` -`VDUP`
    55  		sinkU8 = archsimd.BroadcastUint8x16(255)
    56  	case -1:
    57  		// arm64:`VMOVI [$]255,` -`VDUP`
    58  		sinkI8 = archsimd.BroadcastInt8x16(-1)
    59  	case -2:
    60  		// arm64:`VMOVI [$]254,` -`VDUP`
    61  		sinkI8 = archsimd.BroadcastInt8x16(-2)
    62  	default:
    63  		// arm64:`VMOV R0, V\d+.B\[0\]` `VDUP`
    64  		sinkI8 = archsimd.BroadcastInt8x16(int8(k))
    65  	}
    66  }
    67  
    68  func shiftAllImmFold(k int) {
    69  	switch k {
    70  	case 100:
    71  		// arm64:`VMOVI [$]100,` `VSSHL` -`VDUP`
    72  		sinkI8 = sinkI8.ShiftAllLeft(100)
    73  		// arm64:`VMOVI [$]156,` `VUSHL` -`VDUP`
    74  		sinkU8 = sinkU8.ShiftAllRight(100)
    75  	}
    76  }
    77  
    78  func setHiUint32(x, lo archsimd.Uint32x4) {
    79  	// arm64:`VMOV V1.D\[0\], V0.D\[1\]`
    80  	sinkU32 = loToHiUint32Vec(x, lo)
    81  }
    82  
    83  func setHiFloat64(x, lo archsimd.Float64x2) {
    84  	// arm64:`VMOV V1.D\[0\], V0.D\[1\]`
    85  	sinkF64 = x.SetElem(1, lo.GetElem(0))
    86  }
    87  
    88  func getHiFloat32(x archsimd.Float32x4) {
    89  	// arm64:`VDUP V0.D\[1\],`
    90  	sinkF32 = x.HiToLo()
    91  }
    92  
    93  func getHiFloat64(x archsimd.Float64x2) {
    94  	// arm64:`VDUP V0.D\[1\],`
    95  	sinkF64 = x.HiToLo()
    96  }
    97  
    98  func foldGetHiSetHiMuls(a, b archsimd.Uint16x8) archsimd.Uint16x8 {
    99  	wLo := a.MulWidenLo(b)                     // arm64: `VUMULL V0.H4, V1.H4, V[0-9].S4`
   100  	wHi := a.HiToLo().MulWidenLo(b.HiToLo())   // arm64: `VUMULL2 V1.H8, V0.H8, V[0-9].S4` -`VDUP`
   101  	narrowLo := wLo.TruncToUint16()            // arm64: `VXTN V[0-9]+.S4, V0.H4`
   102  	narrowHi := wHi.TruncToUint16()            // folded into next line
   103  	return loToHiUint16Vec(narrowLo, narrowHi) // arm64: `VXTN2 V[0-9]+.S4, V0.H8`
   104  }
   105  
   106  func carrylessMultiplies(x, y archsimd.Uint64x2) archsimd.Uint64x2 {
   107  	lo := x.CarrylessMultiplyEven(y)                   // arm64:`VPMULL V` -`VPMULL2`
   108  	hi := x.HiToLo().CarrylessMultiplyEven(y.HiToLo()) // arm64:`VPMULL2 V` -`VPMULL `
   109  	return lo.Xor(hi)
   110  }
   111  
   112  func mergeWithNotMask(x, y archsimd.Int8x16, mask archsimd.Mask8x16, f1, f2 archsimd.Float32x4) {
   113  	// arm64:`VBIT` -`VBIF` -`VNOT`
   114  	sinkI8 = x.IfElse(mask.Not(), y)
   115  	// arm64: `VFCMEQ`
   116  	eq := f1.Equal(f2)
   117  	// The next line `ne` should be CSEd with `eq` above
   118  	ne := f1.NotEqual(f2)    // arm64: -`.*`
   119  	feq := f1.IfElse(eq, f2) // arm64:`VBIF`
   120  	fne := f1.IfElse(ne, f2) // arm64:`VBIT`
   121  	sinkF32 = fne.Add(feq)
   122  }
   123  
   124  // loToHiUint32Vec returns a vector with the lower 64 bits of x preserved and
   125  // the upper 64 bits replaced with the lower 64 bits of lo.
   126  // It routes through Float64x2 to stay in the FP/SIMD register file,
   127  // avoiding a round-trip through a GP register.
   128  func loToHiUint32Vec(x, lo archsimd.Uint32x4) archsimd.Uint32x4 {
   129  	return x.ReshapeToUint64s().BitsToFloat64().SetElem(1, lo.ReshapeToUint64s().BitsToFloat64().GetElem(0)).ToBits().ReshapeToUint32s()
   130  }
   131  
   132  // loToHiUint16Vec returns a vector with the lower 64 bits of x preserved and
   133  // the upper 64 bits replaced with the lower 64 bits of lo.
   134  // It routes through Float64x2 to stay in the FP/SIMD register file,
   135  // avoiding a round-trip through a GP register.
   136  func loToHiUint16Vec(x, lo archsimd.Uint16x8) archsimd.Uint16x8 {
   137  	return x.ReshapeToUint64s().BitsToFloat64().SetElem(1, lo.ReshapeToUint64s().BitsToFloat64().GetElem(0)).ToBits().ReshapeToUint16s()
   138  }
   139  

View as plain text