Source file test/codegen/simd_converts_amd64.go

     1  // asmcheck
     2  
     3  // Copyright 2026 The Go Authors. All rights reserved.
     4  // Use of this source code is governed by a BSD-style
     5  // license that can be found in the LICENSE file.
     6  
     7  // These tests check code generation of amd64 SIMD conversions.
     8  
     9  //go:build goexperiment.simd && amd64
    10  
    11  package codegen
    12  
    13  import "simd/archsimd"
    14  
    15  // A widening conversion reads only the low part of its input, so a load of
    16  // the input folds into it, whatever the input's width.
    17  
    18  func simdExtendToInt32LoadFold(s []int8) archsimd.Int32x8 {
    19  	// amd64:`VPMOVSXBD \([A-Z0-9]+\), Y[0-9]+`
    20  	return archsimd.LoadInt8x32(s).ExtendToInt32()
    21  }
    22  
    23  func simdExtendLo8ToInt32LoadFold(s []int8) archsimd.Int32x8 {
    24  	// amd64:`VPMOVSXBD \([A-Z0-9]+\), Y[0-9]+`
    25  	return archsimd.LoadInt8x16(s).ExtendLo8ToInt32()
    26  }
    27  
    28  // The 128- and 256-bit 2:1 narrowing conversions are emulated with AVX and
    29  // AVX2 instructions. Check that they stay inlined and short.
    30  
    31  func simdTruncToUint8x16(x archsimd.Uint16x8) archsimd.Uint8x16 {
    32  	// amd64:`VPSHUFB` -`CALL`
    33  	return x.TruncToUint8()
    34  }
    35  
    36  func simdSaturateToInt8x32(x archsimd.Int16x16) archsimd.Int8x32 {
    37  	// amd64:`VPACKSSWB` `VPERMD` -`CALL`
    38  	return x.SaturateToInt8()
    39  }
    40  
    41  func simdSaturateToUint16x8(x archsimd.Uint32x4) archsimd.Uint16x8 {
    42  	// amd64:`VPMINUD` `VPACKUSDW` -`CALL`
    43  	return x.SaturateToUint16()
    44  }
    45  
    46  func simdSaturateToInt32x4(x archsimd.Int64x2) archsimd.Int32x4 {
    47  	// amd64:`VPCMPGTQ` `VPBLENDVB` `VSHUFPS` -`CALL`
    48  	return x.SaturateToInt32()
    49  }
    50  
    51  // Enlarge is a single VEX-encoded move, which zeroes the upper half of the
    52  // result. It must write a register of the source's width, not of the result's.
    53  
    54  func simdEnlargeInt32x4(x archsimd.Int32x4) archsimd.Int32x8 {
    55  	// amd64:`VMOVDQA X[0-9]+, X[0-9]+` -`CALL`
    56  	return x.Enlarge()
    57  }
    58  
    59  func simdEnlargeFloat64x4(x archsimd.Float64x4) archsimd.Float64x8 {
    60  	// amd64:`VMOVAPD Y[0-9]+, Y[0-9]+` -`CALL`
    61  	return x.Enlarge()
    62  }
    63  
    64  // x.Enlarge().ExtendTo*() replaces the widening ExtendLo* conversions. A
    65  // widening conversion reads only the low part of its input, which is x, so the
    66  // Enlarge disappears and the result is the same single instruction as the
    67  // ExtendLo* conversion, including when it folds a load.
    68  
    69  func simdEnlargeExtendToInt32(x archsimd.Int8x16) archsimd.Int32x8 {
    70  	// amd64:`VPMOVSXBD X[0-9]+, Y[0-9]+` -`VMOVDQA` -`CALL`
    71  	return x.Enlarge().ExtendToInt32()
    72  }
    73  
    74  func simdEnlargeExtendToUint32(x archsimd.Uint8x16) archsimd.Uint32x8 {
    75  	// amd64:`VPMOVZXBD X[0-9]+, Y[0-9]+` -`VMOVDQA` -`CALL`
    76  	return x.Enlarge().ExtendToUint32()
    77  }
    78  
    79  func simdEnlargeExtendToInt64FromInt8(x archsimd.Int8x16) archsimd.Int64x4 {
    80  	// amd64:`VPMOVSXBQ X[0-9]+, Y[0-9]+` -`VMOVDQA` -`CALL`
    81  	return x.Enlarge().ExtendToInt64()
    82  }
    83  
    84  func simdEnlargeExtendToUint64FromUint16(x archsimd.Uint16x8) archsimd.Uint64x4 {
    85  	// amd64:`VPMOVZXWQ X[0-9]+, Y[0-9]+` -`VMOVDQA` -`CALL`
    86  	return x.Enlarge().ExtendToUint64()
    87  }
    88  
    89  func simdEnlargeEnlargeExtendToInt64(x archsimd.Int8x16) archsimd.Int64x8 {
    90  	// amd64:`VPMOVSXBQ X[0-9]+, Z[0-9]+` -`VMOVDQA` -`CALL`
    91  	return x.Enlarge().Enlarge().ExtendToInt64()
    92  }
    93  
    94  func simdEnlargeExtendToInt32LoadFold(s []int8) archsimd.Int32x8 {
    95  	// amd64:`VPMOVSXBD \([A-Z0-9]+\), Y[0-9]+` -`VMOVDQA` -`VMOVDQU`
    96  	return archsimd.LoadInt8x16(s).Enlarge().ExtendToInt32()
    97  }
    98  
    99  func simdEnlargeEnlargeExtendToUint64LoadFold(s []uint8) archsimd.Uint64x8 {
   100  	// amd64:`VPMOVZXBQ \([A-Z0-9]+\), Z[0-9]+` -`VMOVDQA` -`VMOVDQU`
   101  	return archsimd.LoadUint8x16(s).Enlarge().Enlarge().ExtendToUint64()
   102  }
   103  
   104  // The same applies to the other operations that read only the low part of an
   105  // input, such as widening float conversions, and at 512 bits.
   106  
   107  func simdEnlargeConvertToFloat64(x archsimd.Float32x4) archsimd.Float64x4 {
   108  	// amd64:`VCVTPS2PD X[0-9]+, Y[0-9]+` -`VMOVAPS` -`CALL`
   109  	return x.Enlarge().ConvertToFloat64()
   110  }
   111  
   112  func simdEnlargeExtendToInt32Wide(x archsimd.Int16x16) archsimd.Int32x16 {
   113  	// amd64:`VPMOVSXWD Y[0-9]+, Z[0-9]+` -`VMOVDQA` -`CALL`
   114  	return x.Enlarge().ExtendToInt32()
   115  }
   116  
   117  // Masked and merging forms are built from the unmasked conversion, so they
   118  // read x directly as well.
   119  
   120  func simdEnlargeExtendToInt32Masked(x archsimd.Int8x16, m uint8) archsimd.Int32x8 {
   121  	if archsimd.X86.AVX512() {
   122  		mask := archsimd.Mask32x8FromBits(m)
   123  		// amd64:`VPMOVSXBD.Z X[0-9]+, K[0-7], Y[0-9]+` -`VMOVDQA`
   124  		return x.Enlarge().ExtendToInt32().Masked(mask)
   125  	}
   126  	return archsimd.Int32x8{}
   127  }
   128  
   129  func simdEnlargeExtendToInt32Merge(x archsimd.Int8x16, y archsimd.Int32x8, m uint8) archsimd.Int32x8 {
   130  	if archsimd.X86.AVX512() {
   131  		mask := archsimd.Mask32x8FromBits(m)
   132  		// amd64:`VPMOVSXBD X[0-9]+, K[0-7], Y[0-9]+` -`VMOVDQA`
   133  		return x.Enlarge().ExtendToInt32().Merge(y, mask)
   134  	}
   135  	return y
   136  }
   137  
   138  // If the Enlarge has a use that reads all of it, the Enlarge stays, but the
   139  // conversion still reads x.
   140  
   141  func simdEnlargeExtendToInt32Shared(x archsimd.Int8x16, y archsimd.Int8x32) (archsimd.Int32x8, archsimd.Int8x32) {
   142  	e := x.Enlarge()                   // amd64:`VMOVDQA X0, X[0-9]+`
   143  	return e.ExtendToInt32(), e.Add(y) // amd64:`VPMOVSXBD X0, Y[0-9]+`
   144  }
   145  
   146  // A load narrower than the conversion's input type also folds, as long as it
   147  // contains everything the conversion reads.
   148  
   149  func simdEnlargeExtendToInt32LoadFold256(s []int16) archsimd.Int32x16 {
   150  	// amd64:`VPMOVSXWD \([A-Z0-9]+\), Z[0-9]+` -`VMOVDQA` -`VMOVDQU`
   151  	return archsimd.LoadInt16x16(s).Enlarge().ExtendToInt32()
   152  }
   153  
   154  func simdEnlargeConvertToFloat64LoadFold256(s []float32) archsimd.Float64x8 {
   155  	// amd64:`VCVTPS2PD \([A-Z0-9]+\), Z[0-9]+` -`VMOVAPS` -`VMOVUPS` -`VMOVDQU`
   156  	return archsimd.LoadFloat32x8(s).Enlarge().ConvertToFloat64()
   157  }
   158  
   159  // Enlarge needs no move when its input's register is already zero above the
   160  // input, as it is when a VEX or EVEX instruction writes it, including
   161  // through phis, spills, and restores.
   162  
   163  func simdEnlargeOfResult(x, y archsimd.Int32x4) archsimd.Int32x8 {
   164  	// amd64:`VPADDD X[0-9]+, X[0-9]+, X[0-9]+` -`VMOVDQA`
   165  	return x.Add(y).Enlarge()
   166  }
   167  
   168  func simdEnlargeOfLoad(s []int32) archsimd.Int32x8 {
   169  	// amd64:`VMOVDQU \([A-Z0-9]+\), X[0-9]+` -`VMOVDQA`
   170  	return archsimd.LoadInt32x4(s).Enlarge()
   171  }
   172  
   173  func simdEnlargeEnlargeOfResult(x, y archsimd.Float32x4) archsimd.Float32x16 {
   174  	// amd64:`VADDPS X[0-9]+, X[0-9]+, X[0-9]+` -`VMOVAPS`
   175  	return x.Add(y).Enlarge().Enlarge()
   176  }
   177  
   178  func simdEnlargeOfResultFullWidthUse(x, y archsimd.Int32x4, z archsimd.Int32x8) archsimd.Int32x8 {
   179  	// amd64:`VPADDD X[0-9]+, X[0-9]+, X[0-9]+` `VPADDD Y[0-9]+, Y[0-9]+, Y[0-9]+` -`VMOVDQA`
   180  	return x.Add(y).Enlarge().Add(z)
   181  }
   182  
   183  func simdEnlargeOfPhi(x, y archsimd.Int32x4, c bool) archsimd.Int32x8 {
   184  	v := x.Add(y)
   185  	if c {
   186  		v = x.Sub(y)
   187  	}
   188  	// amd64:-`VMOVDQA`
   189  	return v.Enlarge()
   190  }
   191  
   192  var simdSinkInt32x4 archsimd.Int32x4
   193  
   194  //go:noinline
   195  func simdClobber() {}
   196  
   197  func simdEnlargeOfRestored(x, y archsimd.Int32x4) archsimd.Int32x8 {
   198  	v := x.Add(y)
   199  	simdSinkInt32x4 = v
   200  	simdClobber()
   201  	// amd64:`VMOVUPS [^,]+\(SP\), X[0-9]+` -`VMOVDQA`
   202  	return v.Enlarge()
   203  }
   204  
   205  // But Enlarge still needs its move when the upper bits of its input's
   206  // register are unknown, as for arguments and call results (see also
   207  // simdEnlargeInt32x4 above), or a phi with such an input.
   208  
   209  //go:noinline
   210  func simdInt32x4Result() archsimd.Int32x4 {
   211  	return archsimd.Int32x4{}
   212  }
   213  
   214  func simdEnlargeOfCallResult() archsimd.Int32x8 {
   215  	// amd64:`VMOVDQA X[0-9]+, X[0-9]+`
   216  	return simdInt32x4Result().Enlarge()
   217  }
   218  
   219  func simdEnlargeOfPhiWithArg(x, y archsimd.Int32x4, c bool) archsimd.Int32x8 {
   220  	v := x
   221  	if c {
   222  		v = x.Add(y)
   223  	}
   224  	// amd64:`VMOVDQA X[0-9]+, X[0-9]+`
   225  	return v.Enlarge()
   226  }
   227  

View as plain text