Source file src/cmd/compile/internal/ssacompile/expand_calls.go

     1  // Copyright 2020 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package ssacompile
     6  
     7  import (
     8  	"fmt"
     9  
    10  	"cmd/compile/internal/abi"
    11  	"cmd/compile/internal/base"
    12  	"cmd/compile/internal/ir"
    13  	"cmd/compile/internal/ssa"
    14  	"cmd/compile/internal/ssa/block"
    15  	"cmd/compile/internal/ssa/ssaop"
    16  	"cmd/compile/internal/types"
    17  	"cmd/internal/src"
    18  )
    19  
    20  func postExpandCallsDecompose(f *ssa.Func) {
    21  	decomposeUser(f)    // redo user decompose to cleanup after expand calls
    22  	decomposeBuiltin(f) // handles both regular decomposition and cleanup.
    23  }
    24  
    25  func expandCalls(f *ssa.Func) {
    26  	// Convert each aggregate arg to a call into "dismantle aggregate, store/pass parts"
    27  	// Convert each aggregate result from a call into "assemble aggregate from parts"
    28  	// Convert each multivalue exit into "dismantle aggregate, store/return parts"
    29  	// Convert incoming aggregate arg into assembly of parts.
    30  	// Feed modified AST to decompose.
    31  
    32  	sp, _ := f.SpSb()
    33  
    34  	x := &expandState{
    35  		f:               f,
    36  		debug:           f.Pass.Debug,
    37  		regSize:         f.Config.RegSize,
    38  		sp:              sp,
    39  		typs:            &f.Config.Types,
    40  		wideSelects:     make(map[*ssa.Value]*ssa.Value),
    41  		commonArgs:      make(map[selKey]*ssa.Value),
    42  		commonSelectors: make(map[selKey]*ssa.Value),
    43  		memForCall:      make(map[ssa.ID]*ssa.Value),
    44  	}
    45  
    46  	// For 32-bit, need to deal with decomposition of 64-bit integers, which depends on endianness.
    47  	if f.Config.BigEndian {
    48  		x.firstOp = ssaop.OpInt64Hi
    49  		x.secondOp = ssaop.OpInt64Lo
    50  		x.firstType = x.typs.Int32
    51  		x.secondType = x.typs.UInt32
    52  	} else {
    53  		x.firstOp = ssaop.OpInt64Lo
    54  		x.secondOp = ssaop.OpInt64Hi
    55  		x.firstType = x.typs.UInt32
    56  		x.secondType = x.typs.Int32
    57  	}
    58  
    59  	// Defer select processing until after all calls and selects are seen.
    60  	var selects []*ssa.Value
    61  	var calls []*ssa.Value
    62  	var args []*ssa.Value
    63  	var exitBlocks []*ssa.Block
    64  
    65  	var m0 *ssa.Value
    66  
    67  	// Accumulate lists of calls, args, selects, and exit blocks to process,
    68  	// note "wide" selects consumed by stores,
    69  	// rewrite mem for each call,
    70  	// rewrite each OpSelectNAddr.
    71  	for _, b := range f.Blocks {
    72  		for _, v := range b.Values {
    73  			switch v.Op {
    74  			case ssaop.OpInitMem:
    75  				m0 = v
    76  
    77  			case ssaop.OpClosureLECall, ssaop.OpInterLECall, ssaop.OpStaticLECall, ssaop.OpTailLECall, ssaop.OpTailLECallInter:
    78  				calls = append(calls, v)
    79  
    80  			case ssaop.OpArg:
    81  				args = append(args, v)
    82  
    83  			case ssaop.OpStore:
    84  				if a := v.Args[1]; a.Op == ssaop.OpSelectN && !ssa.CanSSA(a.Type) {
    85  					if a.Uses > 1 {
    86  						panic(fmt.Errorf("Saw double use of wide SelectN %s operand of Store %s",
    87  							a.LongString(), v.LongString()))
    88  					}
    89  					x.wideSelects[a] = v
    90  				}
    91  
    92  			case ssaop.OpSelectN:
    93  				if v.Type == types.TypeMem {
    94  					// rewrite the mem selector in place
    95  					call := v.Args[0]
    96  					aux := call.Aux.(*ssa.AuxCall)
    97  					mem := x.memForCall[call.ID]
    98  					if mem == nil {
    99  						v.AuxInt = int64(aux.AbiInfo.OutRegistersUsed())
   100  						x.memForCall[call.ID] = v
   101  					} else {
   102  						panic(fmt.Errorf("Saw two memories for call %v, %v and %v", call, mem, v))
   103  					}
   104  				} else {
   105  					selects = append(selects, v)
   106  				}
   107  
   108  			case ssaop.OpSelectNAddr:
   109  				call := v.Args[0]
   110  				which := v.AuxInt
   111  				aux := call.Aux.(*ssa.AuxCall)
   112  				pt := v.Type
   113  				off := x.offsetFrom(x.f.Entry, x.sp, aux.OffsetOfResult(which), pt)
   114  				v.CopyOf(off)
   115  			}
   116  		}
   117  
   118  		// rewrite function results from an exit block
   119  		// values returned by function need to be split out into registers.
   120  		if isBlockMultiValueExit(b) {
   121  			exitBlocks = append(exitBlocks, b)
   122  		}
   123  	}
   124  
   125  	// Convert each aggregate arg into Make of its parts (and so on, to primitive types)
   126  	for _, v := range args {
   127  		var rc registerCursor
   128  		a := x.prAssignForArg(v)
   129  		aux := x.f.OwnAux
   130  		regs := a.Registers
   131  		var offset int64
   132  		if len(regs) == 0 {
   133  			offset = a.FrameOffset(aux.AbiInfo)
   134  		}
   135  		auxBase := x.offsetFrom(x.f.Entry, x.sp, offset, types.NewPtr(v.Type))
   136  		rc.init(regs, aux.AbiInfo, nil, auxBase, 0)
   137  		x.rewriteSelectOrArg(f.Entry.Pos, f.Entry, v, v, m0, v.Type, rc)
   138  	}
   139  
   140  	// Rewrite selects of results (which may be aggregates) into make-aggregates of register/memory-targeted selects
   141  	for _, v := range selects {
   142  		if v.Op == ssaop.OpInvalid {
   143  			continue
   144  		}
   145  
   146  		call := v.Args[0]
   147  		aux := call.Aux.(*ssa.AuxCall)
   148  		mem := x.memForCall[call.ID]
   149  		if mem == nil {
   150  			mem = call.Block.NewValue1I(call.Pos, ssaop.OpSelectN, types.TypeMem, int64(aux.AbiInfo.OutRegistersUsed()), call)
   151  			x.memForCall[call.ID] = mem
   152  		}
   153  
   154  		i := v.AuxInt
   155  		regs := aux.RegsOfResult(i)
   156  
   157  		// If this select cannot fit into SSA and is stored, either disaggregate to register stores, or mem-mem move.
   158  		if store := x.wideSelects[v]; store != nil {
   159  			// Use the mem that comes from the store operation.
   160  			storeAddr := store.Args[0]
   161  			mem := store.Args[2]
   162  			if len(regs) > 0 {
   163  				// Cannot do a rewrite that builds up a result from pieces; instead, copy pieces to the store operation.
   164  				var rc registerCursor
   165  				rc.init(regs, aux.AbiInfo, nil, storeAddr, 0)
   166  				mem = x.rewriteWideSelectToStores(call.Pos, call.Block, v, mem, v.Type, rc)
   167  				store.CopyOf(mem)
   168  			} else {
   169  				// Move directly from AuxBase to store target; rewrite the store instruction.
   170  				offset := aux.OffsetOfResult(i)
   171  				auxBase := x.offsetFrom(x.f.Entry, x.sp, offset, types.NewPtr(v.Type))
   172  				// was Store dst, v, mem
   173  				// now Move dst, auxBase, mem
   174  				move := store.Block.NewValue3A(store.Pos, ssaop.OpMove, types.TypeMem, v.Type, storeAddr, auxBase, mem)
   175  				move.AuxInt = v.Type.Size()
   176  				store.CopyOf(move)
   177  			}
   178  			continue
   179  		}
   180  
   181  		var auxBase *ssa.Value
   182  		if len(regs) == 0 {
   183  			offset := aux.OffsetOfResult(i)
   184  			auxBase = x.offsetFrom(x.f.Entry, x.sp, offset, types.NewPtr(v.Type))
   185  		}
   186  		var rc registerCursor
   187  		rc.init(regs, aux.AbiInfo, nil, auxBase, 0)
   188  		x.rewriteSelectOrArg(call.Pos, call.Block, v, v, mem, v.Type, rc)
   189  	}
   190  
   191  	rewriteCall := func(v *ssa.Value, newOp ssaop.Op, argStart int) {
   192  		// Break aggregate args passed to call into smaller pieces.
   193  		x.rewriteCallArgs(v, argStart)
   194  		v.Op = newOp
   195  		rts := abi.RegisterTypes(v.Aux.(*ssa.AuxCall).AbiInfo.OutParams())
   196  		v.Type = types.NewResults(append(rts, types.TypeMem))
   197  	}
   198  
   199  	// Rewrite calls
   200  	for _, v := range calls {
   201  		switch v.Op {
   202  		case ssaop.OpStaticLECall:
   203  			rewriteCall(v, ssaop.OpStaticCall, 0)
   204  		case ssaop.OpTailLECall:
   205  			rewriteCall(v, ssaop.OpTailCall, 0)
   206  		case ssaop.OpTailLECallInter:
   207  			rewriteCall(v, ssaop.OpTailCallInter, 1)
   208  		case ssaop.OpClosureLECall:
   209  			rewriteCall(v, ssaop.OpClosureCall, 2)
   210  		case ssaop.OpInterLECall:
   211  			rewriteCall(v, ssaop.OpInterCall, 1)
   212  		}
   213  	}
   214  
   215  	// Rewrite results from exit blocks
   216  	for _, b := range exitBlocks {
   217  		v := b.Controls[0]
   218  		x.rewriteFuncResults(v, b, f.OwnAux)
   219  		b.SetControl(v)
   220  	}
   221  
   222  }
   223  
   224  func (x *expandState) rewriteFuncResults(v *ssa.Value, b *ssa.Block, aux *ssa.AuxCall) {
   225  	// This is very similar to rewriteCallArgs
   226  	// differences:
   227  	// firstArg + preArgs
   228  	// sp vs auxBase
   229  
   230  	m0 := v.MemoryArg()
   231  	mem := m0
   232  
   233  	allResults := []*ssa.Value{}
   234  	var oldArgs []*ssa.Value
   235  	argsWithoutMem := v.Args[:len(v.Args)-1]
   236  
   237  	for j, a := range argsWithoutMem {
   238  		oldArgs = append(oldArgs, a)
   239  		i := int64(j)
   240  		auxType := aux.TypeOfResult(i)
   241  		auxBase := b.NewValue2A(v.Pos, ssaop.OpLocalAddr, types.NewPtr(auxType), aux.NameOfResult(i), x.sp, mem)
   242  		auxOffset := int64(0)
   243  		aRegs := aux.RegsOfResult(int64(j))
   244  		if a.Op == ssaop.OpDereference {
   245  			a.Op = ssaop.OpLoad
   246  		}
   247  		var rc registerCursor
   248  		var result *[]*ssa.Value
   249  		if len(aRegs) > 0 {
   250  			result = &allResults
   251  		} else {
   252  			if a.Op == ssaop.OpLoad && a.Args[0].Op == ssaop.OpLocalAddr && a.Args[0].Aux == aux.NameOfResult(i) {
   253  				continue // Self move to output parameter
   254  			}
   255  		}
   256  		rc.init(aRegs, aux.AbiInfo, result, auxBase, auxOffset)
   257  		mem = x.decomposeAsNecessary(v.Pos, b, a, mem, rc)
   258  	}
   259  	v.ResetArgs()
   260  	v.AddArgs(allResults...)
   261  	v.AddArg(mem)
   262  	for _, a := range oldArgs {
   263  		if a.Uses == 0 {
   264  			if x.debug > 1 {
   265  				x.Printf("...marking %v unused\n", a.LongString())
   266  			}
   267  			x.invalidateRecursively(a)
   268  		}
   269  	}
   270  	v.Type = types.NewResults(append(abi.RegisterTypes(aux.AbiInfo.OutParams()), types.TypeMem))
   271  	return
   272  }
   273  
   274  func (x *expandState) rewriteCallArgs(v *ssa.Value, firstArg int) {
   275  	if x.debug > 1 {
   276  		x.indent(3)
   277  		defer x.indent(-3)
   278  		x.Printf("rewriteCallArgs(%s; %d)\n", v.LongString(), firstArg)
   279  	}
   280  	// Thread the stores on the memory arg
   281  	aux := v.Aux.(*ssa.AuxCall)
   282  	m0 := v.MemoryArg()
   283  	mem := m0
   284  	allResults := []*ssa.Value{}
   285  	oldArgs := []*ssa.Value{}
   286  	argsWithoutMem := v.Args[firstArg : len(v.Args)-1] // Also strip closure/interface Op-specific args
   287  
   288  	sp := x.sp
   289  	if v.Op == ssaop.OpTailLECall || v.Op == ssaop.OpTailLECallInter {
   290  		// For tail call, we unwind the frame before the call so we'll use the caller's
   291  		// SP.
   292  		sp = v.Block.NewValue1(src.NoXPos, ssaop.OpGetCallerSP, x.typs.Uintptr, mem)
   293  	}
   294  
   295  	for i, a := range argsWithoutMem { // skip leading non-parameter SSA Args and trailing mem SSA Arg.
   296  		oldArgs = append(oldArgs, a)
   297  		auxI := int64(i)
   298  		aRegs := aux.RegsOfArg(auxI)
   299  		aType := aux.TypeOfArg(auxI)
   300  
   301  		if a.Op == ssaop.OpDereference {
   302  			a.Op = ssaop.OpLoad
   303  		}
   304  		var rc registerCursor
   305  		var result *[]*ssa.Value
   306  		var aOffset int64
   307  		if len(aRegs) > 0 {
   308  			result = &allResults
   309  		} else {
   310  			aOffset = aux.OffsetOfArg(auxI)
   311  		}
   312  		if v.Op == ssaop.OpTailLECall && a.Op == ssaop.OpArg && a.AuxInt == 0 {
   313  			// It's common for a tail call passing the same arguments (e.g. method wrapper),
   314  			// so this would be a self copy. Detect this and optimize it out.
   315  			n := a.Aux.(*ir.Name)
   316  			if n.Class == ir.PPARAM && n.FrameOffset()+x.f.Config.Ctxt.Arch.FixedFrameSize == aOffset {
   317  				continue
   318  			}
   319  		}
   320  		if x.debug > 1 {
   321  			x.Printf("...storeArg %s, %v, %d\n", a.LongString(), aType, aOffset)
   322  		}
   323  
   324  		rc.init(aRegs, aux.AbiInfo, result, sp, aOffset)
   325  		mem = x.decomposeAsNecessary(v.Pos, v.Block, a, mem, rc)
   326  	}
   327  	var preArgStore [2]*ssa.Value
   328  	preArgs := append(preArgStore[:0], v.Args[0:firstArg]...)
   329  	v.ResetArgs()
   330  	v.AddArgs(preArgs...)
   331  	v.AddArgs(allResults...)
   332  	v.AddArg(mem)
   333  	for _, a := range oldArgs {
   334  		if a.Uses == 0 {
   335  			x.invalidateRecursively(a)
   336  		}
   337  	}
   338  
   339  	return
   340  }
   341  
   342  func (x *expandState) decomposePair(pos src.XPos, b *ssa.Block, a, mem *ssa.Value, t0, t1 *types.Type, o0, o1 ssaop.Op, rc *registerCursor) *ssa.Value {
   343  	e := b.NewValue1(pos, o0, t0, a)
   344  	pos = pos.WithNotStmt()
   345  	mem = x.decomposeAsNecessary(pos, b, e, mem, rc.next(t0))
   346  	e = b.NewValue1(pos, o1, t1, a)
   347  	mem = x.decomposeAsNecessary(pos, b, e, mem, rc.next(t1))
   348  	return mem
   349  }
   350  
   351  func (x *expandState) decomposeOne(pos src.XPos, b *ssa.Block, a, mem *ssa.Value, t0 *types.Type, o0 ssaop.Op, rc *registerCursor) *ssa.Value {
   352  	e := b.NewValue1(pos, o0, t0, a)
   353  	pos = pos.WithNotStmt()
   354  	mem = x.decomposeAsNecessary(pos, b, e, mem, rc.next(t0))
   355  	return mem
   356  }
   357  
   358  // decomposeAsNecessary converts a value (perhaps an aggregate) passed to a call or returned by a function,
   359  // into the appropriate sequence of stores and register assignments to transmit that value in a given ABI, and
   360  // returns the current memory after this convert/rewrite (it may be the input memory, perhaps stores were needed.)
   361  // 'pos' is the source position all this is tied to
   362  // 'b' is the enclosing block
   363  // 'a' is the value to decompose
   364  // 'm0' is the input memory arg used for the first store (or returned if there are no stores)
   365  // 'rc' is a registerCursor which identifies the register/memory destination for the value
   366  func (x *expandState) decomposeAsNecessary(pos src.XPos, b *ssa.Block, a, m0 *ssa.Value, rc registerCursor) *ssa.Value {
   367  	if x.debug > 1 {
   368  		x.indent(3)
   369  		defer x.indent(-3)
   370  	}
   371  	at := a.Type
   372  	if at.Size() == 0 {
   373  		return m0
   374  	}
   375  	if a.Op == ssaop.OpDereference {
   376  		a.Op = ssaop.OpLoad // For purposes of parameter passing expansion, a Dereference is a Load.
   377  	}
   378  
   379  	if !rc.hasRegs() && !ssa.CanSSA(at) {
   380  		dst := x.offsetFrom(b, rc.storeDest, rc.storeOffset, types.NewPtr(at))
   381  		if x.debug > 1 {
   382  			x.Printf("...recur store %s at %s\n", a.LongString(), dst.LongString())
   383  		}
   384  		if a.Op == ssaop.OpLoad {
   385  			m0 = b.NewValue3A(pos, ssaop.OpMove, types.TypeMem, at, dst, a.Args[0], m0)
   386  			m0.AuxInt = at.Size()
   387  			return m0
   388  		} else {
   389  			panic(fmt.Errorf("Store of not a load"))
   390  		}
   391  	}
   392  
   393  	mem := m0
   394  	switch at.Kind() {
   395  	case types.TARRAY:
   396  		et := at.Elem()
   397  		for i := int64(0); i < at.NumElem(); i++ {
   398  			e := b.NewValue1I(pos, ssaop.OpArraySelect, et, i, a)
   399  			pos = pos.WithNotStmt()
   400  			mem = x.decomposeAsNecessary(pos, b, e, mem, rc.next(et))
   401  		}
   402  		return mem
   403  
   404  	case types.TSTRUCT:
   405  		if at.IsSIMD() {
   406  			break // XXX
   407  		}
   408  		for i := 0; i < at.NumFields(); i++ {
   409  			et := at.Field(i).Type // might need to read offsets from the fields
   410  			e := b.NewValue1I(pos, ssaop.OpStructSelect, et, int64(i), a)
   411  			pos = pos.WithNotStmt()
   412  			if x.debug > 1 {
   413  				x.Printf("...recur decompose %s, %v\n", e.LongString(), et)
   414  			}
   415  			mem = x.decomposeAsNecessary(pos, b, e, mem, rc.next(et))
   416  		}
   417  		return mem
   418  
   419  	case types.TSLICE:
   420  		mem = x.decomposeOne(pos, b, a, mem, at.Elem().PtrTo(), ssaop.OpSlicePtr, &rc)
   421  		pos = pos.WithNotStmt()
   422  		mem = x.decomposeOne(pos, b, a, mem, x.typs.Int, ssaop.OpSliceLen, &rc)
   423  		return x.decomposeOne(pos, b, a, mem, x.typs.Int, ssaop.OpSliceCap, &rc)
   424  
   425  	case types.TSTRING:
   426  		return x.decomposePair(pos, b, a, mem, x.typs.BytePtr, x.typs.Int, ssaop.OpStringPtr, ssaop.OpStringLen, &rc)
   427  
   428  	case types.TINTER:
   429  		mem = x.decomposeOne(pos, b, a, mem, x.typs.Uintptr, ssaop.OpITab, &rc)
   430  		pos = pos.WithNotStmt()
   431  		// Immediate interfaces cause so many headaches.
   432  		if a.Op == ssaop.OpIMake {
   433  			data := a.Args[1]
   434  			for data.Op == ssaop.OpStructMake || data.Op == ssaop.OpArrayMake1 {
   435  				// A struct make might have a few zero-sized fields.
   436  				// Use the pointer-y one we know is there.
   437  				for _, a := range data.Args {
   438  					if a.Type.Size() > 0 {
   439  						data = a
   440  						break
   441  					}
   442  				}
   443  			}
   444  			return x.decomposeAsNecessary(pos, b, data, mem, rc.next(data.Type))
   445  		}
   446  		return x.decomposeOne(pos, b, a, mem, x.typs.BytePtr, ssaop.OpIData, &rc)
   447  
   448  	case types.TCOMPLEX64:
   449  		return x.decomposePair(pos, b, a, mem, x.typs.Float32, x.typs.Float32, ssaop.OpComplexReal, ssaop.OpComplexImag, &rc)
   450  
   451  	case types.TCOMPLEX128:
   452  		return x.decomposePair(pos, b, a, mem, x.typs.Float64, x.typs.Float64, ssaop.OpComplexReal, ssaop.OpComplexImag, &rc)
   453  
   454  	case types.TINT64:
   455  		if at.Size() > x.regSize {
   456  			return x.decomposePair(pos, b, a, mem, x.firstType, x.secondType, x.firstOp, x.secondOp, &rc)
   457  		}
   458  	case types.TUINT64:
   459  		if at.Size() > x.regSize {
   460  			return x.decomposePair(pos, b, a, mem, x.typs.UInt32, x.typs.UInt32, x.firstOp, x.secondOp, &rc)
   461  		}
   462  	}
   463  
   464  	// An atomic type, either record the register or store it and update the memory.
   465  
   466  	if rc.hasRegs() {
   467  		if x.debug > 1 {
   468  			x.Printf("...recur addArg %s\n", a.LongString())
   469  		}
   470  		rc.addArg(a)
   471  	} else {
   472  		dst := x.offsetFrom(b, rc.storeDest, rc.storeOffset, types.NewPtr(at))
   473  		if x.debug > 1 {
   474  			x.Printf("...recur store %s at %s\n", a.LongString(), dst.LongString())
   475  		}
   476  		mem = b.NewValue3A(pos, ssaop.OpStore, types.TypeMem, at, dst, a, mem)
   477  	}
   478  
   479  	return mem
   480  }
   481  
   482  // Convert scalar OpArg into the proper OpWhateverArg instruction
   483  // Convert scalar OpSelectN into perhaps-differently-indexed OpSelectN
   484  // Convert aggregate OpArg into Make of its parts (which are eventually scalars)
   485  // Convert aggregate OpSelectN into Make of its parts (which are eventually scalars)
   486  // Returns the converted value.
   487  //
   488  //   - "pos" the position for any generated instructions
   489  //   - "b" the block for any generated instructions
   490  //   - "container" the outermost OpArg/OpSelectN
   491  //   - "a" the instruction to overwrite, if any (only the outermost caller)
   492  //   - "m0" the memory arg for any loads that are necessary
   493  //   - "at" the type of the Arg/part
   494  //   - "rc" the register/memory cursor locating the various parts of the Arg.
   495  func (x *expandState) rewriteSelectOrArg(pos src.XPos, b *ssa.Block, container, a, m0 *ssa.Value, at *types.Type, rc registerCursor) *ssa.Value {
   496  
   497  	if at == types.TypeMem {
   498  		a.CopyOf(m0)
   499  		return a
   500  	}
   501  
   502  	makeOf := func(a *ssa.Value, op ssaop.Op, args []*ssa.Value) *ssa.Value {
   503  		if a == nil {
   504  			a = b.NewValue0(pos, op, at)
   505  			a.AddArgs(args...)
   506  		} else {
   507  			a.ResetArgs()
   508  			a.Aux, a.AuxInt = nil, 0
   509  			a.Pos, a.Op, a.Type = pos, op, at
   510  			a.AddArgs(args...)
   511  		}
   512  		return a
   513  	}
   514  
   515  	if at.Size() == 0 {
   516  		// For consistency, create these values even though they'll ultimately be unused
   517  		return makeOf(a, ssaop.OpEmpty, nil)
   518  	}
   519  
   520  	sk := selKey{from: container, size: 0, offsetOrIndex: rc.storeOffset, typ: at}
   521  	dupe := x.commonSelectors[sk]
   522  	if dupe != nil {
   523  		if a == nil {
   524  			return dupe
   525  		}
   526  		a.CopyOf(dupe)
   527  		return a
   528  	}
   529  
   530  	var argStore [10]*ssa.Value
   531  	args := argStore[:0]
   532  
   533  	addArg := func(a0 *ssa.Value) {
   534  		if a0 == nil {
   535  			as := "<nil>"
   536  			if a != nil {
   537  				as = a.LongString()
   538  			}
   539  			panic(fmt.Errorf("a0 should not be nil, a=%v, container=%v, at=%v", as, container.LongString(), at))
   540  		}
   541  		args = append(args, a0)
   542  	}
   543  
   544  	switch at.Kind() {
   545  	case types.TARRAY:
   546  		et := at.Elem()
   547  		for i := int64(0); i < at.NumElem(); i++ {
   548  			e := x.rewriteSelectOrArg(pos, b, container, nil, m0, et, rc.next(et))
   549  			addArg(e)
   550  		}
   551  		a = makeOf(a, ssaop.OpArrayMake1, args)
   552  		x.commonSelectors[sk] = a
   553  		return a
   554  
   555  	case types.TSTRUCT:
   556  		// Assume ssagen/ssa.go (in buildssa) spills large aggregates so they won't appear here.
   557  		if at.IsSIMD() {
   558  			break // XXX
   559  		}
   560  		for i := 0; i < at.NumFields(); i++ {
   561  			et := at.Field(i).Type
   562  			e := x.rewriteSelectOrArg(pos, b, container, nil, m0, et, rc.next(et))
   563  			if e == nil {
   564  				panic(fmt.Errorf("nil e, et=%v, et.Size()=%d, i=%d", et, et.Size(), i))
   565  			}
   566  			addArg(e)
   567  			pos = pos.WithNotStmt()
   568  		}
   569  		if at.NumFields() > ssa.MaxStruct && !types.IsDirectIface(at) {
   570  			panic(fmt.Errorf("Too many fields (%d, %d bytes), container=%s", at.NumFields(), at.Size(), container.LongString()))
   571  		}
   572  		a = makeOf(a, ssaop.OpStructMake, args)
   573  		x.commonSelectors[sk] = a
   574  		return a
   575  
   576  	case types.TSLICE:
   577  		addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, at.Elem().PtrTo(), rc.next(x.typs.BytePtr)))
   578  		pos = pos.WithNotStmt()
   579  		addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.typs.Int, rc.next(x.typs.Int)))
   580  		addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.typs.Int, rc.next(x.typs.Int)))
   581  		a = makeOf(a, ssaop.OpSliceMake, args)
   582  		x.commonSelectors[sk] = a
   583  		return a
   584  
   585  	case types.TSTRING:
   586  		addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.typs.BytePtr, rc.next(x.typs.BytePtr)))
   587  		pos = pos.WithNotStmt()
   588  		addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.typs.Int, rc.next(x.typs.Int)))
   589  		a = makeOf(a, ssaop.OpStringMake, args)
   590  		x.commonSelectors[sk] = a
   591  		return a
   592  
   593  	case types.TINTER:
   594  		addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.typs.Uintptr, rc.next(x.typs.Uintptr)))
   595  		pos = pos.WithNotStmt()
   596  		addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.typs.BytePtr, rc.next(x.typs.BytePtr)))
   597  		a = makeOf(a, ssaop.OpIMake, args)
   598  		x.commonSelectors[sk] = a
   599  		return a
   600  
   601  	case types.TCOMPLEX64:
   602  		addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.typs.Float32, rc.next(x.typs.Float32)))
   603  		pos = pos.WithNotStmt()
   604  		addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.typs.Float32, rc.next(x.typs.Float32)))
   605  		a = makeOf(a, ssaop.OpComplexMake, args)
   606  		x.commonSelectors[sk] = a
   607  		return a
   608  
   609  	case types.TCOMPLEX128:
   610  		addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.typs.Float64, rc.next(x.typs.Float64)))
   611  		pos = pos.WithNotStmt()
   612  		addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.typs.Float64, rc.next(x.typs.Float64)))
   613  		a = makeOf(a, ssaop.OpComplexMake, args)
   614  		x.commonSelectors[sk] = a
   615  		return a
   616  
   617  	case types.TINT64:
   618  		if at.Size() > x.regSize {
   619  			addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.firstType, rc.next(x.firstType)))
   620  			pos = pos.WithNotStmt()
   621  			addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.secondType, rc.next(x.secondType)))
   622  			if !x.f.Config.BigEndian {
   623  				// Int64Make args are big, little
   624  				args[0], args[1] = args[1], args[0]
   625  			}
   626  			a = makeOf(a, ssaop.OpInt64Make, args)
   627  			x.commonSelectors[sk] = a
   628  			return a
   629  		}
   630  	case types.TUINT64:
   631  		if at.Size() > x.regSize {
   632  			addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.typs.UInt32, rc.next(x.typs.UInt32)))
   633  			pos = pos.WithNotStmt()
   634  			addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.typs.UInt32, rc.next(x.typs.UInt32)))
   635  			if !x.f.Config.BigEndian {
   636  				// Int64Make args are big, little
   637  				args[0], args[1] = args[1], args[0]
   638  			}
   639  			a = makeOf(a, ssaop.OpInt64Make, args)
   640  			x.commonSelectors[sk] = a
   641  			return a
   642  		}
   643  	}
   644  
   645  	// An atomic type, either record the register or store it and update the memory.
   646  
   647  	// Depending on the container Op, the leaves are either OpSelectN or OpArg{Int,Float}Reg
   648  
   649  	if container.Op == ssaop.OpArg {
   650  		if rc.hasRegs() {
   651  			op, i := rc.ArgOpAndRegisterFor()
   652  			name := container.Aux.(*ir.Name)
   653  			a = makeOf(a, op, nil)
   654  			a.AuxInt = i
   655  			a.Aux = &ssa.AuxNameOffset{Name: name, Offset: rc.storeOffset}
   656  		} else {
   657  			key := selKey{container, rc.storeOffset, at.Size(), at}
   658  			w := x.commonArgs[key]
   659  			if w != nil && w.Uses != 0 {
   660  				if a == nil {
   661  					a = w
   662  				} else {
   663  					a.CopyOf(w)
   664  				}
   665  			} else {
   666  				if a == nil {
   667  					aux := container.Aux
   668  					auxInt := container.AuxInt + rc.storeOffset
   669  					a = container.Block.NewValue0IA(container.Pos, ssaop.OpArg, at, auxInt, aux)
   670  				} else {
   671  					// do nothing, the original should be okay.
   672  				}
   673  				x.commonArgs[key] = a
   674  			}
   675  		}
   676  	} else if container.Op == ssaop.OpSelectN {
   677  		call := container.Args[0]
   678  		aux := call.Aux.(*ssa.AuxCall)
   679  		which := container.AuxInt
   680  
   681  		if at == types.TypeMem {
   682  			if a != m0 || a != x.memForCall[call.ID] {
   683  				panic(fmt.Errorf("Memories %s, %s, and %s should all be equal after %s", a.LongString(), m0.LongString(), x.memForCall[call.ID], call.LongString()))
   684  			}
   685  		} else if rc.hasRegs() {
   686  			firstReg := uint32(0)
   687  			for i := 0; i < int(which); i++ {
   688  				firstReg += uint32(len(aux.AbiInfo.OutParam(i).Registers))
   689  			}
   690  			reg := int64(rc.nextSlice + Abi1RO(firstReg))
   691  			a = makeOf(a, ssaop.OpSelectN, []*ssa.Value{call})
   692  			a.AuxInt = reg
   693  		} else {
   694  			off := x.offsetFrom(x.f.Entry, x.sp, rc.storeOffset+aux.OffsetOfResult(which), types.NewPtr(at))
   695  			a = makeOf(a, ssaop.OpLoad, []*ssa.Value{off, m0})
   696  		}
   697  
   698  	} else {
   699  		panic(fmt.Errorf("Expected container OpArg or OpSelectN, saw %v instead", container.LongString()))
   700  	}
   701  
   702  	x.commonSelectors[sk] = a
   703  	return a
   704  }
   705  
   706  // rewriteWideSelectToStores handles the case of a SelectN'd result from a function call that is too large for SSA,
   707  // but is transferred in registers.  In this case the register cursor tracks both operands; the register sources and
   708  // the memory destinations.
   709  // This returns the memory flowing out of the last store
   710  func (x *expandState) rewriteWideSelectToStores(pos src.XPos, b *ssa.Block, container, m0 *ssa.Value, at *types.Type, rc registerCursor) *ssa.Value {
   711  
   712  	if at.Size() == 0 {
   713  		return m0
   714  	}
   715  
   716  	switch at.Kind() {
   717  	case types.TARRAY:
   718  		et := at.Elem()
   719  		for i := int64(0); i < at.NumElem(); i++ {
   720  			m0 = x.rewriteWideSelectToStores(pos, b, container, m0, et, rc.next(et))
   721  		}
   722  		return m0
   723  
   724  	case types.TSTRUCT:
   725  		// Assume ssagen/ssa.go (in buildssa) spills large aggregates so they won't appear here.
   726  		if at.IsSIMD() {
   727  			break // XXX
   728  		}
   729  		for i := 0; i < at.NumFields(); i++ {
   730  			et := at.Field(i).Type
   731  			m0 = x.rewriteWideSelectToStores(pos, b, container, m0, et, rc.next(et))
   732  			pos = pos.WithNotStmt()
   733  		}
   734  		return m0
   735  
   736  	case types.TSLICE:
   737  		m0 = x.rewriteWideSelectToStores(pos, b, container, m0, at.Elem().PtrTo(), rc.next(x.typs.BytePtr))
   738  		pos = pos.WithNotStmt()
   739  		m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.typs.Int, rc.next(x.typs.Int))
   740  		m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.typs.Int, rc.next(x.typs.Int))
   741  		return m0
   742  
   743  	case types.TSTRING:
   744  		m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.typs.BytePtr, rc.next(x.typs.BytePtr))
   745  		pos = pos.WithNotStmt()
   746  		m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.typs.Int, rc.next(x.typs.Int))
   747  		return m0
   748  
   749  	case types.TINTER:
   750  		m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.typs.Uintptr, rc.next(x.typs.Uintptr))
   751  		pos = pos.WithNotStmt()
   752  		m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.typs.BytePtr, rc.next(x.typs.BytePtr))
   753  		return m0
   754  
   755  	case types.TCOMPLEX64:
   756  		m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.typs.Float32, rc.next(x.typs.Float32))
   757  		pos = pos.WithNotStmt()
   758  		m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.typs.Float32, rc.next(x.typs.Float32))
   759  		return m0
   760  
   761  	case types.TCOMPLEX128:
   762  		m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.typs.Float64, rc.next(x.typs.Float64))
   763  		pos = pos.WithNotStmt()
   764  		m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.typs.Float64, rc.next(x.typs.Float64))
   765  		return m0
   766  
   767  	case types.TINT64:
   768  		if at.Size() > x.regSize {
   769  			m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.firstType, rc.next(x.firstType))
   770  			pos = pos.WithNotStmt()
   771  			m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.secondType, rc.next(x.secondType))
   772  			return m0
   773  		}
   774  	case types.TUINT64:
   775  		if at.Size() > x.regSize {
   776  			m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.typs.UInt32, rc.next(x.typs.UInt32))
   777  			pos = pos.WithNotStmt()
   778  			m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.typs.UInt32, rc.next(x.typs.UInt32))
   779  			return m0
   780  		}
   781  	}
   782  
   783  	// TODO could change treatment of too-large OpArg, would deal with it here.
   784  	if container.Op == ssaop.OpSelectN {
   785  		call := container.Args[0]
   786  		aux := call.Aux.(*ssa.AuxCall)
   787  		which := container.AuxInt
   788  
   789  		if rc.hasRegs() {
   790  			firstReg := uint32(0)
   791  			for i := 0; i < int(which); i++ {
   792  				firstReg += uint32(len(aux.AbiInfo.OutParam(i).Registers))
   793  			}
   794  			reg := int64(rc.nextSlice + Abi1RO(firstReg))
   795  			a := b.NewValue1I(pos, ssaop.OpSelectN, at, reg, call)
   796  			dst := x.offsetFrom(b, rc.storeDest, rc.storeOffset, types.NewPtr(at))
   797  			m0 = b.NewValue3A(pos, ssaop.OpStore, types.TypeMem, at, dst, a, m0)
   798  		} else {
   799  			panic(fmt.Errorf("Expected rc to have registers"))
   800  		}
   801  	} else {
   802  		panic(fmt.Errorf("Expected container OpSelectN, saw %v instead", container.LongString()))
   803  	}
   804  	return m0
   805  }
   806  
   807  func isBlockMultiValueExit(b *ssa.Block) bool {
   808  	return (b.Kind == block.BlockRet || b.Kind == block.BlockRetJmp) && b.Controls[0] != nil && b.Controls[0].Op == ssaop.OpMakeResult
   809  }
   810  
   811  type Abi1RO uint8 // An offset within a parameter's slice of register indices, for abi1.
   812  
   813  // A registerCursor tracks which register is used for an Arg or regValues, or a piece of such.
   814  type registerCursor struct {
   815  	storeDest   *ssa.Value // if there are no register targets, then this is the base of the store.
   816  	storeOffset int64
   817  	regs        []abi.RegIndex // the registers available for this Arg/result (which is all in registers or not at all)
   818  	nextSlice   Abi1RO         // the next register/register-slice offset
   819  	config      *abi.ABIConfig
   820  	regValues   *[]*ssa.Value // values assigned to registers accumulate here
   821  }
   822  
   823  func (c *registerCursor) String() string {
   824  	dest := "<none>"
   825  	if c.storeDest != nil {
   826  		dest = fmt.Sprintf("%s+%d", c.storeDest.String(), c.storeOffset)
   827  	}
   828  	regs := "<none>"
   829  	if c.regValues != nil {
   830  		regs = ""
   831  		for i, x := range *c.regValues {
   832  			if i > 0 {
   833  				regs = regs + "; "
   834  			}
   835  			regs = regs + x.LongString()
   836  		}
   837  	}
   838  
   839  	// not printing the config because that has not been useful
   840  	return fmt.Sprintf("RCSR{storeDest=%v, regsLen=%d, nextSlice=%d, regValues=[%s]}", dest, len(c.regs), c.nextSlice, regs)
   841  }
   842  
   843  // next effectively post-increments the register cursor; the receiver is advanced,
   844  // the (aligned) old value is returned.
   845  func (c *registerCursor) next(t *types.Type) registerCursor {
   846  	c.storeOffset = types.RoundUp(c.storeOffset, t.Alignment())
   847  	rc := *c
   848  	c.storeOffset = types.RoundUp(c.storeOffset+t.Size(), t.Alignment())
   849  	if int(c.nextSlice) < len(c.regs) {
   850  		w := c.config.NumParamRegs(t)
   851  		c.nextSlice += Abi1RO(w)
   852  	}
   853  	return rc
   854  }
   855  
   856  // plus returns a register cursor offset from the original, without modifying the original.
   857  func (c *registerCursor) plus(regWidth Abi1RO) registerCursor {
   858  	rc := *c
   859  	rc.nextSlice += regWidth
   860  	return rc
   861  }
   862  
   863  func (c *registerCursor) init(regs []abi.RegIndex, info *abi.ABIParamResultInfo, result *[]*ssa.Value, storeDest *ssa.Value, storeOffset int64) {
   864  	c.regs = regs
   865  	c.nextSlice = 0
   866  	c.storeOffset = storeOffset
   867  	c.storeDest = storeDest
   868  	c.config = info.Config()
   869  	c.regValues = result
   870  }
   871  
   872  func (c *registerCursor) addArg(v *ssa.Value) {
   873  	*c.regValues = append(*c.regValues, v)
   874  }
   875  
   876  func (c *registerCursor) hasRegs() bool {
   877  	return len(c.regs) > 0
   878  }
   879  
   880  func (c *registerCursor) ArgOpAndRegisterFor() (ssaop.Op, int64) {
   881  	r := c.regs[c.nextSlice]
   882  	return ssa.ArgOpAndRegisterFor(r, c.config)
   883  }
   884  
   885  type selKey struct {
   886  	from          *ssa.Value // what is selected from
   887  	offsetOrIndex int64      // whatever is appropriate for the selector
   888  	size          int64
   889  	typ           *types.Type
   890  }
   891  
   892  type expandState struct {
   893  	f       *ssa.Func
   894  	debug   int // odd values log lost statement markers, so likely settings are 1 (stmts), 2 (expansion), and 3 (both)
   895  	regSize int64
   896  	sp      *ssa.Value
   897  	typs    *ssa.Types
   898  
   899  	firstOp    ssaop.Op    // for 64-bit integers on 32-bit machines, first word in memory
   900  	secondOp   ssaop.Op    // for 64-bit integers on 32-bit machines, second word in memory
   901  	firstType  *types.Type // first half type, for Int64
   902  	secondType *types.Type // second half type, for Int64
   903  
   904  	wideSelects     map[*ssa.Value]*ssa.Value // Selects that are not SSA-able, mapped to consuming stores.
   905  	commonSelectors map[selKey]*ssa.Value     // used to de-dupe selectors
   906  	commonArgs      map[selKey]*ssa.Value     // used to de-dupe OpArg/OpArgIntReg/OpArgFloatReg
   907  	memForCall      map[ssa.ID]*ssa.Value     // For a call, need to know the unique selector that gets the mem.
   908  	indentLevel     int                       // Indentation for debugging recursion
   909  }
   910  
   911  // offsetFrom creates an offset from a pointer, simplifying chained offsets and offsets from SP
   912  func (x *expandState) offsetFrom(b *ssa.Block, from *ssa.Value, offset int64, pt *types.Type) *ssa.Value {
   913  	ft := from.Type
   914  	if offset == 0 {
   915  		if ft == pt {
   916  			return from
   917  		}
   918  		// This captures common, (apparently) safe cases.  The unsafe cases involve ft == uintptr
   919  		if (ft.IsPtr() || ft.IsUnsafePtr()) && pt.IsPtr() {
   920  			return from
   921  		}
   922  	}
   923  	// Simplify, canonicalize
   924  	for from.Op == ssaop.OpOffPtr {
   925  		offset += from.AuxInt
   926  		from = from.Args[0]
   927  	}
   928  	if from == x.sp {
   929  		return x.f.ConstOffPtrSP(pt, offset, x.sp)
   930  	}
   931  	return b.NewValue1I(from.Pos.WithNotStmt(), ssaop.OpOffPtr, pt, offset, from)
   932  }
   933  
   934  // prAssignForArg returns the ABIParamAssignment for v, assumed to be an OpArg.
   935  func (x *expandState) prAssignForArg(v *ssa.Value) *abi.ABIParamAssignment {
   936  	if v.Op != ssaop.OpArg {
   937  		panic(fmt.Errorf("Wanted OpArg, instead saw %s", v.LongString()))
   938  	}
   939  	return ssa.ParamAssignmentForArgName(x.f, v.Aux.(*ir.Name))
   940  }
   941  
   942  // indent increments (or decrements) the indentation.
   943  func (x *expandState) indent(n int) {
   944  	x.indentLevel += n
   945  }
   946  
   947  // Printf does an indented fmt.Printf on the format and args.
   948  func (x *expandState) Printf(format string, a ...any) (n int, err error) {
   949  	if x.indentLevel > 0 {
   950  		fmt.Printf("%[1]*s", x.indentLevel, "")
   951  	}
   952  	return fmt.Printf(format, a...)
   953  }
   954  
   955  func (x *expandState) invalidateRecursively(a *ssa.Value) {
   956  	var s string
   957  	if x.debug > 0 {
   958  		plus := " "
   959  		if a.Pos.IsStmt() == src.PosIsStmt {
   960  			plus = " +"
   961  		}
   962  		s = a.String() + plus + a.Pos.LineNumber() + " " + a.LongString()
   963  		if x.debug > 1 {
   964  			x.Printf("...marking %v unused\n", s)
   965  		}
   966  	}
   967  	lost := a.InvalidateRecursively()
   968  	if x.debug&1 != 0 && lost { // For odd values of x.debug, do this.
   969  		x.Printf("Lost statement marker in %s on former %s\n", base.Ctxt.Pkgpath+"."+x.f.Name, s)
   970  	}
   971  }
   972  

View as plain text