pair.go

     1  // Copyright 2026 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package arm64
     6  
     7  import (
     8  	"cmd/compile/internal/base"
     9  	"cmd/compile/internal/objw"
    10  	"cmd/internal/obj"
    11  	"cmd/internal/obj/arm64"
    12  	"cmd/internal/src"
    13  )
    14  
    15  // movdImmMemInfo describes a plain "MOVD reg, off(base)" or "MOVD off(base), reg"
    16  // instruction. base, off, name, and sym record the memory operand's base
    17  // register, offset, addressing class (NAME_AUTO or NAME_PARAM), and symbol;
    18  // reg is the register stored or loaded, and isLoad distinguishes the two
    19  // forms.
    20  type movdImmMemInfo struct {
    21  	base, reg int16
    22  	off       int64
    23  	name      obj.AddrName
    24  	sym       *obj.LSym
    25  	isLoad    bool
    26  }
    27  
    28  // movdImmMem decodes p into a movdImmMemInfo. ok reports whether p is a
    29  // plain MOVD load or store using an immediate-offset addressing form
    30  // suitable for LDP/STP coalescing.
    31  func movdImmMem(p *obj.Prog) (info movdImmMemInfo, ok bool) {
    32  	if p.As != arm64.AMOVD || p.Scond != 0 || p.Reg != 0 {
    33  		return movdImmMemInfo{}, false
    34  	}
    35  	var a *obj.Addr
    36  	switch {
    37  	case p.From.Type == obj.TYPE_MEM && p.To.Type == obj.TYPE_REG:
    38  		// Plain load: MOVD mem, reg.
    39  		a = &p.From
    40  		info.reg = p.To.Reg
    41  		info.isLoad = true
    42  	case p.From.Type == obj.TYPE_REG && p.To.Type == obj.TYPE_MEM:
    43  		// Plain store: MOVD reg, mem.
    44  		a = &p.To
    45  		info.reg = p.From.Reg
    46  	default:
    47  		return movdImmMemInfo{}, false
    48  	}
    49  	if a.Index != 0 || (a.Name != obj.NAME_AUTO && a.Name != obj.NAME_PARAM) {
    50  		return movdImmMemInfo{}, false
    51  	}
    52  	info.base, info.off, info.name, info.sym = a.Reg, a.Offset, a.Name, a.Sym
    53  	return info, true
    54  }
    55  
    56  // ssaGenFinish runs after genssa has emitted all of a function's Progs,
    57  // resolved its branch and jump-table targets, and finalized the frame size
    58  // in defframe.
    59  func ssaGenFinish(pp *objw.Progs) {
    60  	if base.Flag.N != 0 {
    61  		// Keep unoptimized builds unoptimized.
    62  		return
    63  	}
    64  	pairSpills(pp.Text, pp.Text.To.Offset, pp.CurFunc.LSym.Func().JumpTables)
    65  }
    66  
    67  // pairSpills fuses strictly-adjacent MOVD load or store pairs that target the
    68  // same base register at consecutive 8-byte offsets into a single LDP or STP.
    69  // text is the function's first Prog, framesize its final frame size, and
    70  // jumpTables its resolved jump tables.
    71  //
    72  // This recovers spill/reload pairings that the SSA-level pair pass
    73  // (cmd/compile/internal/ssa/pair.go) cannot perform: that pass runs before
    74  // regalloc and only sees source-level loads/stores, so the STR/LDR pairs that
    75  // regalloc later inserts for OpStoreReg/OpLoadReg never get a chance to be
    76  // paired. Doing the fusion here, as the last step of code generation, keeps the
    77  // optimization in the compiler so the assembler can stay a simple translator.
    78  func pairSpills(text *obj.Prog, framesize int64, jumpTables []obj.JumpTable) {
    79  	// Branch and jump-table targets, collected lazily once a candidate pair
    80  	// survives the cheaper checks; most functions have no candidates.
    81  	// Fusing into the first of a pair whose second instruction is a target
    82  	// would silently change control flow: paths that jump directly to the
    83  	// second instruction would skip the work the LDP/STP now does at the
    84  	// first instruction's position.
    85  	var isTarget map[*obj.Prog]bool
    86  	targets := func() map[*obj.Prog]bool {
    87  		if isTarget == nil {
    88  			isTarget = map[*obj.Prog]bool{}
    89  			for p := text; p != nil; p = p.Link {
    90  				if t := p.To.Target(); t != nil {
    91  					isTarget[t] = true
    92  				}
    93  			}
    94  			for _, jt := range jumpTables {
    95  				for _, t := range jt.Targets {
    96  					isTarget[t] = true
    97  				}
    98  			}
    99  		}
   100  		return isTarget
   101  	}
   102  	for p := text; p != nil; p = p.Link {
   103  		q := p.Link
   104  		if q == nil {
   105  			continue
   106  		}
   107  		a, ok := movdImmMem(p)
   108  		if !ok {
   109  			continue
   110  		}
   111  		b, ok := movdImmMem(q)
   112  		if !ok || a.isLoad != b.isLoad {
   113  			continue
   114  		}
   115  		if a.base != b.base || a.name != b.name {
   116  			continue
   117  		}
   118  		if a.reg == b.reg {
   119  			// LDP with Rt1 == Rt2 is CONSTRAINED UNPREDICTABLE. (STP with
   120  			// identical source registers would be fine, but spills never
   121  			// store the same register twice, so don't bother.)
   122  			continue
   123  		}
   124  		if a.isLoad && a.reg == a.base {
   125  			// The first load overwrites the base register: executed
   126  			// sequentially, the second load computes its address from the
   127  			// just-loaded value, while LDP computes both addresses from
   128  			// the original base. (The second load overwriting the base is
   129  			// fine; no address depends on it.)
   130  			continue
   131  		}
   132  		if q.Pos.IsStmt() == src.PosIsStmt {
   133  			// q carries a statement boundary, and may also be registered
   134  			// as an inline mark: genssa promotes instructions it reuses
   135  			// as inline marks to statements. Reducing q to a 0-byte ANOP
   136  			// would drop the statement from the line tables and violate
   137  			// the invariant that inline marks are never zero-sized (they
   138  			// are identified by PC).
   139  			continue
   140  		}
   141  		lo, hi := a, b
   142  		if lo.off > hi.off {
   143  			lo, hi = hi, lo
   144  		}
   145  		if hi.off != lo.off+8 {
   146  			continue
   147  		}
   148  		// Fuse only when the result will encode as a single LDP/STP, whose
   149  		// signed 7-bit immediate scaled by 8 covers offsets [-512, 504].
   150  		// The assembler resolves a NAME_AUTO offset to off+framesize+8
   151  		// (the 8 is the saved LR) and a NAME_PARAM offset to
   152  		// off+framesize+24 or +32 depending on frame alignment padding —
   153  		// or off+8 in a frameless leaf, which is not decided until
   154  		// assembly, so a frameless PARAM must fit both. An out-of-range
   155  		// LDP/STP needs an assembler-synthesized address (ADD + LDP),
   156  		// which is no smaller than the original pair and serializes the
   157  		// accesses through REGTMP.
   158  		var biasLo, biasHi int64
   159  		switch {
   160  		case lo.name == obj.NAME_AUTO:
   161  			// Autos exist only in functions with a frame, and their offsets
   162  			// are negative from FP, so framesize+8 rebases them onto the
   163  			// non-negative SP-relative range the LDP/STP immediate must reach.
   164  			biasLo = framesize + 8
   165  			biasHi = biasLo
   166  		case framesize == 0:
   167  			biasLo, biasHi = 8, 24
   168  		case framesize%16 == 0:
   169  			biasLo = framesize + 24
   170  			biasHi = biasLo
   171  		default:
   172  			biasLo = framesize + 32
   173  			biasHi = biasLo
   174  		}
   175  		if lo.off%8 != 0 || lo.off+biasLo < -512 || lo.off+biasHi > 504 {
   176  			continue
   177  		}
   178  		if targets()[q] {
   179  			continue
   180  		}
   181  		mem := obj.Addr{Type: obj.TYPE_MEM, Reg: lo.base, Offset: lo.off, Name: lo.name, Sym: lo.sym}
   182  		regs := obj.Addr{Type: obj.TYPE_REGREG, Reg: lo.reg, Offset: int64(hi.reg)}
   183  		if a.isLoad {
   184  			p.As = arm64.ALDP
   185  			p.From, p.To = mem, regs
   186  		} else {
   187  			p.As = arm64.ASTP
   188  			p.From, p.To = regs, mem
   189  		}
   190  		// Reduce q to a 0-byte ANOP rather than unlinking it so that
   191  		// branch targets referencing q remain valid.
   192  		obj.Nopout(q)
   193  	}
   194  }
   195
View as plain text