sha256block_amd64_shani.go

     1  // Copyright 2024 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package main
     6  
     7  import (
     8  	. "github.com/mmcloughlin/avo/build"
     9  	. "github.com/mmcloughlin/avo/operand"
    10  	. "github.com/mmcloughlin/avo/reg"
    11  )
    12  
    13  // The sha-ni implementation uses Intel(R) SHA extensions SHA256RNDS2, SHA256MSG1, SHA256MSG2
    14  // It also reuses portions of the flip_mask (half) and K256 table (stride 32) from the avx2 version
    15  //
    16  // Reference
    17  // S. Gulley, et al, "New Instructions Supporting the Secure Hash
    18  // Algorithm on Intel® Architecture Processors", July 2013
    19  // https://www.intel.com/content/www/us/en/developer/articles/technical/intel-sha-extensions.html
    20  
    21  func blockSHANI() {
    22  	Implement("blockSHANI")
    23  	Load(Param("dig"), digestPtr)    //                   init digest hash vector H0, H1,..., H7 pointer
    24  	Load(Param("p").Base(), dataPtr) //                   init input data base pointer
    25  	Load(Param("p").Len(), numBytes) //                   get number of input bytes to hash
    26  	SHRQ(Imm(6), numBytes)           //                   force modulo 64 input buffer length
    27  	SHLQ(Imm(6), numBytes)
    28  	CMPQ(numBytes, Imm(0)) //                             exit early for zero-length input buffer
    29  	JEQ(LabelRef("done"))
    30  	ADDQ(dataPtr, numBytes)                            // point numBytes to end of input buffer
    31  	VMOVDQU(Mem{Base: digestPtr}.Offset(0*16), state0) // load initial hash values and reorder
    32  	VMOVDQU(Mem{Base: digestPtr}.Offset(1*16), state1) // DCBA, HGFE -> ABEF, CDGH
    33  	PSHUFD(Imm(0xb1), state0, state0)                  // CDAB
    34  	PSHUFD(Imm(0x1b), state1, state1)                  // EFGH
    35  	VMOVDQA(state0, m4)
    36  	PALIGNR(Imm(8), state1, state0) //                    ABEF
    37  	PBLENDW(Imm(0xf0), m4, state1)  //                    CDGH
    38  	flip_mask := flip_mask_DATA()
    39  	VMOVDQA(flip_mask, shufMask)
    40  	LEAQ(K256_DATA(), sha256Constants)
    41  
    42  	roundLoop()
    43  	done()
    44  }
    45  
    46  func roundLoop() {
    47  	Label("roundLoop")
    48  	Comment("save hash values for addition after rounds")
    49  	VMOVDQA(state0, abefSave)
    50  	VMOVDQA(state1, cdghSave)
    51  
    52  	Comment("do rounds 0-59")
    53  	rounds0to11(m0, nil, 0, nop)       //                 0-3
    54  	rounds0to11(m1, m0, 1, sha256msg1) //                 4-7
    55  	rounds0to11(m2, m1, 2, sha256msg1) //                8-11
    56  	VMOVDQU(Mem{Base: dataPtr}.Offset(3*16), msg)
    57  	PSHUFB(shufMask, msg)
    58  	rounds12to59(m3, 3, m2, m0, sha256msg1, vmovrev) // 12-15
    59  	rounds12to59(m0, 4, m3, m1, sha256msg1, vmov)    // 16-19
    60  	rounds12to59(m1, 5, m0, m2, sha256msg1, vmov)    // 20-23
    61  	rounds12to59(m2, 6, m1, m3, sha256msg1, vmov)    // 24-27
    62  	rounds12to59(m3, 7, m2, m0, sha256msg1, vmov)    // 28-31
    63  	rounds12to59(m0, 8, m3, m1, sha256msg1, vmov)    // 32-35
    64  	rounds12to59(m1, 9, m0, m2, sha256msg1, vmov)    // 36-39
    65  	rounds12to59(m2, 10, m1, m3, sha256msg1, vmov)   // 40-43
    66  	rounds12to59(m3, 11, m2, m0, sha256msg1, vmov)   // 44-47
    67  	rounds12to59(m0, 12, m3, m1, sha256msg1, vmov)   // 48-51
    68  	rounds12to59(m1, 13, m0, m2, nop, vmov)          // 52-55
    69  	rounds12to59(m2, 14, m1, m3, nop, vmov)          // 56-59
    70  
    71  	Comment("do rounds 60-63")
    72  	VMOVDQA(m3, msg)
    73  	PADDD(Mem{Base: sha256Constants}.Offset(15*32), msg)
    74  	SHA256RNDS2(msg, state0, state1)
    75  	PSHUFD(Imm(0x0e), msg, msg)
    76  	SHA256RNDS2(msg, state1, state0)
    77  
    78  	Comment("add current hash values with previously saved")
    79  	PADDD(abefSave, state0)
    80  	PADDD(cdghSave, state1)
    81  
    82  	Comment("advance data pointer; loop until buffer empty")
    83  	ADDQ(Imm(64), dataPtr)
    84  	CMPQ(numBytes, dataPtr)
    85  	JNE(LabelRef("roundLoop"))
    86  
    87  	Comment("write hash values back in the correct order")
    88  	PSHUFD(Imm(0x1b), state0, state0)
    89  	PSHUFD(Imm(0xb1), state1, state1)
    90  	VMOVDQA(state0, m4)
    91  	PBLENDW(Imm(0xf0), state1, state0)
    92  	PALIGNR(Imm(8), m4, state1)
    93  	VMOVDQU(state0, Mem{Base: digestPtr}.Offset(0*16))
    94  	VMOVDQU(state1, Mem{Base: digestPtr}.Offset(1*16))
    95  }
    96  
    97  func done() {
    98  	Label("done")
    99  	RET()
   100  }
   101  
   102  var (
   103  	digestPtr       GPPhysical  = RDI // input/output, base pointer to digest hash vector H0, H1, ..., H7
   104  	dataPtr                     = RSI // input, base pointer to first input data block
   105  	numBytes                    = RDX // input, number of input bytes to be processed
   106  	sha256Constants             = RAX // round contents from K256 table, indexed by round number x 32
   107  	msg             VecPhysical = X0  // input data
   108  	state0                      = X1  // round intermediates and outputs
   109  	state1                      = X2
   110  	m0                          = X3 //  m0, m1,... m4 -- round message temps
   111  	m1                          = X4
   112  	m2                          = X5
   113  	m3                          = X6
   114  	m4                          = X7
   115  	shufMask                    = X8  // input data endian conversion control mask
   116  	abefSave                    = X9  // digest hash vector inter-block buffer abef
   117  	cdghSave                    = X10 // digest hash vector inter-block buffer cdgh
   118  )
   119  
   120  // nop instead of final SHA256MSG1 for first and last few rounds
   121  func nop(m, a VecPhysical) {
   122  }
   123  
   124  // final SHA256MSG1 for middle rounds that require it
   125  func sha256msg1(m, a VecPhysical) {
   126  	SHA256MSG1(m, a)
   127  }
   128  
   129  // msg copy for all but rounds 12-15
   130  func vmov(a, b VecPhysical) {
   131  	VMOVDQA(a, b)
   132  }
   133  
   134  // reverse copy for rounds 12-15
   135  func vmovrev(a, b VecPhysical) {
   136  	VMOVDQA(b, a)
   137  }
   138  
   139  type VecFunc func(a, b VecPhysical)
   140  
   141  // sha rounds 0 to 11
   142  //
   143  // identical with the exception of the final msg op
   144  // which is replaced with a nop for rounds where it is not needed
   145  // refer to Gulley, et al for more information
   146  func rounds0to11(m, a VecPhysical, c int, sha256msg1 VecFunc) {
   147  	VMOVDQU(Mem{Base: dataPtr}.Offset(c*16), msg)
   148  	PSHUFB(shufMask, msg)
   149  	VMOVDQA(msg, m)
   150  	PADDD(Mem{Base: sha256Constants}.Offset(c*32), msg)
   151  	SHA256RNDS2(msg, state0, state1)
   152  	PSHUFD(U8(0x0e), msg, msg)
   153  	SHA256RNDS2(msg, state1, state0)
   154  	sha256msg1(m, a)
   155  }
   156  
   157  // sha rounds 12 to 59
   158  //
   159  // identical with the exception of the final msg op
   160  // and the reverse copy(m,msg) in round 12 which is required
   161  // after the last data load
   162  // refer to Gulley, et al for more information
   163  func rounds12to59(m VecPhysical, c int, a, t VecPhysical, sha256msg1, movop VecFunc) {
   164  	movop(m, msg)
   165  	PADDD(Mem{Base: sha256Constants}.Offset(c*32), msg)
   166  	SHA256RNDS2(msg, state0, state1)
   167  	VMOVDQA(m, m4)
   168  	PALIGNR(Imm(4), a, m4)
   169  	PADDD(m4, t)
   170  	SHA256MSG2(m, t)
   171  	PSHUFD(Imm(0x0e), msg, msg)
   172  	SHA256RNDS2(msg, state1, state0)
   173  	sha256msg1(m, a)
   174  }
   175
View as plain text