gcm_asm.go

     1  // Copyright 2015 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  //go:build (amd64 || arm64) && !purego
     6  
     7  package gcm
     8  
     9  import (
    10  	"crypto/internal/fips140/aes"
    11  	"crypto/internal/fips140/subtle"
    12  	"crypto/internal/fips140deps/cpu"
    13  	"crypto/internal/impl"
    14  )
    15  
    16  // The following functions are defined in gcm_*.s.
    17  
    18  //go:noescape
    19  func gcmAesInit(productTable *[256]byte, ks []uint32)
    20  
    21  //go:noescape
    22  func gcmAesData(productTable *[256]byte, data []byte, T *[16]byte)
    23  
    24  //go:noescape
    25  func gcmAesEnc(productTable *[256]byte, dst, src []byte, ctr, T *[16]byte, ks []uint32)
    26  
    27  //go:noescape
    28  func gcmAesDec(productTable *[256]byte, dst, src []byte, ctr, T *[16]byte, ks []uint32)
    29  
    30  //go:noescape
    31  func gcmAesFinish(productTable *[256]byte, tagMask, T *[16]byte, pLen, dLen uint64)
    32  
    33  // Keep in sync with crypto/tls.hasAESGCMHardwareSupport.
    34  var supportsAESGCM = cpu.X86HasAES && cpu.X86HasPCLMULQDQ && cpu.X86HasSSE41 && cpu.X86HasSSSE3 ||
    35  	cpu.ARM64HasAES && cpu.ARM64HasPMULL
    36  
    37  func init() {
    38  	if cpu.AMD64 {
    39  		impl.Register("gcm", "AES-NI", &supportsAESGCM)
    40  	}
    41  	if cpu.ARM64 {
    42  		impl.Register("gcm", "Armv8.0", &supportsAESGCM)
    43  	}
    44  }
    45  
    46  // checkGenericIsExpected is called by the variable-time implementation to make
    47  // sure it is not used when hardware support is available. It shouldn't happen,
    48  // but this way it's more evidently correct.
    49  func checkGenericIsExpected() {
    50  	if supportsAESGCM {
    51  		panic("gcm: internal error: using generic implementation despite hardware support")
    52  	}
    53  }
    54  
    55  type gcmPlatformData struct {
    56  	productTable [256]byte
    57  }
    58  
    59  func initGCM(g *GCM) {
    60  	if !supportsAESGCM {
    61  		return
    62  	}
    63  	gcmAesInit(&g.productTable, aes.EncryptionKeySchedule(&g.cipher))
    64  }
    65  
    66  func seal(out []byte, g *GCM, nonce, plaintext, data []byte) {
    67  	if !supportsAESGCM {
    68  		sealGeneric(out, g, nonce, plaintext, data)
    69  		return
    70  	}
    71  
    72  	var counter, tagMask [gcmBlockSize]byte
    73  
    74  	if len(nonce) == gcmStandardNonceSize {
    75  		// Init counter to nonce||1
    76  		copy(counter[:], nonce)
    77  		counter[gcmBlockSize-1] = 1
    78  	} else {
    79  		// Otherwise counter = GHASH(nonce)
    80  		gcmAesData(&g.productTable, nonce, &counter)
    81  		gcmAesFinish(&g.productTable, &tagMask, &counter, uint64(len(nonce)), uint64(0))
    82  	}
    83  
    84  	aes.EncryptBlockInternal(&g.cipher, tagMask[:], counter[:])
    85  
    86  	var tagOut [gcmTagSize]byte
    87  	gcmAesData(&g.productTable, data, &tagOut)
    88  
    89  	if len(plaintext) > 0 {
    90  		gcmAesEnc(&g.productTable, out, plaintext, &counter, &tagOut, aes.EncryptionKeySchedule(&g.cipher))
    91  	}
    92  	gcmAesFinish(&g.productTable, &tagMask, &tagOut, uint64(len(plaintext)), uint64(len(data)))
    93  	copy(out[len(plaintext):], tagOut[:])
    94  }
    95  
    96  func open(out []byte, g *GCM, nonce, ciphertext, data []byte) error {
    97  	if !supportsAESGCM {
    98  		return openGeneric(out, g, nonce, ciphertext, data)
    99  	}
   100  
   101  	tag := ciphertext[len(ciphertext)-g.tagSize:]
   102  	ciphertext = ciphertext[:len(ciphertext)-g.tagSize]
   103  
   104  	// See GCM spec, section 7.1.
   105  	var counter, tagMask [gcmBlockSize]byte
   106  
   107  	if len(nonce) == gcmStandardNonceSize {
   108  		// Init counter to nonce||1
   109  		copy(counter[:], nonce)
   110  		counter[gcmBlockSize-1] = 1
   111  	} else {
   112  		// Otherwise counter = GHASH(nonce)
   113  		gcmAesData(&g.productTable, nonce, &counter)
   114  		gcmAesFinish(&g.productTable, &tagMask, &counter, uint64(len(nonce)), uint64(0))
   115  	}
   116  
   117  	aes.EncryptBlockInternal(&g.cipher, tagMask[:], counter[:])
   118  
   119  	var expectedTag [gcmTagSize]byte
   120  	gcmAesData(&g.productTable, data, &expectedTag)
   121  
   122  	if len(ciphertext) > 0 {
   123  		gcmAesDec(&g.productTable, out, ciphertext, &counter, &expectedTag, aes.EncryptionKeySchedule(&g.cipher))
   124  	}
   125  	gcmAesFinish(&g.productTable, &tagMask, &expectedTag, uint64(len(ciphertext)), uint64(len(data)))
   126  
   127  	if subtle.ConstantTimeCompare(expectedTag[:g.tagSize], tag) != 1 {
   128  		return errOpen
   129  	}
   130  	return nil
   131  }
   132
View as plain text