gcm_s390x.go

     1  // Copyright 2016 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  //go:build !purego
     6  
     7  package gcm
     8  
     9  import (
    10  	"crypto/internal/fips140/aes"
    11  	"crypto/internal/fips140/subtle"
    12  	"crypto/internal/fips140deps/byteorder"
    13  	"crypto/internal/fips140deps/cpu"
    14  	"crypto/internal/impl"
    15  )
    16  
    17  // This file contains two implementations of AES-GCM. The first implementation
    18  // (useGHASH) uses the KMCTR instruction to encrypt using AES in counter mode
    19  // and the KIMD instruction for GHASH. The second implementation (useGCM) uses
    20  // the newer KMA instruction which performs both operations (but still requires
    21  // KIMD to hash large nonces).
    22  
    23  // Keep in sync with crypto/tls.hasAESGCMHardwareSupport.
    24  var useGHASH = cpu.S390XHasAES && cpu.S390XHasAESCTR && cpu.S390XHasGHASH
    25  var useGCM = useGHASH && cpu.S390XHasAESGCM
    26  
    27  func init() {
    28  	impl.Register("gcm", "CPACF/KIMD", &useGHASH)
    29  	impl.Register("gcm", "CPACF/KMA", &useGCM)
    30  }
    31  
    32  func checkGenericIsExpected() {
    33  	if useGHASH || useGCM {
    34  		panic("gcm: internal error: using generic implementation despite hardware support")
    35  	}
    36  }
    37  
    38  // gcmLengths writes len0 || len1 as big-endian values to a 16-byte array.
    39  func gcmLengths(len0, len1 uint64) [16]byte {
    40  	v := [16]byte{}
    41  	byteorder.BEPutUint64(v[0:], len0)
    42  	byteorder.BEPutUint64(v[8:], len1)
    43  	return v
    44  }
    45  
    46  // gcmHashKey represents the 16-byte hash key required by the GHASH algorithm.
    47  type gcmHashKey [16]byte
    48  
    49  type gcmPlatformData struct {
    50  	hashKey gcmHashKey
    51  }
    52  
    53  func initGCM(g *GCM) {
    54  	if !useGCM && !useGHASH {
    55  		return
    56  	}
    57  	// Note that hashKey is also used in the KMA codepath to hash large nonces.
    58  	aes.EncryptBlockInternal(&g.cipher, g.hashKey[:], g.hashKey[:])
    59  }
    60  
    61  // ghashAsm uses the GHASH algorithm to hash data with the given key. The initial
    62  // hash value is given by hash which will be updated with the new hash value.
    63  // The length of data must be a multiple of 16-bytes.
    64  //
    65  //go:noescape
    66  func ghashAsm(key *gcmHashKey, hash *[16]byte, data []byte)
    67  
    68  // paddedGHASH pads data with zeroes until its length is a multiple of
    69  // 16-bytes. It then calculates a new value for hash using the GHASH algorithm.
    70  func paddedGHASH(hashKey *gcmHashKey, hash *[16]byte, data []byte) {
    71  	siz := len(data) &^ 0xf // align size to 16-bytes
    72  	if siz > 0 {
    73  		ghashAsm(hashKey, hash, data[:siz])
    74  		data = data[siz:]
    75  	}
    76  	if len(data) > 0 {
    77  		var s [16]byte
    78  		copy(s[:], data)
    79  		ghashAsm(hashKey, hash, s[:])
    80  	}
    81  }
    82  
    83  // cryptBlocksGCM encrypts src using AES in counter mode using the given
    84  // function code and key. The rightmost 32-bits of the counter are incremented
    85  // between each block as required by the GCM spec. The initial counter value
    86  // is given by cnt, which is updated with the value of the next counter value
    87  // to use.
    88  //
    89  // The lengths of both dst and buf must be greater than or equal to the length
    90  // of src. buf may be partially or completely overwritten during the execution
    91  // of the function.
    92  //
    93  //go:noescape
    94  func cryptBlocksGCM(fn int, key, dst, src, buf []byte, cnt *[gcmBlockSize]byte)
    95  
    96  // counterCrypt encrypts src using AES in counter mode and places the result
    97  // into dst. cnt is the initial count value and will be updated with the next
    98  // count value. The length of dst must be greater than or equal to the length
    99  // of src.
   100  func counterCrypt(g *GCM, dst, src []byte, cnt *[gcmBlockSize]byte) {
   101  	// Copying src into a buffer improves performance on some models when
   102  	// src and dst point to the same underlying array. We also need a
   103  	// buffer for counter values.
   104  	var ctrbuf, srcbuf [2048]byte
   105  	for len(src) >= 16 {
   106  		siz := len(src)
   107  		if len(src) > len(ctrbuf) {
   108  			siz = len(ctrbuf)
   109  		}
   110  		siz &^= 0xf // align siz to 16-bytes
   111  		copy(srcbuf[:], src[:siz])
   112  		cryptBlocksGCM(aes.BlockFunction(&g.cipher), aes.BlockKey(&g.cipher), dst[:siz], srcbuf[:siz], ctrbuf[:], cnt)
   113  		src = src[siz:]
   114  		dst = dst[siz:]
   115  	}
   116  	if len(src) > 0 {
   117  		var x [16]byte
   118  		aes.EncryptBlockInternal(&g.cipher, x[:], cnt[:])
   119  		for i := range src {
   120  			dst[i] = src[i] ^ x[i]
   121  		}
   122  		gcmInc32(cnt)
   123  	}
   124  }
   125  
   126  // deriveCounter computes the initial GCM counter state from the given nonce.
   127  // See NIST SP 800-38D, section 7.1 and deriveCounterGeneric in gcm_generic.go.
   128  func deriveCounter(H *gcmHashKey, counter *[gcmBlockSize]byte, nonce []byte) {
   129  	if len(nonce) == gcmStandardNonceSize {
   130  		copy(counter[:], nonce)
   131  		counter[gcmBlockSize-1] = 1
   132  	} else {
   133  		var hash [16]byte
   134  		paddedGHASH(H, &hash, nonce)
   135  		lens := gcmLengths(0, uint64(len(nonce))*8)
   136  		paddedGHASH(H, &hash, lens[:])
   137  		copy(counter[:], hash[:])
   138  	}
   139  }
   140  
   141  // gcmAuth calculates GHASH(additionalData, ciphertext), masks the result
   142  // with tagMask and writes the result to out.
   143  func gcmAuth(out []byte, H *gcmHashKey, tagMask *[gcmBlockSize]byte, ciphertext, additionalData []byte) {
   144  	var hash [16]byte
   145  	paddedGHASH(H, &hash, additionalData)
   146  	paddedGHASH(H, &hash, ciphertext)
   147  	lens := gcmLengths(uint64(len(additionalData))*8, uint64(len(ciphertext))*8)
   148  	paddedGHASH(H, &hash, lens[:])
   149  
   150  	copy(out, hash[:])
   151  	for i := range out {
   152  		out[i] ^= tagMask[i]
   153  	}
   154  }
   155  
   156  func seal(out []byte, g *GCM, nonce, plaintext, data []byte) {
   157  	switch {
   158  	case useGCM:
   159  		sealKMA(out, g, nonce, plaintext, data)
   160  	case useGHASH:
   161  		sealAsm(out, g, nonce, plaintext, data)
   162  	default:
   163  		sealGeneric(out, g, nonce, plaintext, data)
   164  	}
   165  }
   166  
   167  func sealAsm(out []byte, g *GCM, nonce, plaintext, additionalData []byte) {
   168  	var counter, tagMask [gcmBlockSize]byte
   169  	deriveCounter(&g.hashKey, &counter, nonce)
   170  	counterCrypt(g, tagMask[:], tagMask[:], &counter)
   171  
   172  	counterCrypt(g, out, plaintext, &counter)
   173  
   174  	var tag [gcmTagSize]byte
   175  	gcmAuth(tag[:], &g.hashKey, &tagMask, out[:len(plaintext)], additionalData)
   176  	copy(out[len(plaintext):], tag[:])
   177  }
   178  
   179  func open(out []byte, g *GCM, nonce, ciphertext, data []byte) error {
   180  	switch {
   181  	case useGCM:
   182  		return openKMA(out, g, nonce, ciphertext, data)
   183  	case useGHASH:
   184  		return openAsm(out, g, nonce, ciphertext, data)
   185  	default:
   186  		return openGeneric(out, g, nonce, ciphertext, data)
   187  	}
   188  }
   189  
   190  func openAsm(out []byte, g *GCM, nonce, ciphertext, additionalData []byte) error {
   191  	var counter, tagMask [gcmBlockSize]byte
   192  	deriveCounter(&g.hashKey, &counter, nonce)
   193  	counterCrypt(g, tagMask[:], tagMask[:], &counter)
   194  
   195  	tag := ciphertext[len(ciphertext)-g.tagSize:]
   196  	ciphertext = ciphertext[:len(ciphertext)-g.tagSize]
   197  
   198  	var expectedTag [gcmTagSize]byte
   199  	gcmAuth(expectedTag[:], &g.hashKey, &tagMask, ciphertext, additionalData)
   200  	if subtle.ConstantTimeCompare(expectedTag[:g.tagSize], tag) != 1 {
   201  		return errOpen
   202  	}
   203  
   204  	counterCrypt(g, out, ciphertext, &counter)
   205  
   206  	return nil
   207  }
   208  
   209  // flags for the KMA instruction
   210  const (
   211  	kmaHS      = 1 << 10 // hash subkey supplied
   212  	kmaLAAD    = 1 << 9  // last series of additional authenticated data
   213  	kmaLPC     = 1 << 8  // last series of plaintext or ciphertext blocks
   214  	kmaDecrypt = 1 << 7  // decrypt
   215  )
   216  
   217  // kmaGCM executes the encryption or decryption operation given by fn. The tag
   218  // will be calculated and written to tag. cnt should contain the current
   219  // counter state and will be overwritten with the updated counter state.
   220  // TODO(mundaym): could pass in hash subkey
   221  //
   222  //go:noescape
   223  func kmaGCM(fn int, key, dst, src, aad []byte, tag *[16]byte, cnt *[gcmBlockSize]byte)
   224  
   225  func sealKMA(out []byte, g *GCM, nonce, plaintext, data []byte) {
   226  	var counter [gcmBlockSize]byte
   227  	deriveCounter(&g.hashKey, &counter, nonce)
   228  	fc := aes.BlockFunction(&g.cipher) | kmaLAAD | kmaLPC
   229  
   230  	var tag [gcmTagSize]byte
   231  	kmaGCM(fc, aes.BlockKey(&g.cipher), out[:len(plaintext)], plaintext, data, &tag, &counter)
   232  	copy(out[len(plaintext):], tag[:])
   233  }
   234  
   235  func openKMA(out []byte, g *GCM, nonce, ciphertext, data []byte) error {
   236  	tag := ciphertext[len(ciphertext)-g.tagSize:]
   237  	ciphertext = ciphertext[:len(ciphertext)-g.tagSize]
   238  
   239  	var counter [gcmBlockSize]byte
   240  	deriveCounter(&g.hashKey, &counter, nonce)
   241  	fc := aes.BlockFunction(&g.cipher) | kmaLAAD | kmaLPC | kmaDecrypt
   242  
   243  	var expectedTag [gcmTagSize]byte
   244  	kmaGCM(fc, aes.BlockKey(&g.cipher), out[:len(ciphertext)], ciphertext, data, &expectedTag, &counter)
   245  
   246  	if subtle.ConstantTimeCompare(expectedTag[:g.tagSize], tag) != 1 {
   247  		return errOpen
   248  	}
   249  
   250  	return nil
   251  }
   252
View as plain text