mem.go

     1  // Copyright 2022 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package runtime
     6  
     7  import "unsafe"
     8  
     9  // OS memory management abstraction layer
    10  //
    11  // Regions of the address space managed by the runtime may be in one of four
    12  // states at any given time:
    13  // 1) None - Unreserved and unmapped, the default state of any region.
    14  // 2) Reserved - Owned by the runtime, but accessing it would cause a fault.
    15  //               Does not count against the process' memory footprint.
    16  // 3) Prepared - Reserved, intended not to be backed by physical memory (though
    17  //               an OS may implement this lazily). Can transition efficiently to
    18  //               Ready. Accessing memory in such a region is undefined (may
    19  //               fault, may give back unexpected zeroes, etc.).
    20  // 4) Ready - may be accessed safely.
    21  //
    22  // This set of states is more than strictly necessary to support all the
    23  // currently supported platforms. One could get by with just None, Reserved, and
    24  // Ready. However, the Prepared state gives us flexibility for performance
    25  // purposes. For example, on POSIX-y operating systems, Reserved is usually a
    26  // private anonymous mmap'd region with PROT_NONE set, and to transition
    27  // to Ready would require setting PROT_READ|PROT_WRITE. However the
    28  // underspecification of Prepared lets us use just MADV_FREE to transition from
    29  // Ready to Prepared. Thus with the Prepared state we can set the permission
    30  // bits just once early on, we can efficiently tell the OS that it's free to
    31  // take pages away from us when we don't strictly need them.
    32  //
    33  // This file defines a cross-OS interface for a common set of helpers
    34  // that transition memory regions between these states. The helpers call into
    35  // OS-specific implementations that handle errors, while the interface boundary
    36  // implements cross-OS functionality, like updating runtime accounting.
    37  
    38  // sysAlloc transitions an OS-chosen region of memory from None to Ready.
    39  // More specifically, it obtains a large chunk of zeroed memory from the
    40  // operating system, typically on the order of a hundred kilobytes
    41  // or a megabyte. This memory is always immediately available for use.
    42  //
    43  // sysStat must be non-nil.
    44  //
    45  // Don't split the stack as this function may be invoked without a valid G,
    46  // which prevents us from allocating more stack.
    47  //
    48  //go:nosplit
    49  func sysAlloc(n uintptr, sysStat *sysMemStat, vmaName string) unsafe.Pointer {
    50  	sysStat.add(int64(n))
    51  	gcController.mappedReady.Add(int64(n))
    52  	p := sysAllocOS(n, vmaName)
    53  
    54  	// When using ASAN leak detection, we must tell ASAN about
    55  	// cases where we store pointers in mmapped memory.
    56  	if asanenabled {
    57  		lsanregisterrootregion(p, n)
    58  	}
    59  
    60  	return p
    61  }
    62  
    63  // sysUnused transitions a memory region from Ready to Prepared. It notifies the
    64  // operating system that the physical pages backing this memory region are no
    65  // longer needed and can be reused for other purposes. The contents of a
    66  // sysUnused memory region are considered forfeit and the region must not be
    67  // accessed again until sysUsed is called.
    68  func sysUnused(v unsafe.Pointer, n uintptr) {
    69  	gcController.mappedReady.Add(-int64(n))
    70  	sysUnusedOS(v, n)
    71  }
    72  
    73  // needZeroAfterSysUnused reports whether memory returned by sysUnused must be
    74  // zeroed for use.
    75  func needZeroAfterSysUnused() bool {
    76  	return needZeroAfterSysUnusedOS()
    77  }
    78  
    79  // sysUsed transitions a memory region from Prepared to Ready. It notifies the
    80  // operating system that the memory region is needed and ensures that the region
    81  // may be safely accessed. This is typically a no-op on systems that don't have
    82  // an explicit commit step and hard over-commit limits, but is critical on
    83  // Windows, for example.
    84  //
    85  // This operation is idempotent for memory already in the Prepared state, so
    86  // it is safe to refer, with v and n, to a range of memory that includes both
    87  // Prepared and Ready memory. However, the caller must provide the exact amount
    88  // of Prepared memory for accounting purposes.
    89  func sysUsed(v unsafe.Pointer, n, prepared uintptr) {
    90  	gcController.mappedReady.Add(int64(prepared))
    91  	sysUsedOS(v, n)
    92  }
    93  
    94  // sysHugePage does not transition memory regions, but instead provides a
    95  // hint to the OS that it would be more efficient to back this memory region
    96  // with pages of a larger size transparently.
    97  func sysHugePage(v unsafe.Pointer, n uintptr) {
    98  	sysHugePageOS(v, n)
    99  }
   100  
   101  // sysNoHugePage does not transition memory regions, but instead provides a
   102  // hint to the OS that it would be less efficient to back this memory region
   103  // with pages of a larger size transparently.
   104  func sysNoHugePage(v unsafe.Pointer, n uintptr) {
   105  	sysNoHugePageOS(v, n)
   106  }
   107  
   108  // sysHugePageCollapse attempts to immediately back the provided memory region
   109  // with huge pages. It is best-effort and may fail silently.
   110  func sysHugePageCollapse(v unsafe.Pointer, n uintptr) {
   111  	sysHugePageCollapseOS(v, n)
   112  }
   113  
   114  // sysFree transitions a memory region from any state to None. Therefore, it
   115  // returns memory unconditionally. It is used if an out-of-memory error has been
   116  // detected midway through an allocation or to carve out an aligned section of
   117  // the address space. It is okay if sysFree is a no-op only if sysReserve always
   118  // returns a memory region aligned to the heap allocator's alignment
   119  // restrictions.
   120  //
   121  // sysStat must be non-nil.
   122  //
   123  // Don't split the stack as this function may be invoked without a valid G,
   124  // which prevents us from allocating more stack.
   125  //
   126  //go:nosplit
   127  func sysFree(v unsafe.Pointer, n uintptr, sysStat *sysMemStat) {
   128  	// When using ASAN leak detection, the memory being freed is
   129  	// known by the sanitizer. We need to unregister it so it's
   130  	// not accessed by it.
   131  	if asanenabled {
   132  		lsanunregisterrootregion(v, n)
   133  	}
   134  
   135  	sysStat.add(-int64(n))
   136  	gcController.mappedReady.Add(-int64(n))
   137  	sysFreeOS(v, n)
   138  }
   139  
   140  // sysFault transitions a memory region from Ready to Reserved. It
   141  // marks a region such that it will always fault if accessed. Used only for
   142  // debugging the runtime.
   143  //
   144  // TODO(mknyszek): Currently it's true that all uses of sysFault transition
   145  // memory from Ready to Reserved, but this may not be true in the future
   146  // since on every platform the operation is much more general than that.
   147  // If a transition from Prepared is ever introduced, create a new function
   148  // that elides the Ready state accounting.
   149  func sysFault(v unsafe.Pointer, n uintptr) {
   150  	gcController.mappedReady.Add(-int64(n))
   151  	sysFaultOS(v, n)
   152  }
   153  
   154  // sysReserve transitions a memory region from None to Reserved. It reserves
   155  // address space in such a way that it would cause a fatal fault upon access
   156  // (either via permissions or not committing the memory). Such a reservation is
   157  // thus never backed by physical memory.
   158  //
   159  // If the pointer passed to it is non-nil, the caller wants the
   160  // reservation there, but sysReserve can still choose another
   161  // location if that one is unavailable.
   162  //
   163  // NOTE: sysReserve returns OS-aligned memory, but the heap allocator
   164  // may use larger alignment, so the caller must be careful to realign the
   165  // memory obtained by sysReserve.
   166  func sysReserve(v unsafe.Pointer, n uintptr, vmaName string) unsafe.Pointer {
   167  	p := sysReserveOS(v, n, vmaName)
   168  
   169  	// When using ASAN leak detection, we must tell ASAN about
   170  	// cases where we store pointers in mmapped memory.
   171  	if asanenabled {
   172  		lsanregisterrootregion(p, n)
   173  	}
   174  
   175  	return p
   176  }
   177  
   178  // sysMap transitions a memory region from Reserved to Prepared. It ensures the
   179  // memory region can be efficiently transitioned to Ready.
   180  //
   181  // sysStat must be non-nil.
   182  func sysMap(v unsafe.Pointer, n uintptr, sysStat *sysMemStat, vmaName string) {
   183  	sysStat.add(int64(n))
   184  	sysMapOS(v, n, vmaName)
   185  }
   186
View as plain text