blob: d63c38c209d881ffcd0b549e0146523dba009c68 [file] [log] [blame]
// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package runtime
import (
"runtime/internal/atomic"
"unsafe"
)
const (
_EACCES = 13
_EINVAL = 22
)
// Don't split the stack as this method may be invoked without a valid G, which
// prevents us from allocating more stack.
//
//go:nosplit
func sysAllocOS(n uintptr) unsafe.Pointer {
p, err := mmap(nil, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_PRIVATE, -1, 0)
if err != 0 {
if err == _EACCES {
print("runtime: mmap: access denied\n")
exit(2)
}
if err == _EAGAIN {
print("runtime: mmap: too much locked memory (check 'ulimit -l').\n")
exit(2)
}
return nil
}
return p
}
var adviseUnused = uint32(_MADV_FREE)
const madviseUnsupported = 0
func sysUnusedOS(v unsafe.Pointer, n uintptr) {
if uintptr(v)&(physPageSize-1) != 0 || n&(physPageSize-1) != 0 {
// madvise will round this to any physical page
// *covered* by this range, so an unaligned madvise
// will release more memory than intended.
throw("unaligned sysUnused")
}
advise := atomic.Load(&adviseUnused)
if debug.madvdontneed != 0 && advise != madviseUnsupported {
advise = _MADV_DONTNEED
}
switch advise {
case _MADV_FREE:
if madvise(v, n, _MADV_FREE) == 0 {
break
}
atomic.Store(&adviseUnused, _MADV_DONTNEED)
fallthrough
case _MADV_DONTNEED:
// MADV_FREE was added in Linux 4.5. Fall back on MADV_DONTNEED if it's
// not supported.
if madvise(v, n, _MADV_DONTNEED) == 0 {
break
}
atomic.Store(&adviseUnused, madviseUnsupported)
fallthrough
case madviseUnsupported:
// Since Linux 3.18, support for madvise is optional.
// Fall back on mmap if it's not supported.
// _MAP_ANON|_MAP_FIXED|_MAP_PRIVATE will unmap all the
// pages in the old mapping, and remap the memory region.
mmap(v, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_FIXED|_MAP_PRIVATE, -1, 0)
}
if debug.harddecommit > 0 {
p, err := mmap(v, n, _PROT_NONE, _MAP_ANON|_MAP_FIXED|_MAP_PRIVATE, -1, 0)
if p != v || err != 0 {
throw("runtime: cannot disable permissions in address space")
}
}
}
func sysUsedOS(v unsafe.Pointer, n uintptr) {
if debug.harddecommit > 0 {
p, err := mmap(v, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_FIXED|_MAP_PRIVATE, -1, 0)
if err == _ENOMEM {
throw("runtime: out of memory")
}
if p != v || err != 0 {
throw("runtime: cannot remap pages in address space")
}
return
}
}
func sysHugePageOS(v unsafe.Pointer, n uintptr) {
if physHugePageSize != 0 {
// Round v up to a huge page boundary.
beg := alignUp(uintptr(v), physHugePageSize)
// Round v+n down to a huge page boundary.
end := alignDown(uintptr(v)+n, physHugePageSize)
if beg < end {
madvise(unsafe.Pointer(beg), end-beg, _MADV_HUGEPAGE)
}
}
}
func sysNoHugePageOS(v unsafe.Pointer, n uintptr) {
if uintptr(v)&(physPageSize-1) != 0 {
// The Linux implementation requires that the address
// addr be page-aligned, and allows length to be zero.
throw("unaligned sysNoHugePageOS")
}
madvise(v, n, _MADV_NOHUGEPAGE)
}
func sysHugePageCollapseOS(v unsafe.Pointer, n uintptr) {
if uintptr(v)&(physPageSize-1) != 0 {
// The Linux implementation requires that the address
// addr be page-aligned, and allows length to be zero.
throw("unaligned sysHugePageCollapseOS")
}
if physHugePageSize == 0 {
return
}
// N.B. If you find yourself debugging this code, note that
// this call can fail with EAGAIN because it's best-effort.
// Also, when it returns an error, it's only for the last
// huge page in the region requested.
//
// It can also sometimes return EINVAL if the corresponding
// region hasn't been backed by physical memory. This is
// difficult to guarantee in general, and it also means
// there's no way to distinguish whether this syscall is
// actually available. Oops.
//
// Anyway, that's why this call just doesn't bother checking
// any errors.
madvise(v, n, _MADV_COLLAPSE)
}
// Don't split the stack as this function may be invoked without a valid G,
// which prevents us from allocating more stack.
//
//go:nosplit
func sysFreeOS(v unsafe.Pointer, n uintptr) {
munmap(v, n)
}
func sysFaultOS(v unsafe.Pointer, n uintptr) {
mmap(v, n, _PROT_NONE, _MAP_ANON|_MAP_PRIVATE|_MAP_FIXED, -1, 0)
}
func sysReserveOS(v unsafe.Pointer, n uintptr) unsafe.Pointer {
p, err := mmap(v, n, _PROT_NONE, _MAP_ANON|_MAP_PRIVATE, -1, 0)
if err != 0 {
return nil
}
return p
}
func sysMapOS(v unsafe.Pointer, n uintptr) {
p, err := mmap(v, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_FIXED|_MAP_PRIVATE, -1, 0)
if err == _ENOMEM {
throw("runtime: out of memory")
}
if p != v || err != 0 {
print("runtime: mmap(", v, ", ", n, ") returned ", p, ", ", err, "\n")
throw("runtime: cannot map pages in arena address space")
}
// Disable huge pages if the GODEBUG for it is set.
//
// Note that there are a few sysHugePage calls that can override this, but
// they're all for GC metadata.
if debug.disablethp != 0 {
sysNoHugePageOS(v, n)
}
}