1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
|
// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package runtime
import (
"runtime/internal/atomic"
"unsafe"
)
const (
_EACCES = 13
_EINVAL = 22
)
// Don't split the stack as this method may be invoked without a valid G, which
// prevents us from allocating more stack.
//
//go:nosplit
func sysAllocOS(n uintptr) unsafe.Pointer {
p, err := mmap(nil, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_PRIVATE, -1, 0)
if err != 0 {
if err == _EACCES {
print("runtime: mmap: access denied\n")
exit(2)
}
if err == _EAGAIN {
print("runtime: mmap: too much locked memory (check 'ulimit -l').\n")
exit(2)
}
return nil
}
return p
}
var adviseUnused = uint32(_MADV_FREE)
const madviseUnsupported = 0
func sysUnusedOS(v unsafe.Pointer, n uintptr) {
if uintptr(v)&(physPageSize-1) != 0 || n&(physPageSize-1) != 0 {
// madvise will round this to any physical page
// *covered* by this range, so an unaligned madvise
// will release more memory than intended.
throw("unaligned sysUnused")
}
advise := atomic.Load(&adviseUnused)
if debug.madvdontneed != 0 && advise != madviseUnsupported {
advise = _MADV_DONTNEED
}
switch advise {
case _MADV_FREE:
if madvise(v, n, _MADV_FREE) == 0 {
break
}
atomic.Store(&adviseUnused, _MADV_DONTNEED)
fallthrough
case _MADV_DONTNEED:
// MADV_FREE was added in Linux 4.5. Fall back on MADV_DONTNEED if it's
// not supported.
if madvise(v, n, _MADV_DONTNEED) == 0 {
break
}
atomic.Store(&adviseUnused, madviseUnsupported)
fallthrough
case madviseUnsupported:
// Since Linux 3.18, support for madvise is optional.
// Fall back on mmap if it's not supported.
// _MAP_ANON|_MAP_FIXED|_MAP_PRIVATE will unmap all the
// pages in the old mapping, and remap the memory region.
mmap(v, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_FIXED|_MAP_PRIVATE, -1, 0)
}
if debug.harddecommit > 0 {
p, err := mmap(v, n, _PROT_NONE, _MAP_ANON|_MAP_FIXED|_MAP_PRIVATE, -1, 0)
if p != v || err != 0 {
throw("runtime: cannot disable permissions in address space")
}
}
}
func sysUsedOS(v unsafe.Pointer, n uintptr) {
if debug.harddecommit > 0 {
p, err := mmap(v, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_FIXED|_MAP_PRIVATE, -1, 0)
if err == _ENOMEM {
throw("runtime: out of memory")
}
if p != v || err != 0 {
throw("runtime: cannot remap pages in address space")
}
return
}
}
func sysHugePageOS(v unsafe.Pointer, n uintptr) {
if physHugePageSize != 0 {
// Round v up to a huge page boundary.
beg := alignUp(uintptr(v), physHugePageSize)
// Round v+n down to a huge page boundary.
end := alignDown(uintptr(v)+n, physHugePageSize)
if beg < end {
madvise(unsafe.Pointer(beg), end-beg, _MADV_HUGEPAGE)
}
}
}
func sysNoHugePageOS(v unsafe.Pointer, n uintptr) {
if uintptr(v)&(physPageSize-1) != 0 {
// The Linux implementation requires that the address
// addr be page-aligned, and allows length to be zero.
throw("unaligned sysNoHugePageOS")
}
madvise(v, n, _MADV_NOHUGEPAGE)
}
func sysHugePageCollapseOS(v unsafe.Pointer, n uintptr) {
if uintptr(v)&(physPageSize-1) != 0 {
// The Linux implementation requires that the address
// addr be page-aligned, and allows length to be zero.
throw("unaligned sysHugePageCollapseOS")
}
if physHugePageSize == 0 {
return
}
// N.B. If you find yourself debugging this code, note that
// this call can fail with EAGAIN because it's best-effort.
// Also, when it returns an error, it's only for the last
// huge page in the region requested.
//
// It can also sometimes return EINVAL if the corresponding
// region hasn't been backed by physical memory. This is
// difficult to guarantee in general, and it also means
// there's no way to distinguish whether this syscall is
// actually available. Oops.
//
// Anyway, that's why this call just doesn't bother checking
// any errors.
madvise(v, n, _MADV_COLLAPSE)
}
// Don't split the stack as this function may be invoked without a valid G,
// which prevents us from allocating more stack.
//
//go:nosplit
func sysFreeOS(v unsafe.Pointer, n uintptr) {
munmap(v, n)
}
func sysFaultOS(v unsafe.Pointer, n uintptr) {
mmap(v, n, _PROT_NONE, _MAP_ANON|_MAP_PRIVATE|_MAP_FIXED, -1, 0)
}
func sysReserveOS(v unsafe.Pointer, n uintptr) unsafe.Pointer {
p, err := mmap(v, n, _PROT_NONE, _MAP_ANON|_MAP_PRIVATE, -1, 0)
if err != 0 {
return nil
}
return p
}
func sysMapOS(v unsafe.Pointer, n uintptr) {
p, err := mmap(v, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_FIXED|_MAP_PRIVATE, -1, 0)
if err == _ENOMEM {
throw("runtime: out of memory")
}
if p != v || err != 0 {
print("runtime: mmap(", v, ", ", n, ") returned ", p, ", ", err, "\n")
throw("runtime: cannot map pages in arena address space")
}
// Disable huge pages if the GODEBUG for it is set.
//
// Note that there are a few sysHugePage calls that can override this, but
// they're all for GC metadata.
if debug.disablethp != 0 {
sysNoHugePageOS(v, n)
}
}
|
