aboutsummaryrefslogtreecommitdiffstats
path: root/contrib/go/_std_1.22/src/runtime/cgocheck.go
blob: 3d6de4f855dc335ea699dc8dcf79ee157c78fcbd (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Code to check that pointer writes follow the cgo rules.
// These functions are invoked when GOEXPERIMENT=cgocheck2 is enabled.

package runtime

import (
	"internal/goarch"
	"internal/goexperiment"
	"unsafe"
)

const cgoWriteBarrierFail = "unpinned Go pointer stored into non-Go memory"

// cgoCheckPtrWrite is called whenever a pointer is stored into memory.
// It throws if the program is storing an unpinned Go pointer into non-Go
// memory.
//
// This is called from generated code when GOEXPERIMENT=cgocheck2 is enabled.
//
//go:nosplit
//go:nowritebarrier
func cgoCheckPtrWrite(dst *unsafe.Pointer, src unsafe.Pointer) {
	if !mainStarted {
		// Something early in startup hates this function.
		// Don't start doing any actual checking until the
		// runtime has set itself up.
		return
	}
	if !cgoIsGoPointer(src) {
		return
	}
	if cgoIsGoPointer(unsafe.Pointer(dst)) {
		return
	}

	// If we are running on the system stack then dst might be an
	// address on the stack, which is OK.
	gp := getg()
	if gp == gp.m.g0 || gp == gp.m.gsignal {
		return
	}

	// Allocating memory can write to various mfixalloc structs
	// that look like they are non-Go memory.
	if gp.m.mallocing != 0 {
		return
	}

	// If the object is pinned, it's safe to store it in C memory. The GC
	// ensures it will not be moved or freed.
	if isPinned(src) {
		return
	}

	// It's OK if writing to memory allocated by persistentalloc.
	// Do this check last because it is more expensive and rarely true.
	// If it is false the expense doesn't matter since we are crashing.
	if inPersistentAlloc(uintptr(unsafe.Pointer(dst))) {
		return
	}

	systemstack(func() {
		println("write of unpinned Go pointer", hex(uintptr(src)), "to non-Go memory", hex(uintptr(unsafe.Pointer(dst))))
		throw(cgoWriteBarrierFail)
	})
}

// cgoCheckMemmove is called when moving a block of memory.
// It throws if the program is copying a block that contains an unpinned Go
// pointer into non-Go memory.
//
// This is called from generated code when GOEXPERIMENT=cgocheck2 is enabled.
//
//go:nosplit
//go:nowritebarrier
func cgoCheckMemmove(typ *_type, dst, src unsafe.Pointer) {
	cgoCheckMemmove2(typ, dst, src, 0, typ.Size_)
}

// cgoCheckMemmove2 is called when moving a block of memory.
// dst and src point off bytes into the value to copy.
// size is the number of bytes to copy.
// It throws if the program is copying a block that contains an unpinned Go
// pointer into non-Go memory.
//
//go:nosplit
//go:nowritebarrier
func cgoCheckMemmove2(typ *_type, dst, src unsafe.Pointer, off, size uintptr) {
	if typ.PtrBytes == 0 {
		return
	}
	if !cgoIsGoPointer(src) {
		return
	}
	if cgoIsGoPointer(dst) {
		return
	}
	cgoCheckTypedBlock(typ, src, off, size)
}

// cgoCheckSliceCopy is called when copying n elements of a slice.
// src and dst are pointers to the first element of the slice.
// typ is the element type of the slice.
// It throws if the program is copying slice elements that contain unpinned Go
// pointers into non-Go memory.
//
//go:nosplit
//go:nowritebarrier
func cgoCheckSliceCopy(typ *_type, dst, src unsafe.Pointer, n int) {
	if typ.PtrBytes == 0 {
		return
	}
	if !cgoIsGoPointer(src) {
		return
	}
	if cgoIsGoPointer(dst) {
		return
	}
	p := src
	for i := 0; i < n; i++ {
		cgoCheckTypedBlock(typ, p, 0, typ.Size_)
		p = add(p, typ.Size_)
	}
}

// cgoCheckTypedBlock checks the block of memory at src, for up to size bytes,
// and throws if it finds an unpinned Go pointer. The type of the memory is typ,
// and src is off bytes into that type.
//
//go:nosplit
//go:nowritebarrier
func cgoCheckTypedBlock(typ *_type, src unsafe.Pointer, off, size uintptr) {
	// Anything past typ.PtrBytes is not a pointer.
	if typ.PtrBytes <= off {
		return
	}
	if ptrdataSize := typ.PtrBytes - off; size > ptrdataSize {
		size = ptrdataSize
	}

	if typ.Kind_&kindGCProg == 0 {
		cgoCheckBits(src, typ.GCData, off, size)
		return
	}

	// The type has a GC program. Try to find GC bits somewhere else.
	for _, datap := range activeModules() {
		if cgoInRange(src, datap.data, datap.edata) {
			doff := uintptr(src) - datap.data
			cgoCheckBits(add(src, -doff), datap.gcdatamask.bytedata, off+doff, size)
			return
		}
		if cgoInRange(src, datap.bss, datap.ebss) {
			boff := uintptr(src) - datap.bss
			cgoCheckBits(add(src, -boff), datap.gcbssmask.bytedata, off+boff, size)
			return
		}
	}

	s := spanOfUnchecked(uintptr(src))
	if s.state.get() == mSpanManual {
		// There are no heap bits for value stored on the stack.
		// For a channel receive src might be on the stack of some
		// other goroutine, so we can't unwind the stack even if
		// we wanted to.
		// We can't expand the GC program without extra storage
		// space we can't easily get.
		// Fortunately we have the type information.
		systemstack(func() {
			cgoCheckUsingType(typ, src, off, size)
		})
		return
	}

	// src must be in the regular heap.
	if goexperiment.AllocHeaders {
		tp := s.typePointersOf(uintptr(src), size)
		for {
			var addr uintptr
			if tp, addr = tp.next(uintptr(src) + size); addr == 0 {
				break
			}
			v := *(*unsafe.Pointer)(unsafe.Pointer(addr))
			if cgoIsGoPointer(v) && !isPinned(v) {
				throw(cgoWriteBarrierFail)
			}
		}
	} else {
		hbits := heapBitsForAddr(uintptr(src), size)
		for {
			var addr uintptr
			if hbits, addr = hbits.next(); addr == 0 {
				break
			}
			v := *(*unsafe.Pointer)(unsafe.Pointer(addr))
			if cgoIsGoPointer(v) && !isPinned(v) {
				throw(cgoWriteBarrierFail)
			}
		}
	}
}

// cgoCheckBits checks the block of memory at src, for up to size
// bytes, and throws if it finds an unpinned Go pointer. The gcbits mark each
// pointer value. The src pointer is off bytes into the gcbits.
//
//go:nosplit
//go:nowritebarrier
func cgoCheckBits(src unsafe.Pointer, gcbits *byte, off, size uintptr) {
	skipMask := off / goarch.PtrSize / 8
	skipBytes := skipMask * goarch.PtrSize * 8
	ptrmask := addb(gcbits, skipMask)
	src = add(src, skipBytes)
	off -= skipBytes
	size += off
	var bits uint32
	for i := uintptr(0); i < size; i += goarch.PtrSize {
		if i&(goarch.PtrSize*8-1) == 0 {
			bits = uint32(*ptrmask)
			ptrmask = addb(ptrmask, 1)
		} else {
			bits >>= 1
		}
		if off > 0 {
			off -= goarch.PtrSize
		} else {
			if bits&1 != 0 {
				v := *(*unsafe.Pointer)(add(src, i))
				if cgoIsGoPointer(v) && !isPinned(v) {
					throw(cgoWriteBarrierFail)
				}
			}
		}
	}
}

// cgoCheckUsingType is like cgoCheckTypedBlock, but is a last ditch
// fall back to look for pointers in src using the type information.
// We only use this when looking at a value on the stack when the type
// uses a GC program, because otherwise it's more efficient to use the
// GC bits. This is called on the system stack.
//
//go:nowritebarrier
//go:systemstack
func cgoCheckUsingType(typ *_type, src unsafe.Pointer, off, size uintptr) {
	if typ.PtrBytes == 0 {
		return
	}

	// Anything past typ.PtrBytes is not a pointer.
	if typ.PtrBytes <= off {
		return
	}
	if ptrdataSize := typ.PtrBytes - off; size > ptrdataSize {
		size = ptrdataSize
	}

	if typ.Kind_&kindGCProg == 0 {
		cgoCheckBits(src, typ.GCData, off, size)
		return
	}
	switch typ.Kind_ & kindMask {
	default:
		throw("can't happen")
	case kindArray:
		at := (*arraytype)(unsafe.Pointer(typ))
		for i := uintptr(0); i < at.Len; i++ {
			if off < at.Elem.Size_ {
				cgoCheckUsingType(at.Elem, src, off, size)
			}
			src = add(src, at.Elem.Size_)
			skipped := off
			if skipped > at.Elem.Size_ {
				skipped = at.Elem.Size_
			}
			checked := at.Elem.Size_ - skipped
			off -= skipped
			if size <= checked {
				return
			}
			size -= checked
		}
	case kindStruct:
		st := (*structtype)(unsafe.Pointer(typ))
		for _, f := range st.Fields {
			if off < f.Typ.Size_ {
				cgoCheckUsingType(f.Typ, src, off, size)
			}
			src = add(src, f.Typ.Size_)
			skipped := off
			if skipped > f.Typ.Size_ {
				skipped = f.Typ.Size_
			}
			checked := f.Typ.Size_ - skipped
			off -= skipped
			if size <= checked {
				return
			}
			size -= checked
		}
	}
}