aboutsummaryrefslogtreecommitdiffstats
path: root/contrib/go/_std_1.21/src/runtime/os_linux.go
blob: 0b0561039fa25a787423ee396a1ae9c47185985f (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package runtime

import (
	"internal/abi"
	"internal/goarch"
	"runtime/internal/atomic"
	"runtime/internal/syscall"
	"unsafe"
)

// sigPerThreadSyscall is the same signal (SIGSETXID) used by glibc for
// per-thread syscalls on Linux. We use it for the same purpose in non-cgo
// binaries.
const sigPerThreadSyscall = _SIGRTMIN + 1

type mOS struct {
	// profileTimer holds the ID of the POSIX interval timer for profiling CPU
	// usage on this thread.
	//
	// It is valid when the profileTimerValid field is true. A thread
	// creates and manages its own timer, and these fields are read and written
	// only by this thread. But because some of the reads on profileTimerValid
	// are in signal handling code, this field should be atomic type.
	profileTimer      int32
	profileTimerValid atomic.Bool

	// needPerThreadSyscall indicates that a per-thread syscall is required
	// for doAllThreadsSyscall.
	needPerThreadSyscall atomic.Uint8
}

//go:noescape
func futex(addr unsafe.Pointer, op int32, val uint32, ts, addr2 unsafe.Pointer, val3 uint32) int32

// Linux futex.
//
//	futexsleep(uint32 *addr, uint32 val)
//	futexwakeup(uint32 *addr)
//
// Futexsleep atomically checks if *addr == val and if so, sleeps on addr.
// Futexwakeup wakes up threads sleeping on addr.
// Futexsleep is allowed to wake up spuriously.

const (
	_FUTEX_PRIVATE_FLAG = 128
	_FUTEX_WAIT_PRIVATE = 0 | _FUTEX_PRIVATE_FLAG
	_FUTEX_WAKE_PRIVATE = 1 | _FUTEX_PRIVATE_FLAG
)

// Atomically,
//
//	if(*addr == val) sleep
//
// Might be woken up spuriously; that's allowed.
// Don't sleep longer than ns; ns < 0 means forever.
//
//go:nosplit
func futexsleep(addr *uint32, val uint32, ns int64) {
	// Some Linux kernels have a bug where futex of
	// FUTEX_WAIT returns an internal error code
	// as an errno. Libpthread ignores the return value
	// here, and so can we: as it says a few lines up,
	// spurious wakeups are allowed.
	if ns < 0 {
		futex(unsafe.Pointer(addr), _FUTEX_WAIT_PRIVATE, val, nil, nil, 0)
		return
	}

	var ts timespec
	ts.setNsec(ns)
	futex(unsafe.Pointer(addr), _FUTEX_WAIT_PRIVATE, val, unsafe.Pointer(&ts), nil, 0)
}

// If any procs are sleeping on addr, wake up at most cnt.
//
//go:nosplit
func futexwakeup(addr *uint32, cnt uint32) {
	ret := futex(unsafe.Pointer(addr), _FUTEX_WAKE_PRIVATE, cnt, nil, nil, 0)
	if ret >= 0 {
		return
	}

	// I don't know that futex wakeup can return
	// EAGAIN or EINTR, but if it does, it would be
	// safe to loop and call futex again.
	systemstack(func() {
		print("futexwakeup addr=", addr, " returned ", ret, "\n")
	})

	*(*int32)(unsafe.Pointer(uintptr(0x1006))) = 0x1006
}

func getproccount() int32 {
	// This buffer is huge (8 kB) but we are on the system stack
	// and there should be plenty of space (64 kB).
	// Also this is a leaf, so we're not holding up the memory for long.
	// See golang.org/issue/11823.
	// The suggested behavior here is to keep trying with ever-larger
	// buffers, but we don't have a dynamic memory allocator at the
	// moment, so that's a bit tricky and seems like overkill.
	const maxCPUs = 64 * 1024
	var buf [maxCPUs / 8]byte
	r := sched_getaffinity(0, unsafe.Sizeof(buf), &buf[0])
	if r < 0 {
		return 1
	}
	n := int32(0)
	for _, v := range buf[:r] {
		for v != 0 {
			n += int32(v & 1)
			v >>= 1
		}
	}
	if n == 0 {
		n = 1
	}
	return n
}

// Clone, the Linux rfork.
const (
	_CLONE_VM             = 0x100
	_CLONE_FS             = 0x200
	_CLONE_FILES          = 0x400
	_CLONE_SIGHAND        = 0x800
	_CLONE_PTRACE         = 0x2000
	_CLONE_VFORK          = 0x4000
	_CLONE_PARENT         = 0x8000
	_CLONE_THREAD         = 0x10000
	_CLONE_NEWNS          = 0x20000
	_CLONE_SYSVSEM        = 0x40000
	_CLONE_SETTLS         = 0x80000
	_CLONE_PARENT_SETTID  = 0x100000
	_CLONE_CHILD_CLEARTID = 0x200000
	_CLONE_UNTRACED       = 0x800000
	_CLONE_CHILD_SETTID   = 0x1000000
	_CLONE_STOPPED        = 0x2000000
	_CLONE_NEWUTS         = 0x4000000
	_CLONE_NEWIPC         = 0x8000000

	// As of QEMU 2.8.0 (5ea2fc84d), user emulation requires all six of these
	// flags to be set when creating a thread; attempts to share the other
	// five but leave SYSVSEM unshared will fail with -EINVAL.
	//
	// In non-QEMU environments CLONE_SYSVSEM is inconsequential as we do not
	// use System V semaphores.

	cloneFlags = _CLONE_VM | /* share memory */
		_CLONE_FS | /* share cwd, etc */
		_CLONE_FILES | /* share fd table */
		_CLONE_SIGHAND | /* share sig handler table */
		_CLONE_SYSVSEM | /* share SysV semaphore undo lists (see issue #20763) */
		_CLONE_THREAD /* revisit - okay for now */
)

//go:noescape
func clone(flags int32, stk, mp, gp, fn unsafe.Pointer) int32

// May run with m.p==nil, so write barriers are not allowed.
//
//go:nowritebarrier
func newosproc(mp *m) {
	stk := unsafe.Pointer(mp.g0.stack.hi)
	/*
	 * note: strace gets confused if we use CLONE_PTRACE here.
	 */
	if false {
		print("newosproc stk=", stk, " m=", mp, " g=", mp.g0, " clone=", abi.FuncPCABI0(clone), " id=", mp.id, " ostk=", &mp, "\n")
	}

	// Disable signals during clone, so that the new thread starts
	// with signals disabled. It will enable them in minit.
	var oset sigset
	sigprocmask(_SIG_SETMASK, &sigset_all, &oset)
	ret := retryOnEAGAIN(func() int32 {
		r := clone(cloneFlags, stk, unsafe.Pointer(mp), unsafe.Pointer(mp.g0), unsafe.Pointer(abi.FuncPCABI0(mstart)))
		// clone returns positive TID, negative errno.
		// We don't care about the TID.
		if r >= 0 {
			return 0
		}
		return -r
	})
	sigprocmask(_SIG_SETMASK, &oset, nil)

	if ret != 0 {
		print("runtime: failed to create new OS thread (have ", mcount(), " already; errno=", ret, ")\n")
		if ret == _EAGAIN {
			println("runtime: may need to increase max user processes (ulimit -u)")
		}
		throw("newosproc")
	}
}

// Version of newosproc that doesn't require a valid G.
//
//go:nosplit
func newosproc0(stacksize uintptr, fn unsafe.Pointer) {
	stack := sysAlloc(stacksize, &memstats.stacks_sys)
	if stack == nil {
		writeErrStr(failallocatestack)
		exit(1)
	}
	ret := clone(cloneFlags, unsafe.Pointer(uintptr(stack)+stacksize), nil, nil, fn)
	if ret < 0 {
		writeErrStr(failthreadcreate)
		exit(1)
	}
}

const (
	_AT_NULL   = 0  // End of vector
	_AT_PAGESZ = 6  // System physical page size
	_AT_HWCAP  = 16 // hardware capability bit vector
	_AT_SECURE = 23 // secure mode boolean
	_AT_RANDOM = 25 // introduced in 2.6.29
	_AT_HWCAP2 = 26 // hardware capability bit vector 2
)

var procAuxv = []byte("/proc/self/auxv\x00")

var addrspace_vec [1]byte

func mincore(addr unsafe.Pointer, n uintptr, dst *byte) int32

var auxvreadbuf [128]uintptr

func sysargs(argc int32, argv **byte) {
	n := argc + 1

	// skip over argv, envp to get to auxv
	for argv_index(argv, n) != nil {
		n++
	}

	// skip NULL separator
	n++

	// now argv+n is auxv
	auxvp := (*[1 << 28]uintptr)(add(unsafe.Pointer(argv), uintptr(n)*goarch.PtrSize))

	if pairs := sysauxv(auxvp[:]); pairs != 0 {
		auxv = auxvp[: pairs*2 : pairs*2]
		return
	}
	// In some situations we don't get a loader-provided
	// auxv, such as when loaded as a library on Android.
	// Fall back to /proc/self/auxv.
	fd := open(&procAuxv[0], 0 /* O_RDONLY */, 0)
	if fd < 0 {
		// On Android, /proc/self/auxv might be unreadable (issue 9229), so we fallback to
		// try using mincore to detect the physical page size.
		// mincore should return EINVAL when address is not a multiple of system page size.
		const size = 256 << 10 // size of memory region to allocate
		p, err := mmap(nil, size, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_PRIVATE, -1, 0)
		if err != 0 {
			return
		}
		var n uintptr
		for n = 4 << 10; n < size; n <<= 1 {
			err := mincore(unsafe.Pointer(uintptr(p)+n), 1, &addrspace_vec[0])
			if err == 0 {
				physPageSize = n
				break
			}
		}
		if physPageSize == 0 {
			physPageSize = size
		}
		munmap(p, size)
		return
	}

	n = read(fd, noescape(unsafe.Pointer(&auxvreadbuf[0])), int32(unsafe.Sizeof(auxvreadbuf)))
	closefd(fd)
	if n < 0 {
		return
	}
	// Make sure buf is terminated, even if we didn't read
	// the whole file.
	auxvreadbuf[len(auxvreadbuf)-2] = _AT_NULL
	pairs := sysauxv(auxvreadbuf[:])
	auxv = auxvreadbuf[: pairs*2 : pairs*2]
}

// startupRandomData holds random bytes initialized at startup. These come from
// the ELF AT_RANDOM auxiliary vector.
var startupRandomData []byte

// secureMode holds the value of AT_SECURE passed in the auxiliary vector.
var secureMode bool

func sysauxv(auxv []uintptr) (pairs int) {
	var i int
	for ; auxv[i] != _AT_NULL; i += 2 {
		tag, val := auxv[i], auxv[i+1]
		switch tag {
		case _AT_RANDOM:
			// The kernel provides a pointer to 16-bytes
			// worth of random data.
			startupRandomData = (*[16]byte)(unsafe.Pointer(val))[:]

		case _AT_PAGESZ:
			physPageSize = val

		case _AT_SECURE:
			secureMode = val == 1
		}

		archauxv(tag, val)
		vdsoauxv(tag, val)
	}
	return i / 2
}

var sysTHPSizePath = []byte("/sys/kernel/mm/transparent_hugepage/hpage_pmd_size\x00")

func getHugePageSize() uintptr {
	var numbuf [20]byte
	fd := open(&sysTHPSizePath[0], 0 /* O_RDONLY */, 0)
	if fd < 0 {
		return 0
	}
	ptr := noescape(unsafe.Pointer(&numbuf[0]))
	n := read(fd, ptr, int32(len(numbuf)))
	closefd(fd)
	if n <= 0 {
		return 0
	}
	n-- // remove trailing newline
	v, ok := atoi(slicebytetostringtmp((*byte)(ptr), int(n)))
	if !ok || v < 0 {
		v = 0
	}
	if v&(v-1) != 0 {
		// v is not a power of 2
		return 0
	}
	return uintptr(v)
}

func osinit() {
	ncpu = getproccount()
	physHugePageSize = getHugePageSize()
	if iscgo {
		// #42494 glibc and musl reserve some signals for
		// internal use and require they not be blocked by
		// the rest of a normal C runtime. When the go runtime
		// blocks...unblocks signals, temporarily, the blocked
		// interval of time is generally very short. As such,
		// these expectations of *libc code are mostly met by
		// the combined go+cgo system of threads. However,
		// when go causes a thread to exit, via a return from
		// mstart(), the combined runtime can deadlock if
		// these signals are blocked. Thus, don't block these
		// signals when exiting threads.
		// - glibc: SIGCANCEL (32), SIGSETXID (33)
		// - musl: SIGTIMER (32), SIGCANCEL (33), SIGSYNCCALL (34)
		sigdelset(&sigsetAllExiting, 32)
		sigdelset(&sigsetAllExiting, 33)
		sigdelset(&sigsetAllExiting, 34)
	}
	osArchInit()
}

var urandom_dev = []byte("/dev/urandom\x00")

func getRandomData(r []byte) {
	if startupRandomData != nil {
		n := copy(r, startupRandomData)
		extendRandom(r, n)
		return
	}
	fd := open(&urandom_dev[0], 0 /* O_RDONLY */, 0)
	n := read(fd, unsafe.Pointer(&r[0]), int32(len(r)))
	closefd(fd)
	extendRandom(r, int(n))
}

func goenvs() {
	goenvs_unix()
}

// Called to do synchronous initialization of Go code built with
// -buildmode=c-archive or -buildmode=c-shared.
// None of the Go runtime is initialized.
//
//go:nosplit
//go:nowritebarrierrec
func libpreinit() {
	initsig(true)
}

// Called to initialize a new m (including the bootstrap m).
// Called on the parent thread (main thread in case of bootstrap), can allocate memory.
func mpreinit(mp *m) {
	mp.gsignal = malg(32 * 1024) // Linux wants >= 2K
	mp.gsignal.m = mp
}

func gettid() uint32

// Called to initialize a new m (including the bootstrap m).
// Called on the new thread, cannot allocate memory.
func minit() {
	minitSignals()

	// Cgo-created threads and the bootstrap m are missing a
	// procid. We need this for asynchronous preemption and it's
	// useful in debuggers.
	getg().m.procid = uint64(gettid())
}

// Called from dropm to undo the effect of an minit.
//
//go:nosplit
func unminit() {
	unminitSignals()
}

// Called from exitm, but not from drop, to undo the effect of thread-owned
// resources in minit, semacreate, or elsewhere. Do not take locks after calling this.
func mdestroy(mp *m) {
}

//#ifdef GOARCH_386
//#define sa_handler k_sa_handler
//#endif

func sigreturn__sigaction()
func sigtramp() // Called via C ABI
func cgoSigtramp()

//go:noescape
func sigaltstack(new, old *stackt)

//go:noescape
func setitimer(mode int32, new, old *itimerval)

//go:noescape
func timer_create(clockid int32, sevp *sigevent, timerid *int32) int32

//go:noescape
func timer_settime(timerid int32, flags int32, new, old *itimerspec) int32

//go:noescape
func timer_delete(timerid int32) int32

//go:noescape
func rtsigprocmask(how int32, new, old *sigset, size int32)

//go:nosplit
//go:nowritebarrierrec
func sigprocmask(how int32, new, old *sigset) {
	rtsigprocmask(how, new, old, int32(unsafe.Sizeof(*new)))
}

func raise(sig uint32)
func raiseproc(sig uint32)

//go:noescape
func sched_getaffinity(pid, len uintptr, buf *byte) int32
func osyield()

//go:nosplit
func osyield_no_g() {
	osyield()
}

func pipe2(flags int32) (r, w int32, errno int32)

//go:nosplit
func fcntl(fd, cmd, arg int32) (ret int32, errno int32) {
	r, _, err := syscall.Syscall6(syscall.SYS_FCNTL, uintptr(fd), uintptr(cmd), uintptr(arg), 0, 0, 0)
	return int32(r), int32(err)
}

const (
	_si_max_size    = 128
	_sigev_max_size = 64
)

//go:nosplit
//go:nowritebarrierrec
func setsig(i uint32, fn uintptr) {
	var sa sigactiont
	sa.sa_flags = _SA_SIGINFO | _SA_ONSTACK | _SA_RESTORER | _SA_RESTART
	sigfillset(&sa.sa_mask)
	// Although Linux manpage says "sa_restorer element is obsolete and
	// should not be used". x86_64 kernel requires it. Only use it on
	// x86.
	if GOARCH == "386" || GOARCH == "amd64" {
		sa.sa_restorer = abi.FuncPCABI0(sigreturn__sigaction)
	}
	if fn == abi.FuncPCABIInternal(sighandler) { // abi.FuncPCABIInternal(sighandler) matches the callers in signal_unix.go
		if iscgo {
			fn = abi.FuncPCABI0(cgoSigtramp)
		} else {
			fn = abi.FuncPCABI0(sigtramp)
		}
	}
	sa.sa_handler = fn
	sigaction(i, &sa, nil)
}

//go:nosplit
//go:nowritebarrierrec
func setsigstack(i uint32) {
	var sa sigactiont
	sigaction(i, nil, &sa)
	if sa.sa_flags&_SA_ONSTACK != 0 {
		return
	}
	sa.sa_flags |= _SA_ONSTACK
	sigaction(i, &sa, nil)
}

//go:nosplit
//go:nowritebarrierrec
func getsig(i uint32) uintptr {
	var sa sigactiont
	sigaction(i, nil, &sa)
	return sa.sa_handler
}

// setSignalstackSP sets the ss_sp field of a stackt.
//
//go:nosplit
func setSignalstackSP(s *stackt, sp uintptr) {
	*(*uintptr)(unsafe.Pointer(&s.ss_sp)) = sp
}

//go:nosplit
func (c *sigctxt) fixsigcode(sig uint32) {
}

// sysSigaction calls the rt_sigaction system call.
//
//go:nosplit
func sysSigaction(sig uint32, new, old *sigactiont) {
	if rt_sigaction(uintptr(sig), new, old, unsafe.Sizeof(sigactiont{}.sa_mask)) != 0 {
		// Workaround for bugs in QEMU user mode emulation.
		//
		// QEMU turns calls to the sigaction system call into
		// calls to the C library sigaction call; the C
		// library call rejects attempts to call sigaction for
		// SIGCANCEL (32) or SIGSETXID (33).
		//
		// QEMU rejects calling sigaction on SIGRTMAX (64).
		//
		// Just ignore the error in these case. There isn't
		// anything we can do about it anyhow.
		if sig != 32 && sig != 33 && sig != 64 {
			// Use system stack to avoid split stack overflow on ppc64/ppc64le.
			systemstack(func() {
				throw("sigaction failed")
			})
		}
	}
}

// rt_sigaction is implemented in assembly.
//
//go:noescape
func rt_sigaction(sig uintptr, new, old *sigactiont, size uintptr) int32

func getpid() int
func tgkill(tgid, tid, sig int)

// signalM sends a signal to mp.
func signalM(mp *m, sig int) {
	tgkill(getpid(), int(mp.procid), sig)
}

// validSIGPROF compares this signal delivery's code against the signal sources
// that the profiler uses, returning whether the delivery should be processed.
// To be processed, a signal delivery from a known profiling mechanism should
// correspond to the best profiling mechanism available to this thread. Signals
// from other sources are always considered valid.
//
//go:nosplit
func validSIGPROF(mp *m, c *sigctxt) bool {
	code := int32(c.sigcode())
	setitimer := code == _SI_KERNEL
	timer_create := code == _SI_TIMER

	if !(setitimer || timer_create) {
		// The signal doesn't correspond to a profiling mechanism that the
		// runtime enables itself. There's no reason to process it, but there's
		// no reason to ignore it either.
		return true
	}

	if mp == nil {
		// Since we don't have an M, we can't check if there's an active
		// per-thread timer for this thread. We don't know how long this thread
		// has been around, and if it happened to interact with the Go scheduler
		// at a time when profiling was active (causing it to have a per-thread
		// timer). But it may have never interacted with the Go scheduler, or
		// never while profiling was active. To avoid double-counting, process
		// only signals from setitimer.
		//
		// When a custom cgo traceback function has been registered (on
		// platforms that support runtime.SetCgoTraceback), SIGPROF signals
		// delivered to a thread that cannot find a matching M do this check in
		// the assembly implementations of runtime.cgoSigtramp.
		return setitimer
	}

	// Having an M means the thread interacts with the Go scheduler, and we can
	// check whether there's an active per-thread timer for this thread.
	if mp.profileTimerValid.Load() {
		// If this M has its own per-thread CPU profiling interval timer, we
		// should track the SIGPROF signals that come from that timer (for
		// accurate reporting of its CPU usage; see issue 35057) and ignore any
		// that it gets from the process-wide setitimer (to not over-count its
		// CPU consumption).
		return timer_create
	}

	// No active per-thread timer means the only valid profiler is setitimer.
	return setitimer
}

func setProcessCPUProfiler(hz int32) {
	setProcessCPUProfilerTimer(hz)
}

func setThreadCPUProfiler(hz int32) {
	mp := getg().m
	mp.profilehz = hz

	// destroy any active timer
	if mp.profileTimerValid.Load() {
		timerid := mp.profileTimer
		mp.profileTimerValid.Store(false)
		mp.profileTimer = 0

		ret := timer_delete(timerid)
		if ret != 0 {
			print("runtime: failed to disable profiling timer; timer_delete(", timerid, ") errno=", -ret, "\n")
			throw("timer_delete")
		}
	}

	if hz == 0 {
		// If the goal was to disable profiling for this thread, then the job's done.
		return
	}

	// The period of the timer should be 1/Hz. For every "1/Hz" of additional
	// work, the user should expect one additional sample in the profile.
	//
	// But to scale down to very small amounts of application work, to observe
	// even CPU usage of "one tenth" of the requested period, set the initial
	// timing delay in a different way: So that "one tenth" of a period of CPU
	// spend shows up as a 10% chance of one sample (for an expected value of
	// 0.1 samples), and so that "two and six tenths" periods of CPU spend show
	// up as a 60% chance of 3 samples and a 40% chance of 2 samples (for an
	// expected value of 2.6). Set the initial delay to a value in the unifom
	// random distribution between 0 and the desired period. And because "0"
	// means "disable timer", add 1 so the half-open interval [0,period) turns
	// into (0,period].
	//
	// Otherwise, this would show up as a bias away from short-lived threads and
	// from threads that are only occasionally active: for example, when the
	// garbage collector runs on a mostly-idle system, the additional threads it
	// activates may do a couple milliseconds of GC-related work and nothing
	// else in the few seconds that the profiler observes.
	spec := new(itimerspec)
	spec.it_value.setNsec(1 + int64(fastrandn(uint32(1e9/hz))))
	spec.it_interval.setNsec(1e9 / int64(hz))

	var timerid int32
	var sevp sigevent
	sevp.notify = _SIGEV_THREAD_ID
	sevp.signo = _SIGPROF
	sevp.sigev_notify_thread_id = int32(mp.procid)
	ret := timer_create(_CLOCK_THREAD_CPUTIME_ID, &sevp, &timerid)
	if ret != 0 {
		// If we cannot create a timer for this M, leave profileTimerValid false
		// to fall back to the process-wide setitimer profiler.
		return
	}

	ret = timer_settime(timerid, 0, spec, nil)
	if ret != 0 {
		print("runtime: failed to configure profiling timer; timer_settime(", timerid,
			", 0, {interval: {",
			spec.it_interval.tv_sec, "s + ", spec.it_interval.tv_nsec, "ns} value: {",
			spec.it_value.tv_sec, "s + ", spec.it_value.tv_nsec, "ns}}, nil) errno=", -ret, "\n")
		throw("timer_settime")
	}

	mp.profileTimer = timerid
	mp.profileTimerValid.Store(true)
}

// perThreadSyscallArgs contains the system call number, arguments, and
// expected return values for a system call to be executed on all threads.
type perThreadSyscallArgs struct {
	trap uintptr
	a1   uintptr
	a2   uintptr
	a3   uintptr
	a4   uintptr
	a5   uintptr
	a6   uintptr
	r1   uintptr
	r2   uintptr
}

// perThreadSyscall is the system call to execute for the ongoing
// doAllThreadsSyscall.
//
// perThreadSyscall may only be written while mp.needPerThreadSyscall == 0 on
// all Ms.
var perThreadSyscall perThreadSyscallArgs

// syscall_runtime_doAllThreadsSyscall and executes a specified system call on
// all Ms.
//
// The system call is expected to succeed and return the same value on every
// thread. If any threads do not match, the runtime throws.
//
//go:linkname syscall_runtime_doAllThreadsSyscall syscall.runtime_doAllThreadsSyscall
//go:uintptrescapes
func syscall_runtime_doAllThreadsSyscall(trap, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, err uintptr) {
	if iscgo {
		// In cgo, we are not aware of threads created in C, so this approach will not work.
		panic("doAllThreadsSyscall not supported with cgo enabled")
	}

	// STW to guarantee that user goroutines see an atomic change to thread
	// state. Without STW, goroutines could migrate Ms while change is in
	// progress and e.g., see state old -> new -> old -> new.
	//
	// N.B. Internally, this function does not depend on STW to
	// successfully change every thread. It is only needed for user
	// expectations, per above.
	stopTheWorld(stwAllThreadsSyscall)

	// This function depends on several properties:
	//
	// 1. All OS threads that already exist are associated with an M in
	//    allm. i.e., we won't miss any pre-existing threads.
	// 2. All Ms listed in allm will eventually have an OS thread exist.
	//    i.e., they will set procid and be able to receive signals.
	// 3. OS threads created after we read allm will clone from a thread
	//    that has executed the system call. i.e., they inherit the
	//    modified state.
	//
	// We achieve these through different mechanisms:
	//
	// 1. Addition of new Ms to allm in allocm happens before clone of its
	//    OS thread later in newm.
	// 2. newm does acquirem to avoid being preempted, ensuring that new Ms
	//    created in allocm will eventually reach OS thread clone later in
	//    newm.
	// 3. We take allocmLock for write here to prevent allocation of new Ms
	//    while this function runs. Per (1), this prevents clone of OS
	//    threads that are not yet in allm.
	allocmLock.lock()

	// Disable preemption, preventing us from changing Ms, as we handle
	// this M specially.
	//
	// N.B. STW and lock() above do this as well, this is added for extra
	// clarity.
	acquirem()

	// N.B. allocmLock also prevents concurrent execution of this function,
	// serializing use of perThreadSyscall, mp.needPerThreadSyscall, and
	// ensuring all threads execute system calls from multiple calls in the
	// same order.

	r1, r2, errno := syscall.Syscall6(trap, a1, a2, a3, a4, a5, a6)
	if GOARCH == "ppc64" || GOARCH == "ppc64le" {
		// TODO(https://go.dev/issue/51192 ): ppc64 doesn't use r2.
		r2 = 0
	}
	if errno != 0 {
		releasem(getg().m)
		allocmLock.unlock()
		startTheWorld()
		return r1, r2, errno
	}

	perThreadSyscall = perThreadSyscallArgs{
		trap: trap,
		a1:   a1,
		a2:   a2,
		a3:   a3,
		a4:   a4,
		a5:   a5,
		a6:   a6,
		r1:   r1,
		r2:   r2,
	}

	// Wait for all threads to start.
	//
	// As described above, some Ms have been added to allm prior to
	// allocmLock, but not yet completed OS clone and set procid.
	//
	// At minimum we must wait for a thread to set procid before we can
	// send it a signal.
	//
	// We take this one step further and wait for all threads to start
	// before sending any signals. This prevents system calls from getting
	// applied twice: once in the parent and once in the child, like so:
	//
	//          A                     B                  C
	//                         add C to allm
	// doAllThreadsSyscall
	//   allocmLock.lock()
	//   signal B
	//                         <receive signal>
	//                         execute syscall
	//                         <signal return>
	//                         clone C
	//                                             <thread start>
	//                                             set procid
	//   signal C
	//                                             <receive signal>
	//                                             execute syscall
	//                                             <signal return>
	//
	// In this case, thread C inherited the syscall-modified state from
	// thread B and did not need to execute the syscall, but did anyway
	// because doAllThreadsSyscall could not be sure whether it was
	// required.
	//
	// Some system calls may not be idempotent, so we ensure each thread
	// executes the system call exactly once.
	for mp := allm; mp != nil; mp = mp.alllink {
		for atomic.Load64(&mp.procid) == 0 {
			// Thread is starting.
			osyield()
		}
	}

	// Signal every other thread, where they will execute perThreadSyscall
	// from the signal handler.
	gp := getg()
	tid := gp.m.procid
	for mp := allm; mp != nil; mp = mp.alllink {
		if atomic.Load64(&mp.procid) == tid {
			// Our thread already performed the syscall.
			continue
		}
		mp.needPerThreadSyscall.Store(1)
		signalM(mp, sigPerThreadSyscall)
	}

	// Wait for all threads to complete.
	for mp := allm; mp != nil; mp = mp.alllink {
		if mp.procid == tid {
			continue
		}
		for mp.needPerThreadSyscall.Load() != 0 {
			osyield()
		}
	}

	perThreadSyscall = perThreadSyscallArgs{}

	releasem(getg().m)
	allocmLock.unlock()
	startTheWorld()

	return r1, r2, errno
}

// runPerThreadSyscall runs perThreadSyscall for this M if required.
//
// This function throws if the system call returns with anything other than the
// expected values.
//
//go:nosplit
func runPerThreadSyscall() {
	gp := getg()
	if gp.m.needPerThreadSyscall.Load() == 0 {
		return
	}

	args := perThreadSyscall
	r1, r2, errno := syscall.Syscall6(args.trap, args.a1, args.a2, args.a3, args.a4, args.a5, args.a6)
	if GOARCH == "ppc64" || GOARCH == "ppc64le" {
		// TODO(https://go.dev/issue/51192 ): ppc64 doesn't use r2.
		r2 = 0
	}
	if errno != 0 || r1 != args.r1 || r2 != args.r2 {
		print("trap:", args.trap, ", a123456=[", args.a1, ",", args.a2, ",", args.a3, ",", args.a4, ",", args.a5, ",", args.a6, "]\n")
		print("results: got {r1=", r1, ",r2=", r2, ",errno=", errno, "}, want {r1=", args.r1, ",r2=", args.r2, ",errno=0}\n")
		fatal("AllThreadsSyscall6 results differ between threads; runtime corrupted")
	}

	gp.m.needPerThreadSyscall.Store(0)
}

const (
	_SI_USER  = 0
	_SI_TKILL = -6
)

// sigFromUser reports whether the signal was sent because of a call
// to kill or tgkill.
//
//go:nosplit
func (c *sigctxt) sigFromUser() bool {
	code := int32(c.sigcode())
	return code == _SI_USER || code == _SI_TKILL
}