Source file
src/runtime/os_windows.go
1
2
3
4
5 package runtime
6
7 import (
8 "internal/abi"
9 "internal/goarch"
10 "internal/runtime/atomic"
11 "unsafe"
12 )
13
14
15 const (
16 _NSIG = 65
17 )
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70 type stdFunction unsafe.Pointer
71
72 var (
73
74
75
76 _AddVectoredContinueHandler,
77 _AddVectoredExceptionHandler,
78 _CloseHandle,
79 _CreateEventA,
80 _CreateIoCompletionPort,
81 _CreateThread,
82 _CreateWaitableTimerA,
83 _CreateWaitableTimerExW,
84 _DuplicateHandle,
85 _ExitProcess,
86 _FreeEnvironmentStringsW,
87 _GetConsoleMode,
88 _GetCurrentThreadId,
89 _GetEnvironmentStringsW,
90 _GetErrorMode,
91 _GetProcAddress,
92 _GetProcessAffinityMask,
93 _GetQueuedCompletionStatusEx,
94 _GetStdHandle,
95 _GetSystemDirectoryA,
96 _GetSystemInfo,
97 _GetThreadContext,
98 _SetThreadContext,
99 _LoadLibraryExW,
100 _LoadLibraryW,
101 _PostQueuedCompletionStatus,
102 _QueryPerformanceCounter,
103 _RaiseFailFastException,
104 _ResumeThread,
105 _RtlLookupFunctionEntry,
106 _RtlVirtualUnwind,
107 _SetConsoleCtrlHandler,
108 _SetErrorMode,
109 _SetEvent,
110 _SetProcessPriorityBoost,
111 _SetThreadPriority,
112 _SetUnhandledExceptionFilter,
113 _SetWaitableTimer,
114 _SuspendThread,
115 _SwitchToThread,
116 _TlsAlloc,
117 _VirtualAlloc,
118 _VirtualFree,
119 _VirtualQuery,
120 _WaitForSingleObject,
121 _WaitForMultipleObjects,
122 _WerGetFlags,
123 _WerSetFlags,
124 _WriteConsoleW,
125 _WriteFile,
126 _ stdFunction
127
128
129 _ProcessPrng stdFunction
130
131
132
133
134 _NtCreateWaitCompletionPacket stdFunction
135 _NtAssociateWaitCompletionPacket stdFunction
136 _NtCancelWaitCompletionPacket stdFunction
137 _RtlGetCurrentPeb stdFunction
138 _RtlGetVersion stdFunction
139
140
141 _timeBeginPeriod,
142 _timeEndPeriod,
143 _ stdFunction
144 )
145
146 var (
147 bcryptprimitivesdll = [...]uint16{'b', 'c', 'r', 'y', 'p', 't', 'p', 'r', 'i', 'm', 'i', 't', 'i', 'v', 'e', 's', '.', 'd', 'l', 'l', 0}
148 ntdlldll = [...]uint16{'n', 't', 'd', 'l', 'l', '.', 'd', 'l', 'l', 0}
149 powrprofdll = [...]uint16{'p', 'o', 'w', 'r', 'p', 'r', 'o', 'f', '.', 'd', 'l', 'l', 0}
150 winmmdll = [...]uint16{'w', 'i', 'n', 'm', 'm', '.', 'd', 'l', 'l', 0}
151 )
152
153
154
155 func tstart_stdcall(newm *m)
156
157
158 func wintls()
159
160 type mOS struct {
161 threadLock mutex
162 thread uintptr
163
164 waitsema uintptr
165 resumesema uintptr
166
167 highResTimer uintptr
168 waitIocpTimer uintptr
169 waitIocpHandle uintptr
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192 preemptExtLock uint32
193 }
194
195
196 func open(name *byte, mode, perm int32) int32 {
197 throw("unimplemented")
198 return -1
199 }
200 func closefd(fd int32) int32 {
201 throw("unimplemented")
202 return -1
203 }
204 func read(fd int32, p unsafe.Pointer, n int32) int32 {
205 throw("unimplemented")
206 return -1
207 }
208
209 type sigset struct{}
210
211
212
213 func asmstdcall(fn unsafe.Pointer)
214
215 var asmstdcallAddr unsafe.Pointer
216
217 type winlibcall libcall
218
219 func windowsFindfunc(lib uintptr, name []byte) stdFunction {
220 if name[len(name)-1] != 0 {
221 throw("usage")
222 }
223 f := stdcall2(_GetProcAddress, lib, uintptr(unsafe.Pointer(&name[0])))
224 return stdFunction(unsafe.Pointer(f))
225 }
226
227 const _MAX_PATH = 260
228 var sysDirectory [_MAX_PATH + 1]byte
229 var sysDirectoryLen uintptr
230
231 func initSysDirectory() {
232 l := stdcall2(_GetSystemDirectoryA, uintptr(unsafe.Pointer(&sysDirectory[0])), uintptr(len(sysDirectory)-1))
233 if l == 0 || l > uintptr(len(sysDirectory)-1) {
234 throw("Unable to determine system directory")
235 }
236 sysDirectory[l] = '\\'
237 sysDirectoryLen = l + 1
238 }
239
240
241 func windows_GetSystemDirectory() string {
242 return unsafe.String(&sysDirectory[0], sysDirectoryLen)
243 }
244
245 func windowsLoadSystemLib(name []uint16) uintptr {
246 return stdcall3(_LoadLibraryExW, uintptr(unsafe.Pointer(&name[0])), 0, _LOAD_LIBRARY_SEARCH_SYSTEM32)
247 }
248
249 func loadOptionalSyscalls() {
250 bcryptPrimitives := windowsLoadSystemLib(bcryptprimitivesdll[:])
251 if bcryptPrimitives == 0 {
252 throw("bcryptprimitives.dll not found")
253 }
254 _ProcessPrng = windowsFindfunc(bcryptPrimitives, []byte("ProcessPrng\000"))
255
256 n32 := windowsLoadSystemLib(ntdlldll[:])
257 if n32 == 0 {
258 throw("ntdll.dll not found")
259 }
260 _NtCreateWaitCompletionPacket = windowsFindfunc(n32, []byte("NtCreateWaitCompletionPacket\000"))
261 if _NtCreateWaitCompletionPacket != nil {
262
263 _NtAssociateWaitCompletionPacket = windowsFindfunc(n32, []byte("NtAssociateWaitCompletionPacket\000"))
264 if _NtAssociateWaitCompletionPacket == nil {
265 throw("NtCreateWaitCompletionPacket exists but NtAssociateWaitCompletionPacket does not")
266 }
267 _NtCancelWaitCompletionPacket = windowsFindfunc(n32, []byte("NtCancelWaitCompletionPacket\000"))
268 if _NtCancelWaitCompletionPacket == nil {
269 throw("NtCreateWaitCompletionPacket exists but NtCancelWaitCompletionPacket does not")
270 }
271 }
272 _RtlGetCurrentPeb = windowsFindfunc(n32, []byte("RtlGetCurrentPeb\000"))
273 _RtlGetVersion = windowsFindfunc(n32, []byte("RtlGetVersion\000"))
274 }
275
276 func monitorSuspendResume() {
277 const (
278 _DEVICE_NOTIFY_CALLBACK = 2
279 )
280 type _DEVICE_NOTIFY_SUBSCRIBE_PARAMETERS struct {
281 callback uintptr
282 context uintptr
283 }
284
285 powrprof := windowsLoadSystemLib(powrprofdll[:])
286 if powrprof == 0 {
287 return
288 }
289 powerRegisterSuspendResumeNotification := windowsFindfunc(powrprof, []byte("PowerRegisterSuspendResumeNotification\000"))
290 if powerRegisterSuspendResumeNotification == nil {
291 return
292 }
293 var fn any = func(context uintptr, changeType uint32, setting uintptr) uintptr {
294 for mp := (*m)(atomic.Loadp(unsafe.Pointer(&allm))); mp != nil; mp = mp.alllink {
295 if mp.resumesema != 0 {
296 stdcall1(_SetEvent, mp.resumesema)
297 }
298 }
299 return 0
300 }
301 params := _DEVICE_NOTIFY_SUBSCRIBE_PARAMETERS{
302 callback: compileCallback(*efaceOf(&fn), true),
303 }
304 handle := uintptr(0)
305 stdcall3(powerRegisterSuspendResumeNotification, _DEVICE_NOTIFY_CALLBACK,
306 uintptr(unsafe.Pointer(¶ms)), uintptr(unsafe.Pointer(&handle)))
307 }
308
309 func getproccount() int32 {
310 var mask, sysmask uintptr
311 ret := stdcall3(_GetProcessAffinityMask, currentProcess, uintptr(unsafe.Pointer(&mask)), uintptr(unsafe.Pointer(&sysmask)))
312 if ret != 0 {
313 n := 0
314 maskbits := int(unsafe.Sizeof(mask) * 8)
315 for i := 0; i < maskbits; i++ {
316 if mask&(1<<uint(i)) != 0 {
317 n++
318 }
319 }
320 if n != 0 {
321 return int32(n)
322 }
323 }
324
325 var info systeminfo
326 stdcall1(_GetSystemInfo, uintptr(unsafe.Pointer(&info)))
327 return int32(info.dwnumberofprocessors)
328 }
329
330 func getPageSize() uintptr {
331 var info systeminfo
332 stdcall1(_GetSystemInfo, uintptr(unsafe.Pointer(&info)))
333 return uintptr(info.dwpagesize)
334 }
335
336 const (
337 currentProcess = ^uintptr(0)
338 currentThread = ^uintptr(1)
339 )
340
341
342 func getlasterror() uint32
343
344 var timeBeginPeriodRetValue uint32
345
346
347
348
349
350 const osRelaxMinNS = 60 * 1e6
351
352
353
354
355
356
357
358
359
360
361
362 func osRelax(relax bool) uint32 {
363 if haveHighResTimer {
364
365
366
367 return 0
368 }
369
370 if relax {
371 return uint32(stdcall1(_timeEndPeriod, 1))
372 } else {
373 return uint32(stdcall1(_timeBeginPeriod, 1))
374 }
375 }
376
377
378
379 var haveHighResTimer = false
380
381
382
383
384
385
386 var haveHighResSleep = false
387
388
389
390
391
392 func createHighResTimer() uintptr {
393 const (
394
395
396 _CREATE_WAITABLE_TIMER_HIGH_RESOLUTION = 0x00000002
397
398 _SYNCHRONIZE = 0x00100000
399 _TIMER_QUERY_STATE = 0x0001
400 _TIMER_MODIFY_STATE = 0x0002
401 )
402 return stdcall4(_CreateWaitableTimerExW, 0, 0,
403 _CREATE_WAITABLE_TIMER_HIGH_RESOLUTION,
404 _SYNCHRONIZE|_TIMER_QUERY_STATE|_TIMER_MODIFY_STATE)
405 }
406
407 func initHighResTimer() {
408 h := createHighResTimer()
409 if h != 0 {
410 haveHighResTimer = true
411 haveHighResSleep = _NtCreateWaitCompletionPacket != nil
412 stdcall1(_CloseHandle, h)
413 } else {
414
415
416
417 m32 := windowsLoadSystemLib(winmmdll[:])
418 if m32 == 0 {
419 print("runtime: LoadLibraryExW failed; errno=", getlasterror(), "\n")
420 throw("winmm.dll not found")
421 }
422 _timeBeginPeriod = windowsFindfunc(m32, []byte("timeBeginPeriod\000"))
423 _timeEndPeriod = windowsFindfunc(m32, []byte("timeEndPeriod\000"))
424 if _timeBeginPeriod == nil || _timeEndPeriod == nil {
425 print("runtime: GetProcAddress failed; errno=", getlasterror(), "\n")
426 throw("timeBegin/EndPeriod not found")
427 }
428 }
429 }
430
431
432 var canUseLongPaths bool
433
434
435 func initLongPathSupport() {
436 const (
437 IsLongPathAwareProcess = 0x80
438 PebBitFieldOffset = 3
439 )
440
441
442 info := _OSVERSIONINFOW{}
443 info.osVersionInfoSize = uint32(unsafe.Sizeof(info))
444 stdcall1(_RtlGetVersion, uintptr(unsafe.Pointer(&info)))
445 if info.majorVersion < 10 || (info.majorVersion == 10 && info.minorVersion == 0 && info.buildNumber < 15063) {
446 return
447 }
448
449
450
451
452 bitField := (*byte)(unsafe.Pointer(stdcall0(_RtlGetCurrentPeb) + PebBitFieldOffset))
453 *bitField |= IsLongPathAwareProcess
454
455 canUseLongPaths = true
456 }
457
458 func osinit() {
459 asmstdcallAddr = unsafe.Pointer(abi.FuncPCABI0(asmstdcall))
460
461 loadOptionalSyscalls()
462
463 preventErrorDialogs()
464
465 initExceptionHandler()
466
467 initHighResTimer()
468 timeBeginPeriodRetValue = osRelax(false)
469
470 initSysDirectory()
471 initLongPathSupport()
472
473 ncpu = getproccount()
474
475 physPageSize = getPageSize()
476
477
478
479
480
481 stdcall2(_SetProcessPriorityBoost, currentProcess, 1)
482 }
483
484
485 func readRandom(r []byte) int {
486 n := 0
487 if stdcall2(_ProcessPrng, uintptr(unsafe.Pointer(&r[0])), uintptr(len(r)))&0xff != 0 {
488 n = len(r)
489 }
490 return n
491 }
492
493 func goenvs() {
494
495
496
497 strings := unsafe.Pointer(stdcall0(_GetEnvironmentStringsW))
498 p := (*[1 << 24]uint16)(strings)[:]
499
500 n := 0
501 for from, i := 0, 0; true; i++ {
502 if p[i] == 0 {
503
504 if i == from {
505 break
506 }
507 from = i + 1
508 n++
509 }
510 }
511 envs = make([]string, n)
512
513 for i := range envs {
514 envs[i] = gostringw(&p[0])
515 for p[0] != 0 {
516 p = p[1:]
517 }
518 p = p[1:]
519 }
520
521 stdcall1(_FreeEnvironmentStringsW, uintptr(strings))
522
523
524
525 var fn any = ctrlHandler
526 ctrlHandlerPC := compileCallback(*efaceOf(&fn), true)
527 stdcall2(_SetConsoleCtrlHandler, ctrlHandlerPC, 1)
528
529 monitorSuspendResume()
530 }
531
532
533 var exiting uint32
534
535
536 func exit(code int32) {
537
538
539
540
541 lock(&suspendLock)
542 atomic.Store(&exiting, 1)
543 stdcall1(_ExitProcess, uintptr(code))
544 }
545
546
547
548
549
550
551 func write1(fd uintptr, buf unsafe.Pointer, n int32) int32 {
552 const (
553 _STD_OUTPUT_HANDLE = ^uintptr(10)
554 _STD_ERROR_HANDLE = ^uintptr(11)
555 )
556 var handle uintptr
557 switch fd {
558 case 1:
559 handle = stdcall1(_GetStdHandle, _STD_OUTPUT_HANDLE)
560 case 2:
561 handle = stdcall1(_GetStdHandle, _STD_ERROR_HANDLE)
562 default:
563
564 handle = fd
565 }
566 isASCII := true
567 b := (*[1 << 30]byte)(buf)[:n]
568 for _, x := range b {
569 if x >= 0x80 {
570 isASCII = false
571 break
572 }
573 }
574
575 if !isASCII {
576 var m uint32
577 isConsole := stdcall2(_GetConsoleMode, handle, uintptr(unsafe.Pointer(&m))) != 0
578
579
580 if isConsole {
581 return int32(writeConsole(handle, buf, n))
582 }
583 }
584 var written uint32
585 stdcall5(_WriteFile, handle, uintptr(buf), uintptr(n), uintptr(unsafe.Pointer(&written)), 0)
586 return int32(written)
587 }
588
589 var (
590 utf16ConsoleBack [1000]uint16
591 utf16ConsoleBackLock mutex
592 )
593
594
595
596 func writeConsole(handle uintptr, buf unsafe.Pointer, bufLen int32) int {
597 const surr2 = (surrogateMin + surrogateMax + 1) / 2
598
599
600 lock(&utf16ConsoleBackLock)
601
602 b := (*[1 << 30]byte)(buf)[:bufLen]
603 s := *(*string)(unsafe.Pointer(&b))
604
605 utf16tmp := utf16ConsoleBack[:]
606
607 total := len(s)
608 w := 0
609 for _, r := range s {
610 if w >= len(utf16tmp)-2 {
611 writeConsoleUTF16(handle, utf16tmp[:w])
612 w = 0
613 }
614 if r < 0x10000 {
615 utf16tmp[w] = uint16(r)
616 w++
617 } else {
618 r -= 0x10000
619 utf16tmp[w] = surrogateMin + uint16(r>>10)&0x3ff
620 utf16tmp[w+1] = surr2 + uint16(r)&0x3ff
621 w += 2
622 }
623 }
624 writeConsoleUTF16(handle, utf16tmp[:w])
625 unlock(&utf16ConsoleBackLock)
626 return total
627 }
628
629
630
631
632 func writeConsoleUTF16(handle uintptr, b []uint16) {
633 l := uint32(len(b))
634 if l == 0 {
635 return
636 }
637 var written uint32
638 stdcall5(_WriteConsoleW,
639 handle,
640 uintptr(unsafe.Pointer(&b[0])),
641 uintptr(l),
642 uintptr(unsafe.Pointer(&written)),
643 0,
644 )
645 return
646 }
647
648
649 func semasleep(ns int64) int32 {
650 const (
651 _WAIT_ABANDONED = 0x00000080
652 _WAIT_OBJECT_0 = 0x00000000
653 _WAIT_TIMEOUT = 0x00000102
654 _WAIT_FAILED = 0xFFFFFFFF
655 )
656
657 var result uintptr
658 if ns < 0 {
659 result = stdcall2(_WaitForSingleObject, getg().m.waitsema, uintptr(_INFINITE))
660 } else {
661 start := nanotime()
662 elapsed := int64(0)
663 for {
664 ms := int64(timediv(ns-elapsed, 1000000, nil))
665 if ms == 0 {
666 ms = 1
667 }
668 result = stdcall4(_WaitForMultipleObjects, 2,
669 uintptr(unsafe.Pointer(&[2]uintptr{getg().m.waitsema, getg().m.resumesema})),
670 0, uintptr(ms))
671 if result != _WAIT_OBJECT_0+1 {
672
673 break
674 }
675 elapsed = nanotime() - start
676 if elapsed >= ns {
677 return -1
678 }
679 }
680 }
681 switch result {
682 case _WAIT_OBJECT_0:
683 return 0
684
685 case _WAIT_TIMEOUT:
686 return -1
687
688 case _WAIT_ABANDONED:
689 systemstack(func() {
690 throw("runtime.semasleep wait_abandoned")
691 })
692
693 case _WAIT_FAILED:
694 systemstack(func() {
695 print("runtime: waitforsingleobject wait_failed; errno=", getlasterror(), "\n")
696 throw("runtime.semasleep wait_failed")
697 })
698
699 default:
700 systemstack(func() {
701 print("runtime: waitforsingleobject unexpected; result=", result, "\n")
702 throw("runtime.semasleep unexpected")
703 })
704 }
705
706 return -1
707 }
708
709
710 func semawakeup(mp *m) {
711 if stdcall1(_SetEvent, mp.waitsema) == 0 {
712 systemstack(func() {
713 print("runtime: setevent failed; errno=", getlasterror(), "\n")
714 throw("runtime.semawakeup")
715 })
716 }
717 }
718
719
720 func semacreate(mp *m) {
721 if mp.waitsema != 0 {
722 return
723 }
724 mp.waitsema = stdcall4(_CreateEventA, 0, 0, 0, 0)
725 if mp.waitsema == 0 {
726 systemstack(func() {
727 print("runtime: createevent failed; errno=", getlasterror(), "\n")
728 throw("runtime.semacreate")
729 })
730 }
731 mp.resumesema = stdcall4(_CreateEventA, 0, 0, 0, 0)
732 if mp.resumesema == 0 {
733 systemstack(func() {
734 print("runtime: createevent failed; errno=", getlasterror(), "\n")
735 throw("runtime.semacreate")
736 })
737 stdcall1(_CloseHandle, mp.waitsema)
738 mp.waitsema = 0
739 }
740 }
741
742
743
744
745
746
747
748 func newosproc(mp *m) {
749
750 thandle := stdcall6(_CreateThread, 0, 0,
751 abi.FuncPCABI0(tstart_stdcall), uintptr(unsafe.Pointer(mp)),
752 0, 0)
753
754 if thandle == 0 {
755 if atomic.Load(&exiting) != 0 {
756
757
758
759
760 lock(&deadlock)
761 lock(&deadlock)
762 }
763 print("runtime: failed to create new OS thread (have ", mcount(), " already; errno=", getlasterror(), ")\n")
764 throw("runtime.newosproc")
765 }
766
767
768 stdcall1(_CloseHandle, thandle)
769 }
770
771
772
773
774
775
776
777 func newosproc0(mp *m, stk unsafe.Pointer) {
778
779
780
781 throw("bad newosproc0")
782 }
783
784 func exitThread(wait *atomic.Uint32) {
785
786
787 throw("exitThread")
788 }
789
790
791
792 func mpreinit(mp *m) {
793 }
794
795
796 func sigsave(p *sigset) {
797 }
798
799
800 func msigrestore(sigmask sigset) {
801 }
802
803
804
805 func clearSignalHandlers() {
806 }
807
808
809 func sigblock(exiting bool) {
810 }
811
812
813
814 func minit() {
815 var thandle uintptr
816 if stdcall7(_DuplicateHandle, currentProcess, currentThread, currentProcess, uintptr(unsafe.Pointer(&thandle)), 0, 0, _DUPLICATE_SAME_ACCESS) == 0 {
817 print("runtime.minit: duplicatehandle failed; errno=", getlasterror(), "\n")
818 throw("runtime.minit: duplicatehandle failed")
819 }
820
821 mp := getg().m
822 lock(&mp.threadLock)
823 mp.thread = thandle
824 mp.procid = uint64(stdcall0(_GetCurrentThreadId))
825
826
827 if mp.highResTimer == 0 && haveHighResTimer {
828 mp.highResTimer = createHighResTimer()
829 if mp.highResTimer == 0 {
830 print("runtime: CreateWaitableTimerEx failed; errno=", getlasterror(), "\n")
831 throw("CreateWaitableTimerEx when creating timer failed")
832 }
833 }
834 if mp.waitIocpHandle == 0 && haveHighResSleep {
835 mp.waitIocpTimer = createHighResTimer()
836 if mp.waitIocpTimer == 0 {
837 print("runtime: CreateWaitableTimerEx failed; errno=", getlasterror(), "\n")
838 throw("CreateWaitableTimerEx when creating timer failed")
839 }
840 const GENERIC_ALL = 0x10000000
841 errno := stdcall3(_NtCreateWaitCompletionPacket, uintptr(unsafe.Pointer(&mp.waitIocpHandle)), GENERIC_ALL, 0)
842 if mp.waitIocpHandle == 0 {
843 print("runtime: NtCreateWaitCompletionPacket failed; errno=", errno, "\n")
844 throw("NtCreateWaitCompletionPacket failed")
845 }
846 }
847 unlock(&mp.threadLock)
848
849
850
851 var mbi memoryBasicInformation
852 res := stdcall3(_VirtualQuery, uintptr(unsafe.Pointer(&mbi)), uintptr(unsafe.Pointer(&mbi)), unsafe.Sizeof(mbi))
853 if res == 0 {
854 print("runtime: VirtualQuery failed; errno=", getlasterror(), "\n")
855 throw("VirtualQuery for stack base failed")
856 }
857
858
859
860
861
862
863 base := mbi.allocationBase + 16<<10
864
865 g0 := getg()
866 if base > g0.stack.hi || g0.stack.hi-base > 64<<20 {
867 print("runtime: g0 stack [", hex(base), ",", hex(g0.stack.hi), ")\n")
868 throw("bad g0 stack")
869 }
870 g0.stack.lo = base
871 g0.stackguard0 = g0.stack.lo + stackGuard
872 g0.stackguard1 = g0.stackguard0
873
874 stackcheck()
875 }
876
877
878
879
880 func unminit() {
881 mp := getg().m
882 lock(&mp.threadLock)
883 if mp.thread != 0 {
884 stdcall1(_CloseHandle, mp.thread)
885 mp.thread = 0
886 }
887 unlock(&mp.threadLock)
888
889 mp.procid = 0
890 }
891
892
893
894
895
896 func mdestroy(mp *m) {
897 if mp.highResTimer != 0 {
898 stdcall1(_CloseHandle, mp.highResTimer)
899 mp.highResTimer = 0
900 }
901 if mp.waitIocpTimer != 0 {
902 stdcall1(_CloseHandle, mp.waitIocpTimer)
903 mp.waitIocpTimer = 0
904 }
905 if mp.waitIocpHandle != 0 {
906 stdcall1(_CloseHandle, mp.waitIocpHandle)
907 mp.waitIocpHandle = 0
908 }
909 if mp.waitsema != 0 {
910 stdcall1(_CloseHandle, mp.waitsema)
911 mp.waitsema = 0
912 }
913 if mp.resumesema != 0 {
914 stdcall1(_CloseHandle, mp.resumesema)
915 mp.resumesema = 0
916 }
917 }
918
919
920 func asmstdcall_trampoline(args unsafe.Pointer)
921
922
923
924
925 func stdcall_no_g(fn stdFunction, n int, args uintptr) uintptr {
926 libcall := libcall{
927 fn: uintptr(unsafe.Pointer(fn)),
928 n: uintptr(n),
929 args: args,
930 }
931 asmstdcall_trampoline(noescape(unsafe.Pointer(&libcall)))
932 return libcall.r1
933 }
934
935
936
937
938
939
940 func stdcall(fn stdFunction) uintptr {
941 gp := getg()
942 mp := gp.m
943 mp.libcall.fn = uintptr(unsafe.Pointer(fn))
944 resetLibcall := false
945 if mp.profilehz != 0 && mp.libcallsp == 0 {
946
947 mp.libcallg.set(gp)
948 mp.libcallpc = getcallerpc()
949
950
951 mp.libcallsp = getcallersp()
952 resetLibcall = true
953 }
954 asmcgocall(asmstdcallAddr, unsafe.Pointer(&mp.libcall))
955 if resetLibcall {
956 mp.libcallsp = 0
957 }
958 return mp.libcall.r1
959 }
960
961
962 func stdcall0(fn stdFunction) uintptr {
963 mp := getg().m
964 mp.libcall.n = 0
965 mp.libcall.args = 0
966 return stdcall(fn)
967 }
968
969
970
971 func stdcall1(fn stdFunction, a0 uintptr) uintptr {
972 mp := getg().m
973 mp.libcall.n = 1
974 mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0)))
975 return stdcall(fn)
976 }
977
978
979
980 func stdcall2(fn stdFunction, a0, a1 uintptr) uintptr {
981 mp := getg().m
982 mp.libcall.n = 2
983 mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0)))
984 return stdcall(fn)
985 }
986
987
988
989 func stdcall3(fn stdFunction, a0, a1, a2 uintptr) uintptr {
990 mp := getg().m
991 mp.libcall.n = 3
992 mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0)))
993 return stdcall(fn)
994 }
995
996
997
998 func stdcall4(fn stdFunction, a0, a1, a2, a3 uintptr) uintptr {
999 mp := getg().m
1000 mp.libcall.n = 4
1001 mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0)))
1002 return stdcall(fn)
1003 }
1004
1005
1006
1007 func stdcall5(fn stdFunction, a0, a1, a2, a3, a4 uintptr) uintptr {
1008 mp := getg().m
1009 mp.libcall.n = 5
1010 mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0)))
1011 return stdcall(fn)
1012 }
1013
1014
1015
1016 func stdcall6(fn stdFunction, a0, a1, a2, a3, a4, a5 uintptr) uintptr {
1017 mp := getg().m
1018 mp.libcall.n = 6
1019 mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0)))
1020 return stdcall(fn)
1021 }
1022
1023
1024
1025 func stdcall7(fn stdFunction, a0, a1, a2, a3, a4, a5, a6 uintptr) uintptr {
1026 mp := getg().m
1027 mp.libcall.n = 7
1028 mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0)))
1029 return stdcall(fn)
1030 }
1031
1032
1033
1034 func stdcall8(fn stdFunction, a0, a1, a2, a3, a4, a5, a6, a7 uintptr) uintptr {
1035 mp := getg().m
1036 mp.libcall.n = 8
1037 mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0)))
1038 return stdcall(fn)
1039 }
1040
1041
1042
1043
1044 func osyield_no_g() {
1045 stdcall_no_g(_SwitchToThread, 0, 0)
1046 }
1047
1048
1049 func osyield() {
1050 systemstack(func() {
1051 stdcall0(_SwitchToThread)
1052 })
1053 }
1054
1055
1056 func usleep_no_g(us uint32) {
1057 timeout := uintptr(us) / 1000
1058 args := [...]uintptr{_INVALID_HANDLE_VALUE, timeout}
1059 stdcall_no_g(_WaitForSingleObject, len(args), uintptr(noescape(unsafe.Pointer(&args[0]))))
1060 }
1061
1062
1063 func usleep(us uint32) {
1064 systemstack(func() {
1065 var h, timeout uintptr
1066
1067
1068 if haveHighResTimer && getg().m.highResTimer != 0 {
1069 h = getg().m.highResTimer
1070 dt := -10 * int64(us)
1071 stdcall6(_SetWaitableTimer, h, uintptr(unsafe.Pointer(&dt)), 0, 0, 0, 0)
1072 timeout = _INFINITE
1073 } else {
1074 h = _INVALID_HANDLE_VALUE
1075 timeout = uintptr(us) / 1000
1076 }
1077 stdcall2(_WaitForSingleObject, h, timeout)
1078 })
1079 }
1080
1081 func ctrlHandler(_type uint32) uintptr {
1082 var s uint32
1083
1084 switch _type {
1085 case _CTRL_C_EVENT, _CTRL_BREAK_EVENT:
1086 s = _SIGINT
1087 case _CTRL_CLOSE_EVENT, _CTRL_LOGOFF_EVENT, _CTRL_SHUTDOWN_EVENT:
1088 s = _SIGTERM
1089 default:
1090 return 0
1091 }
1092
1093 if sigsend(s) {
1094 if s == _SIGTERM {
1095
1096
1097
1098
1099 block()
1100 }
1101 return 1
1102 }
1103 return 0
1104 }
1105
1106
1107 func callbackasm1()
1108
1109 var profiletimer uintptr
1110
1111 func profilem(mp *m, thread uintptr) {
1112
1113 var c *context
1114 var cbuf [unsafe.Sizeof(*c) + 15]byte
1115 c = (*context)(unsafe.Pointer((uintptr(unsafe.Pointer(&cbuf[15]))) &^ 15))
1116
1117 c.contextflags = _CONTEXT_CONTROL
1118 stdcall2(_GetThreadContext, thread, uintptr(unsafe.Pointer(c)))
1119
1120 gp := gFromSP(mp, c.sp())
1121
1122 sigprof(c.ip(), c.sp(), c.lr(), gp, mp)
1123 }
1124
1125 func gFromSP(mp *m, sp uintptr) *g {
1126 if gp := mp.g0; gp != nil && gp.stack.lo < sp && sp < gp.stack.hi {
1127 return gp
1128 }
1129 if gp := mp.gsignal; gp != nil && gp.stack.lo < sp && sp < gp.stack.hi {
1130 return gp
1131 }
1132 if gp := mp.curg; gp != nil && gp.stack.lo < sp && sp < gp.stack.hi {
1133 return gp
1134 }
1135 return nil
1136 }
1137
1138 func profileLoop() {
1139 stdcall2(_SetThreadPriority, currentThread, _THREAD_PRIORITY_HIGHEST)
1140
1141 for {
1142 stdcall2(_WaitForSingleObject, profiletimer, _INFINITE)
1143 first := (*m)(atomic.Loadp(unsafe.Pointer(&allm)))
1144 for mp := first; mp != nil; mp = mp.alllink {
1145 if mp == getg().m {
1146
1147 continue
1148 }
1149
1150 lock(&mp.threadLock)
1151
1152
1153
1154 if mp.thread == 0 || mp.profilehz == 0 || mp.blocked {
1155 unlock(&mp.threadLock)
1156 continue
1157 }
1158
1159 var thread uintptr
1160 if stdcall7(_DuplicateHandle, currentProcess, mp.thread, currentProcess, uintptr(unsafe.Pointer(&thread)), 0, 0, _DUPLICATE_SAME_ACCESS) == 0 {
1161 print("runtime: duplicatehandle failed; errno=", getlasterror(), "\n")
1162 throw("duplicatehandle failed")
1163 }
1164 unlock(&mp.threadLock)
1165
1166
1167
1168
1169
1170 if int32(stdcall1(_SuspendThread, thread)) == -1 {
1171
1172 stdcall1(_CloseHandle, thread)
1173 continue
1174 }
1175 if mp.profilehz != 0 && !mp.blocked {
1176
1177
1178 profilem(mp, thread)
1179 }
1180 stdcall1(_ResumeThread, thread)
1181 stdcall1(_CloseHandle, thread)
1182 }
1183 }
1184 }
1185
1186 func setProcessCPUProfiler(hz int32) {
1187 if profiletimer == 0 {
1188 var timer uintptr
1189 if haveHighResTimer {
1190 timer = createHighResTimer()
1191 } else {
1192 timer = stdcall3(_CreateWaitableTimerA, 0, 0, 0)
1193 }
1194 atomic.Storeuintptr(&profiletimer, timer)
1195 newm(profileLoop, nil, -1)
1196 }
1197 }
1198
1199 func setThreadCPUProfiler(hz int32) {
1200 ms := int32(0)
1201 due := ^int64(^uint64(1 << 63))
1202 if hz > 0 {
1203 ms = 1000 / hz
1204 if ms == 0 {
1205 ms = 1
1206 }
1207 due = int64(ms) * -10000
1208 }
1209 stdcall6(_SetWaitableTimer, profiletimer, uintptr(unsafe.Pointer(&due)), uintptr(ms), 0, 0, 0)
1210 atomic.Store((*uint32)(unsafe.Pointer(&getg().m.profilehz)), uint32(hz))
1211 }
1212
1213 const preemptMSupported = true
1214
1215
1216
1217 var suspendLock mutex
1218
1219 func preemptM(mp *m) {
1220 if mp == getg().m {
1221 throw("self-preempt")
1222 }
1223
1224
1225 if !atomic.Cas(&mp.preemptExtLock, 0, 1) {
1226
1227
1228 mp.preemptGen.Add(1)
1229 return
1230 }
1231
1232
1233 lock(&mp.threadLock)
1234 if mp.thread == 0 {
1235
1236 unlock(&mp.threadLock)
1237 atomic.Store(&mp.preemptExtLock, 0)
1238 mp.preemptGen.Add(1)
1239 return
1240 }
1241 var thread uintptr
1242 if stdcall7(_DuplicateHandle, currentProcess, mp.thread, currentProcess, uintptr(unsafe.Pointer(&thread)), 0, 0, _DUPLICATE_SAME_ACCESS) == 0 {
1243 print("runtime.preemptM: duplicatehandle failed; errno=", getlasterror(), "\n")
1244 throw("runtime.preemptM: duplicatehandle failed")
1245 }
1246 unlock(&mp.threadLock)
1247
1248
1249 var c *context
1250 var cbuf [unsafe.Sizeof(*c) + 15]byte
1251 c = (*context)(unsafe.Pointer((uintptr(unsafe.Pointer(&cbuf[15]))) &^ 15))
1252 c.contextflags = _CONTEXT_CONTROL
1253
1254
1255
1256
1257
1258
1259 lock(&suspendLock)
1260
1261
1262 if int32(stdcall1(_SuspendThread, thread)) == -1 {
1263 unlock(&suspendLock)
1264 stdcall1(_CloseHandle, thread)
1265 atomic.Store(&mp.preemptExtLock, 0)
1266
1267
1268 mp.preemptGen.Add(1)
1269 return
1270 }
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281 stdcall2(_GetThreadContext, thread, uintptr(unsafe.Pointer(c)))
1282
1283 unlock(&suspendLock)
1284
1285
1286 gp := gFromSP(mp, c.sp())
1287 if gp != nil && wantAsyncPreempt(gp) {
1288 if ok, newpc := isAsyncSafePoint(gp, c.ip(), c.sp(), c.lr()); ok {
1289
1290 targetPC := abi.FuncPCABI0(asyncPreempt)
1291 switch GOARCH {
1292 default:
1293 throw("unsupported architecture")
1294 case "386", "amd64":
1295
1296 sp := c.sp()
1297 sp -= goarch.PtrSize
1298 *(*uintptr)(unsafe.Pointer(sp)) = newpc
1299 c.set_sp(sp)
1300 c.set_ip(targetPC)
1301
1302 case "arm":
1303
1304
1305
1306
1307
1308 sp := c.sp()
1309 sp -= goarch.PtrSize
1310 c.set_sp(sp)
1311 *(*uint32)(unsafe.Pointer(sp)) = uint32(c.lr())
1312 c.set_lr(newpc - 1)
1313 c.set_ip(targetPC)
1314
1315 case "arm64":
1316
1317
1318
1319
1320 sp := c.sp() - 16
1321 c.set_sp(sp)
1322 *(*uint64)(unsafe.Pointer(sp)) = uint64(c.lr())
1323 c.set_lr(newpc)
1324 c.set_ip(targetPC)
1325 }
1326 stdcall2(_SetThreadContext, thread, uintptr(unsafe.Pointer(c)))
1327 }
1328 }
1329
1330 atomic.Store(&mp.preemptExtLock, 0)
1331
1332
1333 mp.preemptGen.Add(1)
1334
1335 stdcall1(_ResumeThread, thread)
1336 stdcall1(_CloseHandle, thread)
1337 }
1338
1339
1340
1341
1342
1343
1344
1345
1346 func osPreemptExtEnter(mp *m) {
1347 for !atomic.Cas(&mp.preemptExtLock, 0, 1) {
1348
1349
1350
1351
1352
1353
1354
1355
1356 osyield()
1357 }
1358
1359 }
1360
1361
1362
1363
1364
1365
1366
1367 func osPreemptExtExit(mp *m) {
1368 atomic.Store(&mp.preemptExtLock, 0)
1369 }
1370
View as plain text