var modinfo … var m0 … var g0 … var mcache0 … var raceprocctx0 … var raceFiniLock … var runtime_inittasks … var main_init_done … //go:linkname main_main main.main func main_main() var mainStarted … var runtimeInitTime … var initSigmask … // The main goroutine. func main() { … } // os_beforeExit is called from os.Exit(0). // //go:linkname os_beforeExit os.runtime_beforeExit func os_beforeExit(exitCode int) { … } func init() { … } func runExitHooks(code int) { … } // start forcegc helper goroutine func init() { … } func forcegchelper() { … } // Gosched yields the processor, allowing other goroutines to run. It does not // suspend the current goroutine, so execution resumes automatically. // //go:nosplit func Gosched() { … } // goschedguarded yields the processor like gosched, but also checks // for forbidden states and opts out of the yield in those cases. // //go:nosplit func goschedguarded() { … } // goschedIfBusy yields the processor like gosched, but only does so if // there are no idle Ps or if we're on the only P and there's nothing in // the run queue. In both cases, there is freely available idle time. // //go:nosplit func goschedIfBusy() { … } // Puts the current goroutine into a waiting state and calls unlockf on the // system stack. // // If unlockf returns false, the goroutine is resumed. // // unlockf must not access this G's stack, as it may be moved between // the call to gopark and the call to unlockf. // // Note that because unlockf is called after putting the G into a waiting // state, the G may have already been readied by the time unlockf is called // unless there is external synchronization preventing the G from being // readied. If unlockf returns false, it must guarantee that the G cannot be // externally readied. // // Reason explains why the goroutine has been parked. It is displayed in stack // traces and heap dumps. Reasons should be unique and descriptive. Do not // re-use reasons, add new ones. // // gopark should be an internal detail, // but widely used packages access it using linkname. // Notable members of the hall of shame include: // - gvisor.dev/gvisor // - github.com/sagernet/gvisor // // Do not remove or change the type signature. // See go.dev/issue/67401. // //go:linkname gopark func gopark(unlockf func(*g, unsafe.Pointer) bool, lock unsafe.Pointer, reason waitReason, traceReason traceBlockReason, traceskip int) { … } // Puts the current goroutine into a waiting state and unlocks the lock. // The goroutine can be made runnable again by calling goready(gp). func goparkunlock(lock *mutex, reason waitReason, traceReason traceBlockReason, traceskip int) { … } // goready should be an internal detail, // but widely used packages access it using linkname. // Notable members of the hall of shame include: // - gvisor.dev/gvisor // - github.com/sagernet/gvisor // // Do not remove or change the type signature. // See go.dev/issue/67401. // //go:linkname goready func goready(gp *g, traceskip int) { … } //go:nosplit func acquireSudog() *sudog { … } //go:nosplit func releaseSudog(s *sudog) { … } // called from assembly. func badmcall(fn func(*g)) { … } func badmcall2(fn func(*g)) { … } func badreflectcall() { … } //go:nosplit //go:nowritebarrierrec func badmorestackg0() { … } //go:nosplit //go:nowritebarrierrec func badmorestackgsignal() { … } //go:nosplit func badctxt() { … } var gcrash … var crashingG … // Switch to crashstack and call fn, with special handling of // concurrent and recursive cases. // // Nosplit as it is called in a bad stack condition (we know // morestack would fail). // //go:nosplit //go:nowritebarrierrec func switchToCrashStack(fn func()) { … } const crashStackImplemented … //go:noescape func switchToCrashStack0(fn func()) func lockedOSThread() bool { … } var allglock … var allgs … var allglen … var allgptr … func allgadd(gp *g) { … } // allGsSnapshot returns a snapshot of the slice of all Gs. // // The world must be stopped or allglock must be held. func allGsSnapshot() []*g { … } // atomicAllG returns &allgs[0] and len(allgs) for use with atomicAllGIndex. func atomicAllG() (**g, uintptr) { … } // atomicAllGIndex returns ptr[i] with the allgptr returned from atomicAllG. func atomicAllGIndex(ptr **g, i uintptr) *g { … } // forEachG calls fn on every G from allgs. // // forEachG takes a lock to exclude concurrent addition of new Gs. func forEachG(fn func(gp *g)) { … } // forEachGRace calls fn on every G from allgs. // // forEachGRace avoids locking, but does not exclude addition of new Gs during // execution, which may be missed. func forEachGRace(fn func(gp *g)) { … } const _GoidCacheBatch … // cpuinit sets up CPU feature flags and calls internal/cpu.Initialize. env should be the complete // value of the GODEBUG environment variable. func cpuinit(env string) { … } // getGodebugEarly extracts the environment variable GODEBUG from the environment on // Unix-like operating systems and returns it. This function exists to extract GODEBUG // early before much of the runtime is initialized. func getGodebugEarly() string { … } // The bootstrap sequence is: // // call osinit // call schedinit // make & queue new G // call runtime·mstart // // The new G calls runtime·main. func schedinit() { … } func dumpgstatus(gp *g) { … } // sched.lock must be held. func checkmcount() { … } // mReserveID returns the next ID to use for a new m. This new m is immediately // considered 'running' by checkdead. // // sched.lock must be held. func mReserveID() int64 { … } // Pre-allocated ID may be passed as 'id', or omitted by passing -1. func mcommoninit(mp *m, id int64) { … } // mProfStackInit is used to eagerly initialize stack trace buffers for // profiling. Lazy allocation would have to deal with reentrancy issues in // malloc and runtime locks for mLockProfile. // TODO(mknyszek): Implement lazy allocation if this becomes a problem. func mProfStackInit(mp *m) { … } // makeProfStackFP creates a buffer large enough to hold a maximum-sized stack // trace as well as any additional frames needed for frame pointer unwinding // with delayed inline expansion. func makeProfStackFP() []uintptr { … } // makeProfStack returns a buffer large enough to hold a maximum-sized stack // trace. func makeProfStack() []uintptr { … } //go:linkname pprof_makeProfStack func pprof_makeProfStack() []uintptr { … } func (mp *m) becomeSpinning() { … } func (mp *m) hasCgoOnStack() bool { … } const osHasLowResTimer … const osHasLowResClockInt … const osHasLowResClock … // Mark gp ready to run. func ready(gp *g, traceskip int, next bool) { … } const freezeStopWait … var freezing … // Similar to stopTheWorld but best-effort and can be called several times. // There is no reverse operation, used during crashing. // This function must not lock any mutexes. func freezetheworld() { … } // All reads and writes of g's status go through readgstatus, casgstatus // castogscanstatus, casfrom_Gscanstatus. // //go:nosplit func readgstatus(gp *g) uint32 { … } // The Gscanstatuses are acting like locks and this releases them. // If it proves to be a performance hit we should be able to make these // simple atomic stores but for now we are going to throw if // we see an inconsistent state. func casfrom_Gscanstatus(gp *g, oldval, newval uint32) { … } // This will return false if the gp is not in the expected status and the cas fails. // This acts like a lock acquire while the casfromgstatus acts like a lock release. func castogscanstatus(gp *g, oldval, newval uint32) bool { … } var casgstatusAlwaysTrack … // If asked to move to or from a Gscanstatus this will throw. Use the castogscanstatus // and casfrom_Gscanstatus instead. // casgstatus will loop if the g->atomicstatus is in a Gscan status until the routine that // put it in the Gscan state is finished. // //go:nosplit func casgstatus(gp *g, oldval, newval uint32) { … } // casGToWaiting transitions gp from old to _Gwaiting, and sets the wait reason. // // Use this over casgstatus when possible to ensure that a waitreason is set. func casGToWaiting(gp *g, old uint32, reason waitReason) { … } // casGToWaitingForGC transitions gp from old to _Gwaiting, and sets the wait reason. // The wait reason must be a valid isWaitingForGC wait reason. // // Use this over casgstatus when possible to ensure that a waitreason is set. func casGToWaitingForGC(gp *g, old uint32, reason waitReason) { … } // casgstatus(gp, oldstatus, Gcopystack), assuming oldstatus is Gwaiting or Grunnable. // Returns old status. Cannot call casgstatus directly, because we are racing with an // async wakeup that might come in from netpoll. If we see Gwaiting from the readgstatus, // it might have become Grunnable by the time we get to the cas. If we called casgstatus, // it would loop waiting for the status to go back to Gwaiting, which it never will. // //go:nosplit func casgcopystack(gp *g) uint32 { … } // casGToPreemptScan transitions gp from _Grunning to _Gscan|_Gpreempted. // // TODO(austin): This is the only status operation that both changes // the status and locks the _Gscan bit. Rethink this. func casGToPreemptScan(gp *g, old, new uint32) { … } // casGFromPreempted attempts to transition gp from _Gpreempted to // _Gwaiting. If successful, the caller is responsible for // re-scheduling gp. func casGFromPreempted(gp *g, old, new uint32) bool { … } type stwReason … const stwUnknown … const stwGCMarkTerm … const stwGCSweepTerm … const stwWriteHeapDump … const stwGoroutineProfile … const stwGoroutineProfileCleanup … const stwAllGoroutinesStack … const stwReadMemStats … const stwAllThreadsSyscall … const stwGOMAXPROCS … const stwStartTrace … const stwStopTrace … const stwForTestCountPagesInUse … const stwForTestReadMetricsSlow … const stwForTestReadMemStatsSlow … const stwForTestPageCachePagesLeaked … const stwForTestResetDebugLog … func (r stwReason) String() string { … } func (r stwReason) isGC() bool { … } var stwReasonStrings … type worldStop … var stopTheWorldContext … // stopTheWorld stops all P's from executing goroutines, interrupting // all goroutines at GC safe points and records reason as the reason // for the stop. On return, only the current goroutine's P is running. // stopTheWorld must not be called from a system stack and the caller // must not hold worldsema. The caller must call startTheWorld when // other P's should resume execution. // // stopTheWorld is safe for multiple goroutines to call at the // same time. Each will execute its own stop, and the stops will // be serialized. // // This is also used by routines that do stack dumps. If the system is // in panic or being exited, this may not reliably stop all // goroutines. // // Returns the STW context. When starting the world, this context must be // passed to startTheWorld. func stopTheWorld(reason stwReason) worldStop { … } // startTheWorld undoes the effects of stopTheWorld. // // w must be the worldStop returned by stopTheWorld. func startTheWorld(w worldStop) { … } // stopTheWorldGC has the same effect as stopTheWorld, but blocks // until the GC is not running. It also blocks a GC from starting // until startTheWorldGC is called. func stopTheWorldGC(reason stwReason) worldStop { … } // startTheWorldGC undoes the effects of stopTheWorldGC. // // w must be the worldStop returned by stopTheWorld. func startTheWorldGC(w worldStop) { … } var worldsema … var gcsema … // stopTheWorldWithSema is the core implementation of stopTheWorld. // The caller is responsible for acquiring worldsema and disabling // preemption first and then should stopTheWorldWithSema on the system // stack: // // semacquire(&worldsema, 0) // m.preemptoff = "reason" // var stw worldStop // systemstack(func() { // stw = stopTheWorldWithSema(reason) // }) // // When finished, the caller must either call startTheWorld or undo // these three operations separately: // // m.preemptoff = "" // systemstack(func() { // now = startTheWorldWithSema(stw) // }) // semrelease(&worldsema) // // It is allowed to acquire worldsema once and then execute multiple // startTheWorldWithSema/stopTheWorldWithSema pairs. // Other P's are able to execute between successive calls to // startTheWorldWithSema and stopTheWorldWithSema. // Holding worldsema causes any other goroutines invoking // stopTheWorld to block. // // Returns the STW context. When starting the world, this context must be // passed to startTheWorldWithSema. func stopTheWorldWithSema(reason stwReason) worldStop { … } // reason is the same STW reason passed to stopTheWorld. start is the start // time returned by stopTheWorld. // // now is the current time; prefer to pass 0 to capture a fresh timestamp. // // stattTheWorldWithSema returns now. func startTheWorldWithSema(now int64, w worldStop) int64 { … } // usesLibcall indicates whether this runtime performs system calls // via libcall. func usesLibcall() bool { … } // mStackIsSystemAllocated indicates whether this runtime starts on a // system-allocated stack. func mStackIsSystemAllocated() bool { … } // mstart is the entry-point for new Ms. // It is written in assembly, uses ABI0, is marked TOPFRAME, and calls mstart0. func mstart() // mstart0 is the Go entry-point for new Ms. // This must not split the stack because we may not even have stack // bounds set up yet. // // May run during STW (because it doesn't have a P yet), so write // barriers are not allowed. // //go:nosplit //go:nowritebarrierrec func mstart0() { … } // The go:noinline is to guarantee the sys.GetCallerPC/sys.GetCallerSP below are safe, // so that we can set up g0.sched to return to the call of mstart1 above. // //go:noinline func mstart1() { … } // mstartm0 implements part of mstart1 that only runs on the m0. // // Write barriers are allowed here because we know the GC can't be // running yet, so they'll be no-ops. // //go:yeswritebarrierrec func mstartm0() { … } // mPark causes a thread to park itself, returning once woken. // //go:nosplit func mPark() { … } // mexit tears down and exits the current thread. // // Don't call this directly to exit the thread, since it must run at // the top of the thread stack. Instead, use gogo(&gp.m.g0.sched) to // unwind the stack to the point that exits the thread. // // It is entered with m.p != nil, so write barriers are allowed. It // will release the P before exiting. // //go:yeswritebarrierrec func mexit(osStack bool) { … } // forEachP calls fn(p) for every P p when p reaches a GC safe point. // If a P is currently executing code, this will bring the P to a GC // safe point and execute fn on that P. If the P is not executing code // (it is idle or in a syscall), this will call fn(p) directly while // preventing the P from exiting its state. This does not ensure that // fn will run on every CPU executing Go code, but it acts as a global // memory barrier. GC uses this as a "ragged barrier." // // The caller must hold worldsema. fn must not refer to any // part of the current goroutine's stack, since the GC may move it. func forEachP(reason waitReason, fn func(*p)) { … } // forEachPInternal calls fn(p) for every P p when p reaches a GC safe point. // It is the internal implementation of forEachP. // // The caller must hold worldsema and either must ensure that a GC is not // running (otherwise this may deadlock with the GC trying to preempt this P) // or it must leave its goroutine in a preemptible state before it switches // to the systemstack. Due to these restrictions, prefer forEachP when possible. // //go:systemstack func forEachPInternal(fn func(*p)) { … } // runSafePointFn runs the safe point function, if any, for this P. // This should be called like // // if getg().m.p.runSafePointFn != 0 { // runSafePointFn() // } // // runSafePointFn must be checked on any transition in to _Pidle or // _Psyscall to avoid a race where forEachP sees that the P is running // just before the P goes into _Pidle/_Psyscall and neither forEachP // nor the P run the safe-point function. func runSafePointFn() { … } var cgoThreadStart … type cgothreadstart … // Allocate a new m unassociated with any thread. // Can use p for allocation context if needed. // fn is recorded as the new m's m.mstartfn. // id is optional pre-allocated m ID. Omit by passing -1. // // This function is allowed to have write barriers even if the caller // isn't because it borrows pp. // //go:yeswritebarrierrec func allocm(pp *p, fn func(), id int64) *m { … } // needm is called when a cgo callback happens on a // thread without an m (a thread not created by Go). // In this case, needm is expected to find an m to use // and return with m, g initialized correctly. // Since m and g are not set now (likely nil, but see below) // needm is limited in what routines it can call. In particular // it can only call nosplit functions (textflag 7) and cannot // do any scheduling that requires an m. // // In order to avoid needing heavy lifting here, we adopt // the following strategy: there is a stack of available m's // that can be stolen. Using compare-and-swap // to pop from the stack has ABA races, so we simulate // a lock by doing an exchange (via Casuintptr) to steal the stack // head and replace the top pointer with MLOCKED (1). // This serves as a simple spin lock that we can use even // without an m. The thread that locks the stack in this way // unlocks the stack by storing a valid stack head pointer. // // In order to make sure that there is always an m structure // available to be stolen, we maintain the invariant that there // is always one more than needed. At the beginning of the // program (if cgo is in use) the list is seeded with a single m. // If needm finds that it has taken the last m off the list, its job // is - once it has installed its own m so that it can do things like // allocate memory - to create a spare m and put it on the list. // // Each of these extra m's also has a g0 and a curg that are // pressed into service as the scheduling stack and current // goroutine for the duration of the cgo callback. // // It calls dropm to put the m back on the list, // 1. when the callback is done with the m in non-pthread platforms, // 2. or when the C thread exiting on pthread platforms. // // The signal argument indicates whether we're called from a signal // handler. // //go:nosplit func needm(signal bool) { … } // Acquire an extra m and bind it to the C thread when a pthread key has been created. // //go:nosplit func needAndBindM() { … } // newextram allocates m's and puts them on the extra list. // It is called with a working local m, so that it can do things // like call schedlock and allocate. func newextram() { … } // oneNewExtraM allocates an m and puts it on the extra list. func oneNewExtraM() { … } // dropm puts the current m back onto the extra list. // // 1. On systems without pthreads, like Windows // dropm is called when a cgo callback has called needm but is now // done with the callback and returning back into the non-Go thread. // // The main expense here is the call to signalstack to release the // m's signal stack, and then the call to needm on the next callback // from this thread. It is tempting to try to save the m for next time, // which would eliminate both these costs, but there might not be // a next time: the current thread (which Go does not control) might exit. // If we saved the m for that thread, there would be an m leak each time // such a thread exited. Instead, we acquire and release an m on each // call. These should typically not be scheduling operations, just a few // atomics, so the cost should be small. // // 2. On systems with pthreads // dropm is called while a non-Go thread is exiting. // We allocate a pthread per-thread variable using pthread_key_create, // to register a thread-exit-time destructor. // And store the g into a thread-specific value associated with the pthread key, // when first return back to C. // So that the destructor would invoke dropm while the non-Go thread is exiting. // This is much faster since it avoids expensive signal-related syscalls. // // This always runs without a P, so //go:nowritebarrierrec is required. // // This may run with a different stack than was recorded in g0 (there is no // call to callbackUpdateSystemStack prior to dropm), so this must be // //go:nosplit to avoid the stack bounds check. // //go:nowritebarrierrec //go:nosplit func dropm() { … } // bindm store the g0 of the current m into a thread-specific value. // // We allocate a pthread per-thread variable using pthread_key_create, // to register a thread-exit-time destructor. // We are here setting the thread-specific value of the pthread key, to enable the destructor. // So that the pthread_key_destructor would dropm while the C thread is exiting. // // And the saved g will be used in pthread_key_destructor, // since the g stored in the TLS by Go might be cleared in some platforms, // before the destructor invoked, so, we restore g by the stored g, before dropm. // // We store g0 instead of m, to make the assembly code simpler, // since we need to restore g0 in runtime.cgocallback. // // On systems without pthreads, like Windows, bindm shouldn't be used. // // NOTE: this always runs without a P, so, nowritebarrierrec required. // //go:nosplit //go:nowritebarrierrec func cgoBindM() { … } // A helper function for EnsureDropM. // // getm should be an internal detail, // but widely used packages access it using linkname. // Notable members of the hall of shame include: // - fortio.org/log // // Do not remove or change the type signature. // See go.dev/issue/67401. // //go:linkname getm func getm() uintptr { … } var extraM … var extraMLength … var extraMWaiters … var extraMInUse … // lockextra locks the extra list and returns the list head. // The caller must unlock the list by storing a new list head // to extram. If nilokay is true, then lockextra will // return a nil list head if that's what it finds. If nilokay is false, // lockextra will keep waiting until the list head is no longer nil. // //go:nosplit func lockextra(nilokay bool) *m { … } //go:nosplit func unlockextra(mp *m, delta int32) { … } // Return an M from the extra M list. Returns last == true if the list becomes // empty because of this call. // // Spins waiting for an extra M, so caller must ensure that the list always // contains or will soon contain at least one M. // //go:nosplit func getExtraM() (mp *m, last bool) { … } // Returns an extra M back to the list. mp must be from getExtraM. Newly // allocated M's should use addExtraM. // //go:nosplit func putExtraM(mp *m) { … } // Adds a newly allocated M to the extra M list. // //go:nosplit func addExtraM(mp *m) { … } var allocmLock … var execLock … const failthreadcreate … const failallocatestack … var newmHandoff … // Create a new m. It will start off with a call to fn, or else the scheduler. // fn needs to be static and not a heap allocated closure. // May run with m.p==nil, so write barriers are not allowed. // // id is optional pre-allocated m ID. Omit by passing -1. // //go:nowritebarrierrec func newm(fn func(), pp *p, id int64) { … } func newm1(mp *m) { … } // startTemplateThread starts the template thread if it is not already // running. // // The calling thread must itself be in a known-good state. func startTemplateThread() { … } // templateThread is a thread in a known-good state that exists solely // to start new threads in known-good states when the calling thread // may not be in a good state. // // Many programs never need this, so templateThread is started lazily // when we first enter a state that might lead to running on a thread // in an unknown state. // // templateThread runs on an M without a P, so it must not have write // barriers. // //go:nowritebarrierrec func templateThread() { … } // Stops execution of the current m until new work is available. // Returns with acquired P. func stopm() { … } func mspinning() { … } // Schedules some M to run the p (creates an M if necessary). // If p==nil, tries to get an idle P, if no idle P's does nothing. // May run with m.p==nil, so write barriers are not allowed. // If spinning is set, the caller has incremented nmspinning and must provide a // P. startm will set m.spinning in the newly started M. // // Callers passing a non-nil P must call from a non-preemptible context. See // comment on acquirem below. // // Argument lockheld indicates whether the caller already acquired the // scheduler lock. Callers holding the lock when making the call must pass // true. The lock might be temporarily dropped, but will be reacquired before // returning. // // Must not have write barriers because this may be called without a P. // //go:nowritebarrierrec func startm(pp *p, spinning, lockheld bool) { … } // Hands off P from syscall or locked M. // Always runs without a P, so write barriers are not allowed. // //go:nowritebarrierrec func handoffp(pp *p) { … } // Tries to add one more P to execute G's. // Called when a G is made runnable (newproc, ready). // Must be called with a P. // // wakep should be an internal detail, // but widely used packages access it using linkname. // Notable members of the hall of shame include: // - gvisor.dev/gvisor // // Do not remove or change the type signature. // See go.dev/issue/67401. // //go:linkname wakep func wakep() { … } // Stops execution of the current m that is locked to a g until the g is runnable again. // Returns with acquired P. func stoplockedm() { … } // Schedules the locked m to run the locked gp. // May run during STW, so write barriers are not allowed. // //go:nowritebarrierrec func startlockedm(gp *g) { … } // Stops the current m for stopTheWorld. // Returns when the world is restarted. func gcstopm() { … } // Schedules gp to run on the current M. // If inheritTime is true, gp inherits the remaining time in the // current time slice. Otherwise, it starts a new time slice. // Never returns. // // Write barriers are allowed because this is called immediately after // acquiring a P in several places. // //go:yeswritebarrierrec func execute(gp *g, inheritTime bool) { … } // Finds a runnable goroutine to execute. // Tries to steal from other P's, get g from local or global queue, poll network. // tryWakeP indicates that the returned goroutine is not normal (GC worker, trace // reader) so the caller should try to wake a P. func findRunnable() (gp *g, inheritTime, tryWakeP bool) { … } // pollWork reports whether there is non-background work this P could // be doing. This is a fairly lightweight check to be used for // background work loops, like idle GC. It checks a subset of the // conditions checked by the actual scheduler. func pollWork() bool { … } // stealWork attempts to steal a runnable goroutine or timer from any P. // // If newWork is true, new work may have been readied. // // If now is not 0 it is the current time. stealWork returns the passed time or // the current time if now was passed as 0. func stealWork(now int64) (gp *g, inheritTime bool, rnow, pollUntil int64, newWork bool) { … } // Check all Ps for a runnable G to steal. // // On entry we have no P. If a G is available to steal and a P is available, // the P is returned which the caller should acquire and attempt to steal the // work to. func checkRunqsNoP(allpSnapshot []*p, idlepMaskSnapshot pMask) *p { … } // Check all Ps for a timer expiring sooner than pollUntil. // // Returns updated pollUntil value. func checkTimersNoP(allpSnapshot []*p, timerpMaskSnapshot pMask, pollUntil int64) int64 { … } // Check for idle-priority GC, without a P on entry. // // If some GC work, a P, and a worker G are all available, the P and G will be // returned. The returned P has not been wired yet. func checkIdleGCNoP() (*p, *g) { … } // wakeNetPoller wakes up the thread sleeping in the network poller if it isn't // going to wake up before the when argument; or it wakes an idle P to service // timers and the network poller if there isn't one already. func wakeNetPoller(when int64) { … } func resetspinning() { … } // injectglist adds each runnable G on the list to some run queue, // and clears glist. If there is no current P, they are added to the // global queue, and up to npidle M's are started to run them. // Otherwise, for each idle P, this adds a G to the global queue // and starts an M. Any remaining G's are added to the current P's // local run queue. // This may temporarily acquire sched.lock. // Can run concurrently with GC. func injectglist(glist *gList) { … } // One round of scheduler: find a runnable goroutine and execute it. // Never returns. func schedule() { … } // dropg removes the association between m and the current goroutine m->curg (gp for short). // Typically a caller sets gp's status away from Grunning and then // immediately calls dropg to finish the job. The caller is also responsible // for arranging that gp will be restarted using ready at an // appropriate time. After calling dropg and arranging for gp to be // readied later, the caller can do other work but eventually should // call schedule to restart the scheduling of goroutines on this m. func dropg() { … } func parkunlock_c(gp *g, lock unsafe.Pointer) bool { … } // park continuation on g0. func park_m(gp *g) { … } func goschedImpl(gp *g, preempted bool) { … } // Gosched continuation on g0. func gosched_m(gp *g) { … } // goschedguarded is a forbidden-states-avoided version of gosched_m. func goschedguarded_m(gp *g) { … } func gopreempt_m(gp *g) { … } // preemptPark parks gp and puts it in _Gpreempted. // //go:systemstack func preemptPark(gp *g) { … } // goyield is like Gosched, but it: // - emits a GoPreempt trace event instead of a GoSched trace event // - puts the current G on the runq of the current P instead of the globrunq // // goyield should be an internal detail, // but widely used packages access it using linkname. // Notable members of the hall of shame include: // - gvisor.dev/gvisor // - github.com/sagernet/gvisor // // Do not remove or change the type signature. // See go.dev/issue/67401. // //go:linkname goyield func goyield() { … } func goyield_m(gp *g) { … } // Finishes execution of the current goroutine. func goexit1() { … } // goexit continuation on g0. func goexit0(gp *g) { … } func gdestroy(gp *g) { … } // save updates getg().sched to refer to pc and sp so that a following // gogo will restore pc and sp. // // save must not have write barriers because invoking a write barrier // can clobber getg().sched. // //go:nosplit //go:nowritebarrierrec func save(pc, sp, bp uintptr) { … } // The goroutine g is about to enter a system call. // Record that it's not using the cpu anymore. // This is called only from the go syscall library and cgocall, // not from the low-level system calls used by the runtime. // // Entersyscall cannot split the stack: the save must // make g->sched refer to the caller's stack segment, because // entersyscall is going to return immediately after. // // Nothing entersyscall calls can split the stack either. // We cannot safely move the stack during an active call to syscall, // because we do not know which of the uintptr arguments are // really pointers (back into the stack). // In practice, this means that we make the fast path run through // entersyscall doing no-split things, and the slow path has to use systemstack // to run bigger things on the system stack. // // reentersyscall is the entry point used by cgo callbacks, where explicitly // saved SP and PC are restored. This is needed when exitsyscall will be called // from a function further up in the call stack than the parent, as g->syscallsp // must always point to a valid stack frame. entersyscall below is the normal // entry point for syscalls, which obtains the SP and PC from the caller. // //go:nosplit func reentersyscall(pc, sp, bp uintptr) { … } // Standard syscall entry used by the go syscall library and normal cgo calls. // // This is exported via linkname to assembly in the syscall package and x/sys. // // Other packages should not be accessing entersyscall directly, // but widely used packages access it using linkname. // Notable members of the hall of shame include: // - gvisor.dev/gvisor // // Do not remove or change the type signature. // See go.dev/issue/67401. // //go:nosplit //go:linkname entersyscall func entersyscall() { … } func entersyscall_sysmon() { … } func entersyscall_gcwait() { … } // entersyscallblock should be an internal detail, // but widely used packages access it using linkname. // Notable members of the hall of shame include: // - gvisor.dev/gvisor // // Do not remove or change the type signature. // See go.dev/issue/67401. // //go:linkname entersyscallblock //go:nosplit func entersyscallblock() { … } func entersyscallblock_handoff() { … } // The goroutine g exited its system call. // Arrange for it to run on a cpu again. // This is called only from the go syscall library, not // from the low-level system calls used by the runtime. // // Write barriers are not allowed because our P may have been stolen. // // This is exported via linkname to assembly in the syscall package. // // exitsyscall should be an internal detail, // but widely used packages access it using linkname. // Notable members of the hall of shame include: // - gvisor.dev/gvisor // // Do not remove or change the type signature. // See go.dev/issue/67401. // //go:nosplit //go:nowritebarrierrec //go:linkname exitsyscall func exitsyscall() { … } //go:nosplit func exitsyscallfast(oldp *p) bool { … } // exitsyscallfast_reacquired is the exitsyscall path on which this G // has successfully reacquired the P it was running on before the // syscall. // //go:nosplit func exitsyscallfast_reacquired(trace traceLocker) { … } func exitsyscallfast_pidle() bool { … } // exitsyscall slow path on g0. // Failed to acquire P, enqueue gp as runnable. // // Called via mcall, so gp is the calling g from this M. // //go:nowritebarrierrec func exitsyscall0(gp *g) { … } // Called from syscall package before fork. // // syscall_runtime_BeforeFork is for package syscall, // but widely used packages access it using linkname. // Notable members of the hall of shame include: // - gvisor.dev/gvisor // // Do not remove or change the type signature. // See go.dev/issue/67401. // //go:linkname syscall_runtime_BeforeFork syscall.runtime_BeforeFork //go:nosplit func syscall_runtime_BeforeFork() { … } // Called from syscall package after fork in parent. // // syscall_runtime_AfterFork is for package syscall, // but widely used packages access it using linkname. // Notable members of the hall of shame include: // - gvisor.dev/gvisor // // Do not remove or change the type signature. // See go.dev/issue/67401. // //go:linkname syscall_runtime_AfterFork syscall.runtime_AfterFork //go:nosplit func syscall_runtime_AfterFork() { … } var inForkedChild … // Called from syscall package after fork in child. // It resets non-sigignored signals to the default handler, and // restores the signal mask in preparation for the exec. // // Because this might be called during a vfork, and therefore may be // temporarily sharing address space with the parent process, this must // not change any global variables or calling into C code that may do so. // // syscall_runtime_AfterForkInChild is for package syscall, // but widely used packages access it using linkname. // Notable members of the hall of shame include: // - gvisor.dev/gvisor // // Do not remove or change the type signature. // See go.dev/issue/67401. // //go:linkname syscall_runtime_AfterForkInChild syscall.runtime_AfterForkInChild //go:nosplit //go:nowritebarrierrec func syscall_runtime_AfterForkInChild() { … } var pendingPreemptSignals … // Called from syscall package before Exec. // //go:linkname syscall_runtime_BeforeExec syscall.runtime_BeforeExec func syscall_runtime_BeforeExec() { … } // Called from syscall package after Exec. // //go:linkname syscall_runtime_AfterExec syscall.runtime_AfterExec func syscall_runtime_AfterExec() { … } // Allocate a new g, with a stack big enough for stacksize bytes. func malg(stacksize int32) *g { … } // Create a new g running fn. // Put it on the queue of g's waiting to run. // The compiler turns a go statement into a call to this. func newproc(fn *funcval) { … } // Create a new g in state _Grunnable (or _Gwaiting if parked is true), starting at fn. // callerpc is the address of the go statement that created this. The caller is responsible // for adding the new g to the scheduler. If parked is true, waitreason must be non-zero. func newproc1(fn *funcval, callergp *g, callerpc uintptr, parked bool, waitreason waitReason) *g { … } // saveAncestors copies previous ancestors of the given caller g and // includes info for the current caller into a new set of tracebacks for // a g being created. func saveAncestors(callergp *g) *[]ancestorInfo { … } // Put on gfree list. // If local list is too long, transfer a batch to the global list. func gfput(pp *p, gp *g) { … } // Get from gfree list. // If local list is empty, grab a batch from global list. func gfget(pp *p) *g { … } // Purge all cached G's from gfree list to the global list. func gfpurge(pp *p) { … } // Breakpoint executes a breakpoint trap. func Breakpoint() { … } // dolockOSThread is called by LockOSThread and lockOSThread below // after they modify m.locked. Do not allow preemption during this call, // or else the m might be different in this function than in the caller. // //go:nosplit func dolockOSThread() { … } // LockOSThread wires the calling goroutine to its current operating system thread. // The calling goroutine will always execute in that thread, // and no other goroutine will execute in it, // until the calling goroutine has made as many calls to // [UnlockOSThread] as to LockOSThread. // If the calling goroutine exits without unlocking the thread, // the thread will be terminated. // // All init functions are run on the startup thread. Calling LockOSThread // from an init function will cause the main function to be invoked on // that thread. // // A goroutine should call LockOSThread before calling OS services or // non-Go library functions that depend on per-thread state. // //go:nosplit func LockOSThread() { … } //go:nosplit func lockOSThread() { … } // dounlockOSThread is called by UnlockOSThread and unlockOSThread below // after they update m->locked. Do not allow preemption during this call, // or else the m might be in different in this function than in the caller. // //go:nosplit func dounlockOSThread() { … } // UnlockOSThread undoes an earlier call to LockOSThread. // If this drops the number of active LockOSThread calls on the // calling goroutine to zero, it unwires the calling goroutine from // its fixed operating system thread. // If there are no active LockOSThread calls, this is a no-op. // // Before calling UnlockOSThread, the caller must ensure that the OS // thread is suitable for running other goroutines. If the caller made // any permanent changes to the state of the thread that would affect // other goroutines, it should not call this function and thus leave // the goroutine locked to the OS thread until the goroutine (and // hence the thread) exits. // //go:nosplit func UnlockOSThread() { … } //go:nosplit func unlockOSThread() { … } func badunlockosthread() { … } func gcount() int32 { … } func mcount() int32 { … } var prof … func _System() { … } func _ExternalCode() { … } func _LostExternalCode() { … } func _GC() { … } func _LostSIGPROFDuringAtomic64() { … } func _LostContendedRuntimeLock() { … } func _VDSO() { … } // Called if we receive a SIGPROF signal. // Called by the signal handler, may run during STW. // //go:nowritebarrierrec func sigprof(pc, sp, lr uintptr, gp *g, mp *m) { … } // setcpuprofilerate sets the CPU profiling rate to hz times per second. // If hz <= 0, setcpuprofilerate turns off CPU profiling. func setcpuprofilerate(hz int32) { … } // init initializes pp, which may be a freshly allocated p or a // previously destroyed p, and transitions it to status _Pgcstop. func (pp *p) init(id int32) { … } // destroy releases all of the resources associated with pp and // transitions it to status _Pdead. // // sched.lock must be held and the world must be stopped. func (pp *p) destroy() { … } // Change number of processors. // // sched.lock must be held, and the world must be stopped. // // gcworkbufs must not be being modified by either the GC or the write barrier // code, so the GC must not be running if the number of Ps actually changes. // // Returns list of Ps with local work, they need to be scheduled by the caller. func procresize(nprocs int32) *p { … } // Associate p and the current m. // // This function is allowed to have write barriers even if the caller // isn't because it immediately acquires pp. // //go:yeswritebarrierrec func acquirep(pp *p) { … } // wirep is the first step of acquirep, which actually associates the // current M to pp. This is broken out so we can disallow write // barriers for this part, since we don't yet have a P. // //go:nowritebarrierrec //go:nosplit func wirep(pp *p) { … } // Disassociate p and the current m. func releasep() *p { … } // Disassociate p and the current m without tracing an event. func releasepNoTrace() *p { … } func incidlelocked(v int32) { … } // Check for deadlock situation. // The check is based on number of running M's, if 0 -> deadlock. // sched.lock must be held. func checkdead() { … } var forcegcperiod … var needSysmonWorkaround … const haveSysmon … // Always runs without a P, so write barriers are not allowed. // //go:nowritebarrierrec func sysmon() { … } type sysmontick … const forcePreemptNS … func retake(now int64) uint32 { … } // Tell all goroutines that they have been preempted and they should stop. // This function is purely best-effort. It can fail to inform a goroutine if a // processor just started running it. // No locks need to be held. // Returns true if preemption request was issued to at least one goroutine. func preemptall() bool { … } // Tell the goroutine running on processor P to stop. // This function is purely best-effort. It can incorrectly fail to inform the // goroutine. It can inform the wrong goroutine. Even if it informs the // correct goroutine, that goroutine might ignore the request if it is // simultaneously executing newstack. // No lock needs to be held. // Returns true if preemption request was issued. // The actual preemption will happen at some point in the future // and will be indicated by the gp->status no longer being // Grunning func preemptone(pp *p) bool { … } var starttime … func schedtrace(detailed bool) { … } // schedEnableUser enables or disables the scheduling of user // goroutines. // // This does not stop already running user goroutines, so the caller // should first stop the world when disabling user goroutines. func schedEnableUser(enable bool) { … } // schedEnabled reports whether gp should be scheduled. It returns // false is scheduling of gp is disabled. // // sched.lock must be held. func schedEnabled(gp *g) bool { … } // Put mp on midle list. // sched.lock must be held. // May run during STW, so write barriers are not allowed. // //go:nowritebarrierrec func mput(mp *m) { … } // Try to get an m from midle list. // sched.lock must be held. // May run during STW, so write barriers are not allowed. // //go:nowritebarrierrec func mget() *m { … } // Put gp on the global runnable queue. // sched.lock must be held. // May run during STW, so write barriers are not allowed. // //go:nowritebarrierrec func globrunqput(gp *g) { … } // Put gp at the head of the global runnable queue. // sched.lock must be held. // May run during STW, so write barriers are not allowed. // //go:nowritebarrierrec func globrunqputhead(gp *g) { … } // Put a batch of runnable goroutines on the global runnable queue. // This clears *batch. // sched.lock must be held. // May run during STW, so write barriers are not allowed. // //go:nowritebarrierrec func globrunqputbatch(batch *gQueue, n int32) { … } // Try get a batch of G's from the global runnable queue. // sched.lock must be held. func globrunqget(pp *p, max int32) *g { … } type pMask … // read returns true if P id's bit is set. func (p pMask) read(id uint32) bool { … } // set sets P id's bit. func (p pMask) set(id int32) { … } // clear clears P id's bit. func (p pMask) clear(id int32) { … } // pidleput puts p on the _Pidle list. now must be a relatively recent call // to nanotime or zero. Returns now or the current time if now was zero. // // This releases ownership of p. Once sched.lock is released it is no longer // safe to use p. // // sched.lock must be held. // // May run during STW, so write barriers are not allowed. // //go:nowritebarrierrec func pidleput(pp *p, now int64) int64 { … } // pidleget tries to get a p from the _Pidle list, acquiring ownership. // // sched.lock must be held. // // May run during STW, so write barriers are not allowed. // //go:nowritebarrierrec func pidleget(now int64) (*p, int64) { … } // pidlegetSpinning tries to get a p from the _Pidle list, acquiring ownership. // This is called by spinning Ms (or callers than need a spinning M) that have // found work. If no P is available, this must synchronized with non-spinning // Ms that may be preparing to drop their P without discovering this work. // // sched.lock must be held. // // May run during STW, so write barriers are not allowed. // //go:nowritebarrierrec func pidlegetSpinning(now int64) (*p, int64) { … } // runqempty reports whether pp has no Gs on its local run queue. // It never returns true spuriously. func runqempty(pp *p) bool { … } const randomizeScheduler … // runqput tries to put g on the local runnable queue. // If next is false, runqput adds g to the tail of the runnable queue. // If next is true, runqput puts g in the pp.runnext slot. // If the run queue is full, runnext puts g on the global queue. // Executed only by the owner P. func runqput(pp *p, gp *g, next bool) { … } // Put g and a batch of work from local runnable queue on global queue. // Executed only by the owner P. func runqputslow(pp *p, gp *g, h, t uint32) bool { … } // runqputbatch tries to put all the G's on q on the local runnable queue. // If the queue is full, they are put on the global queue; in that case // this will temporarily acquire the scheduler lock. // Executed only by the owner P. func runqputbatch(pp *p, q *gQueue, qsize int) { … } // Get g from local runnable queue. // If inheritTime is true, gp should inherit the remaining time in the // current time slice. Otherwise, it should start a new time slice. // Executed only by the owner P. func runqget(pp *p) (gp *g, inheritTime bool) { … } // runqdrain drains the local runnable queue of pp and returns all goroutines in it. // Executed only by the owner P. func runqdrain(pp *p) (drainQ gQueue, n uint32) { … } // Grabs a batch of goroutines from pp's runnable queue into batch. // Batch is a ring buffer starting at batchHead. // Returns number of grabbed goroutines. // Can be executed by any P. func runqgrab(pp *p, batch *[256]guintptr, batchHead uint32, stealRunNextG bool) uint32 { … } // Steal half of elements from local runnable queue of p2 // and put onto local runnable queue of p. // Returns one of the stolen elements (or nil if failed). func runqsteal(pp, p2 *p, stealRunNextG bool) *g { … } type gQueue … // empty reports whether q is empty. func (q *gQueue) empty() bool { … } // push adds gp to the head of q. func (q *gQueue) push(gp *g) { … } // pushBack adds gp to the tail of q. func (q *gQueue) pushBack(gp *g) { … } // pushBackAll adds all Gs in q2 to the tail of q. After this q2 must // not be used. func (q *gQueue) pushBackAll(q2 gQueue) { … } // pop removes and returns the head of queue q. It returns nil if // q is empty. func (q *gQueue) pop() *g { … } // popList takes all Gs in q and returns them as a gList. func (q *gQueue) popList() gList { … } type gList … // empty reports whether l is empty. func (l *gList) empty() bool { … } // push adds gp to the head of l. func (l *gList) push(gp *g) { … } // pushAll prepends all Gs in q to l. func (l *gList) pushAll(q gQueue) { … } // pop removes and returns the head of l. If l is empty, it returns nil. func (l *gList) pop() *g { … } //go:linkname setMaxThreads runtime/debug.setMaxThreads func setMaxThreads(in int) (out int) { … } // procPin should be an internal detail, // but widely used packages access it using linkname. // Notable members of the hall of shame include: // - github.com/bytedance/gopkg // - github.com/choleraehyq/pid // - github.com/songzhibin97/gkit // // Do not remove or change the type signature. // See go.dev/issue/67401. // //go:linkname procPin //go:nosplit func procPin() int { … } // procUnpin should be an internal detail, // but widely used packages access it using linkname. // Notable members of the hall of shame include: // - github.com/bytedance/gopkg // - github.com/choleraehyq/pid // - github.com/songzhibin97/gkit // // Do not remove or change the type signature. // See go.dev/issue/67401. // //go:linkname procUnpin //go:nosplit func procUnpin() { … } //go:linkname sync_runtime_procPin sync.runtime_procPin //go:nosplit func sync_runtime_procPin() int { … } //go:linkname sync_runtime_procUnpin sync.runtime_procUnpin //go:nosplit func sync_runtime_procUnpin() { … } //go:linkname sync_atomic_runtime_procPin sync/atomic.runtime_procPin //go:nosplit func sync_atomic_runtime_procPin() int { … } //go:linkname sync_atomic_runtime_procUnpin sync/atomic.runtime_procUnpin //go:nosplit func sync_atomic_runtime_procUnpin() { … } // Active spinning for sync.Mutex. // // sync_runtime_canSpin should be an internal detail, // but widely used packages access it using linkname. // Notable members of the hall of shame include: // - github.com/livekit/protocol // - github.com/sagernet/gvisor // - gvisor.dev/gvisor // // Do not remove or change the type signature. // See go.dev/issue/67401. // //go:linkname sync_runtime_canSpin sync.runtime_canSpin //go:nosplit func sync_runtime_canSpin(i int) bool { … } // sync_runtime_doSpin should be an internal detail, // but widely used packages access it using linkname. // Notable members of the hall of shame include: // - github.com/livekit/protocol // - github.com/sagernet/gvisor // - gvisor.dev/gvisor // // Do not remove or change the type signature. // See go.dev/issue/67401. // //go:linkname sync_runtime_doSpin sync.runtime_doSpin //go:nosplit func sync_runtime_doSpin() { … } var stealOrder … type randomOrder … type randomEnum … func (ord *randomOrder) reset(count uint32) { … } func (ord *randomOrder) start(i uint32) randomEnum { … } func (enum *randomEnum) done() bool { … } func (enum *randomEnum) next() { … } func (enum *randomEnum) position() uint32 { … } func gcd(a, b uint32) uint32 { … } type initTask … var inittrace … type tracestat … func doInit(ts []*initTask) { … } func doInit1(t *initTask) { … }
Codebase Browser
go