Source file src/runtime/mgcsweep.go

Documentation: runtime

     1  // Copyright 2009 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Garbage collector: sweeping
     6  
     7  // The sweeper consists of two different algorithms:
     8  //
     9  // * The object reclaimer finds and frees unmarked slots in spans. It
    10  //   can free a whole span if none of the objects are marked, but that
    11  //   isn't its goal. This can be driven either synchronously by
    12  //   mcentral.cacheSpan for mcentral spans, or asynchronously by
    13  //   sweepone, which looks at all the mcentral lists.
    14  //
    15  // * The span reclaimer looks for spans that contain no marked objects
    16  //   and frees whole spans. This is a separate algorithm because
    17  //   freeing whole spans is the hardest task for the object reclaimer,
    18  //   but is critical when allocating new spans. The entry point for
    19  //   this is mheap_.reclaim and it's driven by a sequential scan of
    20  //   the page marks bitmap in the heap arenas.
    21  //
    22  // Both algorithms ultimately call mspan.sweep, which sweeps a single
    23  // heap span.
    24  
    25  package runtime
    26  
    27  import (
    28  	"runtime/internal/atomic"
    29  	"unsafe"
    30  )
    31  
    32  var sweep sweepdata
    33  
    34  // State of background sweep.
    35  type sweepdata struct {
    36  	lock    mutex
    37  	g       *g
    38  	parked  bool
    39  	started bool
    40  
    41  	nbgsweep    uint32
    42  	npausesweep uint32
    43  
    44  	// centralIndex is the current unswept span class.
    45  	// It represents an index into the mcentral span
    46  	// sets. Accessed and updated via its load and
    47  	// update methods. Not protected by a lock.
    48  	//
    49  	// Reset at mark termination.
    50  	// Used by mheap.nextSpanForSweep.
    51  	centralIndex sweepClass
    52  }
    53  
    54  // sweepClass is a spanClass and one bit to represent whether we're currently
    55  // sweeping partial or full spans.
    56  type sweepClass uint32
    57  
    58  const (
    59  	numSweepClasses            = numSpanClasses * 2
    60  	sweepClassDone  sweepClass = sweepClass(^uint32(0))
    61  )
    62  
    63  func (s *sweepClass) load() sweepClass {
    64  	return sweepClass(atomic.Load((*uint32)(s)))
    65  }
    66  
    67  func (s *sweepClass) update(sNew sweepClass) {
    68  	// Only update *s if its current value is less than sNew,
    69  	// since *s increases monotonically.
    70  	sOld := s.load()
    71  	for sOld < sNew && !atomic.Cas((*uint32)(s), uint32(sOld), uint32(sNew)) {
    72  		sOld = s.load()
    73  	}
    74  	// TODO(mknyszek): This isn't the only place we have
    75  	// an atomic monotonically increasing counter. It would
    76  	// be nice to have an "atomic max" which is just implemented
    77  	// as the above on most architectures. Some architectures
    78  	// like RISC-V however have native support for an atomic max.
    79  }
    80  
    81  func (s *sweepClass) clear() {
    82  	atomic.Store((*uint32)(s), 0)
    83  }
    84  
    85  // split returns the underlying span class as well as
    86  // whether we're interested in the full or partial
    87  // unswept lists for that class, indicated as a boolean
    88  // (true means "full").
    89  func (s sweepClass) split() (spc spanClass, full bool) {
    90  	return spanClass(s >> 1), s&1 == 0
    91  }
    92  
    93  // nextSpanForSweep finds and pops the next span for sweeping from the
    94  // central sweep buffers. It returns ownership of the span to the caller.
    95  // Returns nil if no such span exists.
    96  func (h *mheap) nextSpanForSweep() *mspan {
    97  	sg := h.sweepgen
    98  	for sc := sweep.centralIndex.load(); sc < numSweepClasses; sc++ {
    99  		spc, full := sc.split()
   100  		c := &h.central[spc].mcentral
   101  		var s *mspan
   102  		if full {
   103  			s = c.fullUnswept(sg).pop()
   104  		} else {
   105  			s = c.partialUnswept(sg).pop()
   106  		}
   107  		if s != nil {
   108  			// Write down that we found something so future sweepers
   109  			// can start from here.
   110  			sweep.centralIndex.update(sc)
   111  			return s
   112  		}
   113  	}
   114  	// Write down that we found nothing.
   115  	sweep.centralIndex.update(sweepClassDone)
   116  	return nil
   117  }
   118  
   119  // finishsweep_m ensures that all spans are swept.
   120  //
   121  // The world must be stopped. This ensures there are no sweeps in
   122  // progress.
   123  //
   124  //go:nowritebarrier
   125  func finishsweep_m() {
   126  	assertWorldStopped()
   127  
   128  	// Sweeping must be complete before marking commences, so
   129  	// sweep any unswept spans. If this is a concurrent GC, there
   130  	// shouldn't be any spans left to sweep, so this should finish
   131  	// instantly. If GC was forced before the concurrent sweep
   132  	// finished, there may be spans to sweep.
   133  	for sweepone() != ^uintptr(0) {
   134  		sweep.npausesweep++
   135  	}
   136  
   137  	// Reset all the unswept buffers, which should be empty.
   138  	// Do this in sweep termination as opposed to mark termination
   139  	// so that we can catch unswept spans and reclaim blocks as
   140  	// soon as possible.
   141  	sg := mheap_.sweepgen
   142  	for i := range mheap_.central {
   143  		c := &mheap_.central[i].mcentral
   144  		c.partialUnswept(sg).reset()
   145  		c.fullUnswept(sg).reset()
   146  	}
   147  
   148  	// Sweeping is done, so if the scavenger isn't already awake,
   149  	// wake it up. There's definitely work for it to do at this
   150  	// point.
   151  	wakeScavenger()
   152  
   153  	nextMarkBitArenaEpoch()
   154  }
   155  
   156  func bgsweep() {
   157  	sweep.g = getg()
   158  
   159  	lockInit(&sweep.lock, lockRankSweep)
   160  	lock(&sweep.lock)
   161  	sweep.parked = true
   162  	gcenable_setup <- 1
   163  	goparkunlock(&sweep.lock, waitReasonGCSweepWait, traceEvGoBlock, 1)
   164  
   165  	for {
   166  		for sweepone() != ^uintptr(0) {
   167  			sweep.nbgsweep++
   168  			Gosched()
   169  		}
   170  		for freeSomeWbufs(true) {
   171  			Gosched()
   172  		}
   173  		lock(&sweep.lock)
   174  		if !isSweepDone() {
   175  			// This can happen if a GC runs between
   176  			// gosweepone returning ^0 above
   177  			// and the lock being acquired.
   178  			unlock(&sweep.lock)
   179  			continue
   180  		}
   181  		sweep.parked = true
   182  		goparkunlock(&sweep.lock, waitReasonGCSweepWait, traceEvGoBlock, 1)
   183  	}
   184  }
   185  
   186  // sweepLocker acquires sweep ownership of spans and blocks sweep
   187  // completion.
   188  type sweepLocker struct {
   189  	// sweepGen is the sweep generation of the heap.
   190  	sweepGen uint32
   191  	// blocking indicates that this tracker is blocking sweep
   192  	// completion, usually as a result of acquiring sweep
   193  	// ownership of at least one span.
   194  	blocking bool
   195  }
   196  
   197  // sweepLocked represents sweep ownership of a span.
   198  type sweepLocked struct {
   199  	*mspan
   200  }
   201  
   202  func newSweepLocker() sweepLocker {
   203  	return sweepLocker{
   204  		sweepGen: mheap_.sweepgen,
   205  	}
   206  }
   207  
   208  // tryAcquire attempts to acquire sweep ownership of span s. If it
   209  // successfully acquires ownership, it blocks sweep completion.
   210  func (l *sweepLocker) tryAcquire(s *mspan) (sweepLocked, bool) {
   211  	// Check before attempting to CAS.
   212  	if atomic.Load(&s.sweepgen) != l.sweepGen-2 {
   213  		return sweepLocked{}, false
   214  	}
   215  	// Add ourselves to sweepers before potentially taking
   216  	// ownership.
   217  	l.blockCompletion()
   218  	// Attempt to acquire sweep ownership of s.
   219  	if !atomic.Cas(&s.sweepgen, l.sweepGen-2, l.sweepGen-1) {
   220  		return sweepLocked{}, false
   221  	}
   222  	return sweepLocked{s}, true
   223  }
   224  
   225  // blockCompletion blocks sweep completion without acquiring any
   226  // specific spans.
   227  func (l *sweepLocker) blockCompletion() {
   228  	if !l.blocking {
   229  		atomic.Xadd(&mheap_.sweepers, +1)
   230  		l.blocking = true
   231  	}
   232  }
   233  
   234  func (l *sweepLocker) dispose() {
   235  	if !l.blocking {
   236  		return
   237  	}
   238  	// Decrement the number of active sweepers and if this is the
   239  	// last one, mark sweep as complete.
   240  	l.blocking = false
   241  	if atomic.Xadd(&mheap_.sweepers, -1) == 0 && atomic.Load(&mheap_.sweepDrained) != 0 {
   242  		l.sweepIsDone()
   243  	}
   244  }
   245  
   246  func (l *sweepLocker) sweepIsDone() {
   247  	if debug.gcpacertrace > 0 {
   248  		print("pacer: sweep done at heap size ", gcController.heapLive>>20, "MB; allocated ", (gcController.heapLive-mheap_.sweepHeapLiveBasis)>>20, "MB during sweep; swept ", mheap_.pagesSwept, " pages at ", mheap_.sweepPagesPerByte, " pages/byte\n")
   249  	}
   250  }
   251  
   252  // sweepone sweeps some unswept heap span and returns the number of pages returned
   253  // to the heap, or ^uintptr(0) if there was nothing to sweep.
   254  func sweepone() uintptr {
   255  	_g_ := getg()
   256  
   257  	// increment locks to ensure that the goroutine is not preempted
   258  	// in the middle of sweep thus leaving the span in an inconsistent state for next GC
   259  	_g_.m.locks++
   260  	if atomic.Load(&mheap_.sweepDrained) != 0 {
   261  		_g_.m.locks--
   262  		return ^uintptr(0)
   263  	}
   264  	// TODO(austin): sweepone is almost always called in a loop;
   265  	// lift the sweepLocker into its callers.
   266  	sl := newSweepLocker()
   267  
   268  	// Find a span to sweep.
   269  	npages := ^uintptr(0)
   270  	var noMoreWork bool
   271  	for {
   272  		s := mheap_.nextSpanForSweep()
   273  		if s == nil {
   274  			noMoreWork = atomic.Cas(&mheap_.sweepDrained, 0, 1)
   275  			break
   276  		}
   277  		if state := s.state.get(); state != mSpanInUse {
   278  			// This can happen if direct sweeping already
   279  			// swept this span, but in that case the sweep
   280  			// generation should always be up-to-date.
   281  			if !(s.sweepgen == sl.sweepGen || s.sweepgen == sl.sweepGen+3) {
   282  				print("runtime: bad span s.state=", state, " s.sweepgen=", s.sweepgen, " sweepgen=", sl.sweepGen, "\n")
   283  				throw("non in-use span in unswept list")
   284  			}
   285  			continue
   286  		}
   287  		if s, ok := sl.tryAcquire(s); ok {
   288  			// Sweep the span we found.
   289  			npages = s.npages
   290  			if s.sweep(false) {
   291  				// Whole span was freed. Count it toward the
   292  				// page reclaimer credit since these pages can
   293  				// now be used for span allocation.
   294  				atomic.Xadduintptr(&mheap_.reclaimCredit, npages)
   295  			} else {
   296  				// Span is still in-use, so this returned no
   297  				// pages to the heap and the span needs to
   298  				// move to the swept in-use list.
   299  				npages = 0
   300  			}
   301  			break
   302  		}
   303  	}
   304  
   305  	sl.dispose()
   306  
   307  	if noMoreWork {
   308  		// The sweep list is empty. There may still be
   309  		// concurrent sweeps running, but we're at least very
   310  		// close to done sweeping.
   311  
   312  		// Move the scavenge gen forward (signalling
   313  		// that there's new work to do) and wake the scavenger.
   314  		//
   315  		// The scavenger is signaled by the last sweeper because once
   316  		// sweeping is done, we will definitely have useful work for
   317  		// the scavenger to do, since the scavenger only runs over the
   318  		// heap once per GC cyle. This update is not done during sweep
   319  		// termination because in some cases there may be a long delay
   320  		// between sweep done and sweep termination (e.g. not enough
   321  		// allocations to trigger a GC) which would be nice to fill in
   322  		// with scavenging work.
   323  		systemstack(func() {
   324  			lock(&mheap_.lock)
   325  			mheap_.pages.scavengeStartGen()
   326  			unlock(&mheap_.lock)
   327  		})
   328  		// Since we might sweep in an allocation path, it's not possible
   329  		// for us to wake the scavenger directly via wakeScavenger, since
   330  		// it could allocate. Ask sysmon to do it for us instead.
   331  		readyForScavenger()
   332  	}
   333  
   334  	_g_.m.locks--
   335  	return npages
   336  }
   337  
   338  // isSweepDone reports whether all spans are swept.
   339  //
   340  // Note that this condition may transition from false to true at any
   341  // time as the sweeper runs. It may transition from true to false if a
   342  // GC runs; to prevent that the caller must be non-preemptible or must
   343  // somehow block GC progress.
   344  func isSweepDone() bool {
   345  	// Check that all spans have at least begun sweeping and there
   346  	// are no active sweepers. If both are true, then all spans
   347  	// have finished sweeping.
   348  	return atomic.Load(&mheap_.sweepDrained) != 0 && atomic.Load(&mheap_.sweepers) == 0
   349  }
   350  
   351  // Returns only when span s has been swept.
   352  //go:nowritebarrier
   353  func (s *mspan) ensureSwept() {
   354  	// Caller must disable preemption.
   355  	// Otherwise when this function returns the span can become unswept again
   356  	// (if GC is triggered on another goroutine).
   357  	_g_ := getg()
   358  	if _g_.m.locks == 0 && _g_.m.mallocing == 0 && _g_ != _g_.m.g0 {
   359  		throw("mspan.ensureSwept: m is not locked")
   360  	}
   361  
   362  	sl := newSweepLocker()
   363  	// The caller must be sure that the span is a mSpanInUse span.
   364  	if s, ok := sl.tryAcquire(s); ok {
   365  		s.sweep(false)
   366  		sl.dispose()
   367  		return
   368  	}
   369  	sl.dispose()
   370  
   371  	// unfortunate condition, and we don't have efficient means to wait
   372  	for {
   373  		spangen := atomic.Load(&s.sweepgen)
   374  		if spangen == sl.sweepGen || spangen == sl.sweepGen+3 {
   375  			break
   376  		}
   377  		osyield()
   378  	}
   379  }
   380  
   381  // Sweep frees or collects finalizers for blocks not marked in the mark phase.
   382  // It clears the mark bits in preparation for the next GC round.
   383  // Returns true if the span was returned to heap.
   384  // If preserve=true, don't return it to heap nor relink in mcentral lists;
   385  // caller takes care of it.
   386  func (sl *sweepLocked) sweep(preserve bool) bool {
   387  	// It's critical that we enter this function with preemption disabled,
   388  	// GC must not start while we are in the middle of this function.
   389  	_g_ := getg()
   390  	if _g_.m.locks == 0 && _g_.m.mallocing == 0 && _g_ != _g_.m.g0 {
   391  		throw("mspan.sweep: m is not locked")
   392  	}
   393  
   394  	s := sl.mspan
   395  	if !preserve {
   396  		// We'll release ownership of this span. Nil it out to
   397  		// prevent the caller from accidentally using it.
   398  		sl.mspan = nil
   399  	}
   400  
   401  	sweepgen := mheap_.sweepgen
   402  	if state := s.state.get(); state != mSpanInUse || s.sweepgen != sweepgen-1 {
   403  		print("mspan.sweep: state=", state, " sweepgen=", s.sweepgen, " mheap.sweepgen=", sweepgen, "\n")
   404  		throw("mspan.sweep: bad span state")
   405  	}
   406  
   407  	if trace.enabled {
   408  		traceGCSweepSpan(s.npages * _PageSize)
   409  	}
   410  
   411  	atomic.Xadd64(&mheap_.pagesSwept, int64(s.npages))
   412  
   413  	spc := s.spanclass
   414  	size := s.elemsize
   415  
   416  	// The allocBits indicate which unmarked objects don't need to be
   417  	// processed since they were free at the end of the last GC cycle
   418  	// and were not allocated since then.
   419  	// If the allocBits index is >= s.freeindex and the bit
   420  	// is not marked then the object remains unallocated
   421  	// since the last GC.
   422  	// This situation is analogous to being on a freelist.
   423  
   424  	// Unlink & free special records for any objects we're about to free.
   425  	// Two complications here:
   426  	// 1. An object can have both finalizer and profile special records.
   427  	//    In such case we need to queue finalizer for execution,
   428  	//    mark the object as live and preserve the profile special.
   429  	// 2. A tiny object can have several finalizers setup for different offsets.
   430  	//    If such object is not marked, we need to queue all finalizers at once.
   431  	// Both 1 and 2 are possible at the same time.
   432  	hadSpecials := s.specials != nil
   433  	siter := newSpecialsIter(s)
   434  	for siter.valid() {
   435  		// A finalizer can be set for an inner byte of an object, find object beginning.
   436  		objIndex := uintptr(siter.s.offset) / size
   437  		p := s.base() + objIndex*size
   438  		mbits := s.markBitsForIndex(objIndex)
   439  		if !mbits.isMarked() {
   440  			// This object is not marked and has at least one special record.
   441  			// Pass 1: see if it has at least one finalizer.
   442  			hasFin := false
   443  			endOffset := p - s.base() + size
   444  			for tmp := siter.s; tmp != nil && uintptr(tmp.offset) < endOffset; tmp = tmp.next {
   445  				if tmp.kind == _KindSpecialFinalizer {
   446  					// Stop freeing of object if it has a finalizer.
   447  					mbits.setMarkedNonAtomic()
   448  					hasFin = true
   449  					break
   450  				}
   451  			}
   452  			// Pass 2: queue all finalizers _or_ handle profile record.
   453  			for siter.valid() && uintptr(siter.s.offset) < endOffset {
   454  				// Find the exact byte for which the special was setup
   455  				// (as opposed to object beginning).
   456  				special := siter.s
   457  				p := s.base() + uintptr(special.offset)
   458  				if special.kind == _KindSpecialFinalizer || !hasFin {
   459  					siter.unlinkAndNext()
   460  					freeSpecial(special, unsafe.Pointer(p), size)
   461  				} else {
   462  					// The object has finalizers, so we're keeping it alive.
   463  					// All other specials only apply when an object is freed,
   464  					// so just keep the special record.
   465  					siter.next()
   466  				}
   467  			}
   468  		} else {
   469  			// object is still live
   470  			if siter.s.kind == _KindSpecialReachable {
   471  				special := siter.unlinkAndNext()
   472  				(*specialReachable)(unsafe.Pointer(special)).reachable = true
   473  				freeSpecial(special, unsafe.Pointer(p), size)
   474  			} else {
   475  				// keep special record
   476  				siter.next()
   477  			}
   478  		}
   479  	}
   480  	if hadSpecials && s.specials == nil {
   481  		spanHasNoSpecials(s)
   482  	}
   483  
   484  	if debug.allocfreetrace != 0 || debug.clobberfree != 0 || raceenabled || msanenabled {
   485  		// Find all newly freed objects. This doesn't have to
   486  		// efficient; allocfreetrace has massive overhead.
   487  		mbits := s.markBitsForBase()
   488  		abits := s.allocBitsForIndex(0)
   489  		for i := uintptr(0); i < s.nelems; i++ {
   490  			if !mbits.isMarked() && (abits.index < s.freeindex || abits.isMarked()) {
   491  				x := s.base() + i*s.elemsize
   492  				if debug.allocfreetrace != 0 {
   493  					tracefree(unsafe.Pointer(x), size)
   494  				}
   495  				if debug.clobberfree != 0 {
   496  					clobberfree(unsafe.Pointer(x), size)
   497  				}
   498  				if raceenabled {
   499  					racefree(unsafe.Pointer(x), size)
   500  				}
   501  				if msanenabled {
   502  					msanfree(unsafe.Pointer(x), size)
   503  				}
   504  			}
   505  			mbits.advance()
   506  			abits.advance()
   507  		}
   508  	}
   509  
   510  	// Check for zombie objects.
   511  	if s.freeindex < s.nelems {
   512  		// Everything < freeindex is allocated and hence
   513  		// cannot be zombies.
   514  		//
   515  		// Check the first bitmap byte, where we have to be
   516  		// careful with freeindex.
   517  		obj := s.freeindex
   518  		if (*s.gcmarkBits.bytep(obj / 8)&^*s.allocBits.bytep(obj / 8))>>(obj%8) != 0 {
   519  			s.reportZombies()
   520  		}
   521  		// Check remaining bytes.
   522  		for i := obj/8 + 1; i < divRoundUp(s.nelems, 8); i++ {
   523  			if *s.gcmarkBits.bytep(i)&^*s.allocBits.bytep(i) != 0 {
   524  				s.reportZombies()
   525  			}
   526  		}
   527  	}
   528  
   529  	// Count the number of free objects in this span.
   530  	nalloc := uint16(s.countAlloc())
   531  	nfreed := s.allocCount - nalloc
   532  	if nalloc > s.allocCount {
   533  		// The zombie check above should have caught this in
   534  		// more detail.
   535  		print("runtime: nelems=", s.nelems, " nalloc=", nalloc, " previous allocCount=", s.allocCount, " nfreed=", nfreed, "\n")
   536  		throw("sweep increased allocation count")
   537  	}
   538  
   539  	s.allocCount = nalloc
   540  	s.freeindex = 0 // reset allocation index to start of span.
   541  	if trace.enabled {
   542  		getg().m.p.ptr().traceReclaimed += uintptr(nfreed) * s.elemsize
   543  	}
   544  
   545  	// gcmarkBits becomes the allocBits.
   546  	// get a fresh cleared gcmarkBits in preparation for next GC
   547  	s.allocBits = s.gcmarkBits
   548  	s.gcmarkBits = newMarkBits(s.nelems)
   549  
   550  	// Initialize alloc bits cache.
   551  	s.refillAllocCache(0)
   552  
   553  	// The span must be in our exclusive ownership until we update sweepgen,
   554  	// check for potential races.
   555  	if state := s.state.get(); state != mSpanInUse || s.sweepgen != sweepgen-1 {
   556  		print("mspan.sweep: state=", state, " sweepgen=", s.sweepgen, " mheap.sweepgen=", sweepgen, "\n")
   557  		throw("mspan.sweep: bad span state after sweep")
   558  	}
   559  	if s.sweepgen == sweepgen+1 || s.sweepgen == sweepgen+3 {
   560  		throw("swept cached span")
   561  	}
   562  
   563  	// We need to set s.sweepgen = h.sweepgen only when all blocks are swept,
   564  	// because of the potential for a concurrent free/SetFinalizer.
   565  	//
   566  	// But we need to set it before we make the span available for allocation
   567  	// (return it to heap or mcentral), because allocation code assumes that a
   568  	// span is already swept if available for allocation.
   569  	//
   570  	// Serialization point.
   571  	// At this point the mark bits are cleared and allocation ready
   572  	// to go so release the span.
   573  	atomic.Store(&s.sweepgen, sweepgen)
   574  
   575  	if spc.sizeclass() != 0 {
   576  		// Handle spans for small objects.
   577  		if nfreed > 0 {
   578  			// Only mark the span as needing zeroing if we've freed any
   579  			// objects, because a fresh span that had been allocated into,
   580  			// wasn't totally filled, but then swept, still has all of its
   581  			// free slots zeroed.
   582  			s.needzero = 1
   583  			stats := memstats.heapStats.acquire()
   584  			atomic.Xadduintptr(&stats.smallFreeCount[spc.sizeclass()], uintptr(nfreed))
   585  			memstats.heapStats.release()
   586  		}
   587  		if !preserve {
   588  			// The caller may not have removed this span from whatever
   589  			// unswept set its on but taken ownership of the span for
   590  			// sweeping by updating sweepgen. If this span still is in
   591  			// an unswept set, then the mcentral will pop it off the
   592  			// set, check its sweepgen, and ignore it.
   593  			if nalloc == 0 {
   594  				// Free totally free span directly back to the heap.
   595  				mheap_.freeSpan(s)
   596  				return true
   597  			}
   598  			// Return span back to the right mcentral list.
   599  			if uintptr(nalloc) == s.nelems {
   600  				mheap_.central[spc].mcentral.fullSwept(sweepgen).push(s)
   601  			} else {
   602  				mheap_.central[spc].mcentral.partialSwept(sweepgen).push(s)
   603  			}
   604  		}
   605  	} else if !preserve {
   606  		// Handle spans for large objects.
   607  		if nfreed != 0 {
   608  			// Free large object span to heap.
   609  
   610  			// NOTE(rsc,dvyukov): The original implementation of efence
   611  			// in CL 22060046 used sysFree instead of sysFault, so that
   612  			// the operating system would eventually give the memory
   613  			// back to us again, so that an efence program could run
   614  			// longer without running out of memory. Unfortunately,
   615  			// calling sysFree here without any kind of adjustment of the
   616  			// heap data structures means that when the memory does
   617  			// come back to us, we have the wrong metadata for it, either in
   618  			// the mspan structures or in the garbage collection bitmap.
   619  			// Using sysFault here means that the program will run out of
   620  			// memory fairly quickly in efence mode, but at least it won't
   621  			// have mysterious crashes due to confused memory reuse.
   622  			// It should be possible to switch back to sysFree if we also
   623  			// implement and then call some kind of mheap.deleteSpan.
   624  			if debug.efence > 0 {
   625  				s.limit = 0 // prevent mlookup from finding this span
   626  				sysFault(unsafe.Pointer(s.base()), size)
   627  			} else {
   628  				mheap_.freeSpan(s)
   629  			}
   630  			stats := memstats.heapStats.acquire()
   631  			atomic.Xadduintptr(&stats.largeFreeCount, 1)
   632  			atomic.Xadduintptr(&stats.largeFree, size)
   633  			memstats.heapStats.release()
   634  			return true
   635  		}
   636  
   637  		// Add a large span directly onto the full+swept list.
   638  		mheap_.central[spc].mcentral.fullSwept(sweepgen).push(s)
   639  	}
   640  	return false
   641  }
   642  
   643  // reportZombies reports any marked but free objects in s and throws.
   644  //
   645  // This generally means one of the following:
   646  //
   647  // 1. User code converted a pointer to a uintptr and then back
   648  // unsafely, and a GC ran while the uintptr was the only reference to
   649  // an object.
   650  //
   651  // 2. User code (or a compiler bug) constructed a bad pointer that
   652  // points to a free slot, often a past-the-end pointer.
   653  //
   654  // 3. The GC two cycles ago missed a pointer and freed a live object,
   655  // but it was still live in the last cycle, so this GC cycle found a
   656  // pointer to that object and marked it.
   657  func (s *mspan) reportZombies() {
   658  	printlock()
   659  	print("runtime: marked free object in span ", s, ", elemsize=", s.elemsize, " freeindex=", s.freeindex, " (bad use of unsafe.Pointer? try -d=checkptr)\n")
   660  	mbits := s.markBitsForBase()
   661  	abits := s.allocBitsForIndex(0)
   662  	for i := uintptr(0); i < s.nelems; i++ {
   663  		addr := s.base() + i*s.elemsize
   664  		print(hex(addr))
   665  		alloc := i < s.freeindex || abits.isMarked()
   666  		if alloc {
   667  			print(" alloc")
   668  		} else {
   669  			print(" free ")
   670  		}
   671  		if mbits.isMarked() {
   672  			print(" marked  ")
   673  		} else {
   674  			print(" unmarked")
   675  		}
   676  		zombie := mbits.isMarked() && !alloc
   677  		if zombie {
   678  			print(" zombie")
   679  		}
   680  		print("\n")
   681  		if zombie {
   682  			length := s.elemsize
   683  			if length > 1024 {
   684  				length = 1024
   685  			}
   686  			hexdumpWords(addr, addr+length, nil)
   687  		}
   688  		mbits.advance()
   689  		abits.advance()
   690  	}
   691  	throw("found pointer to free object")
   692  }
   693  
   694  // deductSweepCredit deducts sweep credit for allocating a span of
   695  // size spanBytes. This must be performed *before* the span is
   696  // allocated to ensure the system has enough credit. If necessary, it
   697  // performs sweeping to prevent going in to debt. If the caller will
   698  // also sweep pages (e.g., for a large allocation), it can pass a
   699  // non-zero callerSweepPages to leave that many pages unswept.
   700  //
   701  // deductSweepCredit makes a worst-case assumption that all spanBytes
   702  // bytes of the ultimately allocated span will be available for object
   703  // allocation.
   704  //
   705  // deductSweepCredit is the core of the "proportional sweep" system.
   706  // It uses statistics gathered by the garbage collector to perform
   707  // enough sweeping so that all pages are swept during the concurrent
   708  // sweep phase between GC cycles.
   709  //
   710  // mheap_ must NOT be locked.
   711  func deductSweepCredit(spanBytes uintptr, callerSweepPages uintptr) {
   712  	if mheap_.sweepPagesPerByte == 0 {
   713  		// Proportional sweep is done or disabled.
   714  		return
   715  	}
   716  
   717  	if trace.enabled {
   718  		traceGCSweepStart()
   719  	}
   720  
   721  retry:
   722  	sweptBasis := atomic.Load64(&mheap_.pagesSweptBasis)
   723  
   724  	// Fix debt if necessary.
   725  	newHeapLive := uintptr(atomic.Load64(&gcController.heapLive)-mheap_.sweepHeapLiveBasis) + spanBytes
   726  	pagesTarget := int64(mheap_.sweepPagesPerByte*float64(newHeapLive)) - int64(callerSweepPages)
   727  	for pagesTarget > int64(atomic.Load64(&mheap_.pagesSwept)-sweptBasis) {
   728  		if sweepone() == ^uintptr(0) {
   729  			mheap_.sweepPagesPerByte = 0
   730  			break
   731  		}
   732  		if atomic.Load64(&mheap_.pagesSweptBasis) != sweptBasis {
   733  			// Sweep pacing changed. Recompute debt.
   734  			goto retry
   735  		}
   736  	}
   737  
   738  	if trace.enabled {
   739  		traceGCSweepDone()
   740  	}
   741  }
   742  
   743  // clobberfree sets the memory content at x to bad content, for debugging
   744  // purposes.
   745  func clobberfree(x unsafe.Pointer, size uintptr) {
   746  	// size (span.elemsize) is always a multiple of 4.
   747  	for i := uintptr(0); i < size; i += 4 {
   748  		*(*uint32)(add(x, i)) = 0xdeadbeef
   749  	}
   750  }
   751  

View as plain text