在《切片传递的隐藏危机》一文中,小菜刀有简单地提及到切片扩容的问题。在读者讨论群中,有人举了以下例子,想得到一个合理的回答。
package main
func main() {
s := []int{1,2}
s = append(s, 3,4,5)
println(cap(s))
}
// output: 6
append
appendbuiltinbuiltin.go
// The append built-in function appends elements to the end of a slice. If
// it has sufficient capacity, the destination is resliced to accommodate the
// new elements. If it does not, a new underlying array will be allocated.
// Append returns the updated slice. It is therefore necessary to store the
// result of append, often in the variable holding the slice itself:
// slice = append(slice, elem1, elem2)
// slice = append(slice, anotherSlice...)
// As a special case, it is legal to append a string to a byte slice, like this:
// slice = append([]byte("hello "), "world"...)
func append(slice []Type, elems ...Type) []Type
append 会追加一个或多个数据至 slice 中,这些数据会存储至 slice 的底层数组。其中,数组长度是固定的,如果数组的剩余空间足以容纳追加的数据,则可以正常地将数据存入该数组。一旦追加数据后总长度超过原数组长度,原数组就无法满足存储追加数据的要求。此时会怎么处理呢?
同时我们发现,该文件中仅仅定义了函数签名,并没有包含函数实现的任何代码。这里我们不免好奇,append究竟是如何实现的呢?
编译过程
为了回答上述问题,我们不妨从编译入手。Go编译可分为四个阶段:词法与语法分析、类型检查与抽象语法树(AST)转换、中间代码生成和生成最后的机器码。
src/cmd/compile/internal/gc/typecheck.go
func typecheck1(n *Node, top int) (res *Node) {
...
switch n.Op {
case OAPPEND:
...
}
src/cmd/compile/internal/gc/walk.go
func walkexpr(n *Node, init *Nodes) *Node {
...
case OAPPEND:
// x = append(...)
r := n.Right
if r.Type.Elem().NotInHeap() {
yyerror("%v can't be allocated in Go; it is incomplete (or unallocatable)", r.Type.Elem())
}
switch {
case isAppendOfMake(r):
// x = append(y, make([]T, y)...)
r = extendslice(r, init)
case r.IsDDD():
r = appendslice(r, init) // also works for append(slice, string).
default:
r = walkappend(r, init, n)
}
...
}
src/cmd/compile/internal/gc/ssa.go
// append converts an OAPPEND node to SSA.
// If inplace is false, it converts the OAPPEND expression n to an ssa.Value,
// adds it to s, and returns the Value.
// If inplace is true, it writes the result of the OAPPEND expression n
// back to the slice being appended to, and returns nil.
// inplace MUST be set to false if the slice can be SSA'd.
func (s *state) append(n *Node, inplace bool) *ssa.Value {
...
}
state.appendinplacestate.appendappend(s, e1, e2, e3)
// If inplace is false, process as expression "append(s, e1, e2, e3)":
ptr, len, cap := s
newlen := len + 3
if newlen > cap {
ptr, len, cap = growslice(s, newlen)
newlen = len + 3 // recalculate to avoid a spill
}
// with write barriers, if needed:
*(ptr+len) = e1
*(ptr+len+1) = e2
*(ptr+len+2) = e3
return makeslice(ptr, newlen, cap)
slice = append(slice, 1, 2, 3)
// If inplace is true, process as statement "s = append(s, e1, e2, e3)":
a := &s
ptr, len, cap := s
newlen := len + 3
if uint(newlen) > uint(cap) {
newptr, len, newcap = growslice(ptr, len, cap, newlen)
vardef(a) // if necessary, advise liveness we are writing a new a
*a.cap = newcap // write before ptr to avoid a spill
*a.ptr = newptr // with write barrier
}
newlen = len + 3 // recalculate to avoid a spill
*a.len = newlen
// with write barriers, if needed:
*(ptr+len) = e1
*(ptr+len+1) = e2
*(ptr+len+2) = e3
inpalceruntime.growslice
slice=append(slice,1)
情况1,切片的底层数组还有可容纳追加元素的空间。
情况2,切片的底层数组已无可容纳追加元素的空间,需调用扩容函数,进行扩容。
扩容函数
growslicecap
growslice
- 初步确定切片容量
func growslice(et *_type, old slice, cap int) slice {
...
newcap := old.cap
doublecap := newcap + newcap
if cap > doublecap {
newcap = cap
} else {
if old.len < 1024 {
newcap = doublecap
} else {
// Check 0 < newcap to detect overflow
// and prevent an infinite loop.
for 0 < newcap && newcap < cap {
newcap += newcap / 4
}
// Set newcap to the requested cap when
// the newcap calculation overflowed.
if newcap <= 0 {
newcap = cap
}
}
}
...
}
capdoublecapnewcap
- 计算容量所需内存大小
var overflow bool var lenmem, newlenmem, capmem uintptr switch { case et.size == 1: lenmem = uintptr(old.len) newlenmem = uintptr(cap) capmem = roundupsize(uintptr(newcap)) overflow = uintptr(newcap) > maxAlloc newcap = int(capmem) case et.size == sys.PtrSize: lenmem = uintptr(old.len) * sys.PtrSize newlenmem = uintptr(cap) * sys.PtrSize capmem = roundupsize(uintptr(newcap) * sys.PtrSize) overflow = uintptr(newcap) > maxAlloc/sys.PtrSize newcap = int(capmem / sys.PtrSize) case isPowerOfTwo(et.size): var shift uintptr if sys.PtrSize == 8 { // Mask shift for better code generation. shift = uintptr(sys.Ctz64(uint64(et.size))) & 63 } else { shift = uintptr(sys.Ctz32(uint32(et.size))) & 31 } lenmem = uintptr(old.len) << shift newlenmem = uintptr(cap) << shift capmem = roundupsize(uintptr(newcap) << shift) overflow = uintptr(newcap) > (maxAlloc >> shift) newcap = int(capmem >> shift) default: lenmem = uintptr(old.len) * et.size newlenmem = uintptr(cap) * et.size capmem, overflow = math.MulUintptr(et.size, uintptr(newcap)) capmem = roundupsize(capmem) newcap = int(capmem / et.size) }
在该环节,通过判断切片元素的字节大小是否为1,系统指针大小(32位为4,64位为8)或2的倍数,进入相应所需内存大小的计算逻辑。
roundupsizesizemallocgc
func roundupsize(size uintptr) uintptr { if size < _MaxSmallSize { if size <= smallSizeMax-8 { return uintptr(class_to_size[size_to_class8[divRoundUp(size, smallSizeDiv)]]) } else { return uintptr(class_to_size[size_to_class128[divRoundUp(size-smallSizeMax, largeSizeDiv)]]) } } // Go的内存管理虚拟地址页大小为 8k(_PageSize) // 当size的大小即将溢出时,就不采用向上取整的做法,直接用当前期望size值。 if size+_PageSize < size { return size } return alignUp(size, _PageSize)}
<_MaxSmallSizedivRoundUpclass_to_sizesize_to_class8size_to_class128
// _NumSizeClasses = 67 代表67种特定大小的对象类型var class_to_size = [_NumSizeClasses]uint16{0, 8, 16, 32, 48, 64, 80, 96, 112,...}
alignUpsize_PageSize
- 内存分配
if overflow || capmem > maxAlloc { panic(errorString("growslice: cap out of range")) } var p unsafe.Pointer if et.ptrdata == 0 { p = mallocgc(capmem, nil, false) memclrNoHeapPointers(add(p, newlenmem), capmem-newlenmem) } else { p = mallocgc(capmem, et, true) if lenmem > 0 && writeBarrier.enabled { bulkBarrierPreWriteSrcOnly(uintptr(p), uintptr(old.array), lenmem-et.size+et.ptrdata) } } memmove(p, old.array, lenmem) return slice{p, old.len, newcap}
panic
mallocgccapmemmemclrNoHeapPointersbulkBarrierPreWriteSrcOnly
memmovep
growslicelen =3cap=3slice=append(slice,1)
growslicelen
总结
这里回到文章开头中的例子
package mainfunc main() { s := []int{1,2} s = append(s, 3,4,5) println(cap(s))}
sappendgrowslicecapdoublecapdoublecapcapnewcap=5intsys.PtrSizeroundupsizecapmemnewcap
append
在扩容的容量确定上,相对比较复杂,它与CPU位数、元素大小、是否包含指针、追加个数等都有关系。当我们看完扩容源码逻辑后,发现去纠结它的扩容确切值并没什么必要。
在实际使用中,如果能够确定切片的容量范围,比较合适的做法是:切片初始化时就分配足够的容量空间,在append追加操作时,就不用再考虑扩容带来的性能损耗问题。
func BenchmarkAppendFixCap(b *testing.B) { for i := 0; i < b.N; i++ { a := make([]int, 0, 1000) for i := 0; i < 1000; i++ { a = append(a, i) } }}func BenchmarkAppend(b *testing.B) { for i := 0; i < b.N; i++ { a := make([]int, 0) for i := 0; i < 1000; i++ { a = append(a, i) } }}
它们的压测结果如下,孰优孰劣,一目了然。
$ go test -bench=. -benchmem
BenchmarkAppendFixCap-8 1953373 617 ns/op 0 B/op 0 allocs/op
BenchmarkAppend-8 426882 2832 ns/op 16376 B/op 11 allocs/op