Source file
src/runtime/slice.go
1
2
3
4
5 package runtime
6
7 import (
8 "internal/abi"
9 "internal/goarch"
10 "internal/goexperiment"
11 "internal/runtime/math"
12 "internal/runtime/sys"
13 "unsafe"
14 )
15
16 type slice struct {
17 array unsafe.Pointer
18 len int
19 cap int
20 }
21
22
23 type notInHeapSlice struct {
24 array *notInHeap
25 len int
26 cap int
27 }
28
29 func panicmakeslicelen() {
30 panic(errorString("makeslice: len out of range"))
31 }
32
33 func panicmakeslicecap() {
34 panic(errorString("makeslice: cap out of range"))
35 }
36
37
38
39 func makeslicecopy(et *_type, tolen int, fromlen int, from unsafe.Pointer) unsafe.Pointer {
40 var tomem, copymem uintptr
41 if uintptr(tolen) > uintptr(fromlen) {
42 var overflow bool
43 tomem, overflow = math.MulUintptr(et.Size_, uintptr(tolen))
44 if overflow || tomem > maxAlloc || tolen < 0 {
45 panicmakeslicelen()
46 }
47 copymem = et.Size_ * uintptr(fromlen)
48 } else {
49
50
51
52 tomem = et.Size_ * uintptr(tolen)
53 copymem = tomem
54 }
55
56 var to unsafe.Pointer
57 if !et.Pointers() {
58 to = mallocgc(tomem, nil, false)
59 if copymem < tomem {
60 memclrNoHeapPointers(add(to, copymem), tomem-copymem)
61 }
62 } else {
63
64 to = mallocgc(tomem, et, true)
65 if copymem > 0 && writeBarrier.enabled {
66
67
68
69
70
71
72 bulkBarrierPreWriteSrcOnly(uintptr(to), uintptr(from), copymem, et)
73 }
74 }
75
76 if raceenabled {
77 callerpc := sys.GetCallerPC()
78 pc := abi.FuncPCABIInternal(makeslicecopy)
79 racereadrangepc(from, copymem, callerpc, pc)
80 }
81 if msanenabled {
82 msanread(from, copymem)
83 }
84 if asanenabled {
85 asanread(from, copymem)
86 }
87
88 memmove(to, from, copymem)
89
90 return to
91 }
92
93
94
95
96
97
98
99
100
101
102 func makeslice(et *_type, len, cap int) unsafe.Pointer {
103 mem, overflow := math.MulUintptr(et.Size_, uintptr(cap))
104 if overflow || mem > maxAlloc || len < 0 || len > cap {
105
106
107
108
109
110 mem, overflow := math.MulUintptr(et.Size_, uintptr(len))
111 if overflow || mem > maxAlloc || len < 0 {
112 panicmakeslicelen()
113 }
114 panicmakeslicecap()
115 }
116
117 return mallocgc(mem, et, true)
118 }
119
120 func makeslice64(et *_type, len64, cap64 int64) unsafe.Pointer {
121 len := int(len64)
122 if int64(len) != len64 {
123 panicmakeslicelen()
124 }
125
126 cap := int(cap64)
127 if int64(cap) != cap64 {
128 panicmakeslicecap()
129 }
130
131 return makeslice(et, len, cap)
132 }
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178 func growslice(oldPtr unsafe.Pointer, newLen, oldCap, num int, et *_type) slice {
179 oldLen := newLen - num
180 if raceenabled {
181 callerpc := sys.GetCallerPC()
182 racereadrangepc(oldPtr, uintptr(oldLen*int(et.Size_)), callerpc, abi.FuncPCABIInternal(growslice))
183 }
184 if msanenabled {
185 msanread(oldPtr, uintptr(oldLen*int(et.Size_)))
186 }
187 if asanenabled {
188 asanread(oldPtr, uintptr(oldLen*int(et.Size_)))
189 }
190
191 if newLen < 0 {
192 panic(errorString("growslice: len out of range"))
193 }
194
195 if et.Size_ == 0 {
196
197
198 return slice{unsafe.Pointer(&zerobase), newLen, newLen}
199 }
200
201 newcap := nextslicecap(newLen, oldCap)
202
203 var overflow bool
204 var lenmem, newlenmem, capmem uintptr
205
206
207
208
209 noscan := !et.Pointers()
210 switch {
211 case et.Size_ == 1:
212 lenmem = uintptr(oldLen)
213 newlenmem = uintptr(newLen)
214 capmem = roundupsize(uintptr(newcap), noscan)
215 overflow = uintptr(newcap) > maxAlloc
216 newcap = int(capmem)
217 case et.Size_ == goarch.PtrSize:
218 lenmem = uintptr(oldLen) * goarch.PtrSize
219 newlenmem = uintptr(newLen) * goarch.PtrSize
220 capmem = roundupsize(uintptr(newcap)*goarch.PtrSize, noscan)
221 overflow = uintptr(newcap) > maxAlloc/goarch.PtrSize
222 newcap = int(capmem / goarch.PtrSize)
223 case isPowerOfTwo(et.Size_):
224 var shift uintptr
225 if goarch.PtrSize == 8 {
226
227 shift = uintptr(sys.TrailingZeros64(uint64(et.Size_))) & 63
228 } else {
229 shift = uintptr(sys.TrailingZeros32(uint32(et.Size_))) & 31
230 }
231 lenmem = uintptr(oldLen) << shift
232 newlenmem = uintptr(newLen) << shift
233 capmem = roundupsize(uintptr(newcap)<<shift, noscan)
234 overflow = uintptr(newcap) > (maxAlloc >> shift)
235 newcap = int(capmem >> shift)
236 capmem = uintptr(newcap) << shift
237 default:
238 lenmem = uintptr(oldLen) * et.Size_
239 newlenmem = uintptr(newLen) * et.Size_
240 capmem, overflow = math.MulUintptr(et.Size_, uintptr(newcap))
241 capmem = roundupsize(capmem, noscan)
242 newcap = int(capmem / et.Size_)
243 capmem = uintptr(newcap) * et.Size_
244 }
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259 if overflow || capmem > maxAlloc {
260 panic(errorString("growslice: len out of range"))
261 }
262
263 var p unsafe.Pointer
264 if !et.Pointers() {
265 p = mallocgc(capmem, nil, false)
266
267
268
269
270 memclrNoHeapPointers(add(p, newlenmem), capmem-newlenmem)
271 } else {
272
273 p = mallocgc(capmem, et, true)
274 if lenmem > 0 && writeBarrier.enabled {
275
276
277
278
279
280
281 bulkBarrierPreWriteSrcOnly(uintptr(p), uintptr(oldPtr), lenmem-et.Size_+et.PtrBytes, et)
282 }
283 }
284 memmove(p, oldPtr, lenmem)
285
286 return slice{p, newLen, newcap}
287 }
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303 func growsliceNoAlias(oldPtr unsafe.Pointer, newLen, oldCap, num int, et *_type) slice {
304 s := growslice(oldPtr, newLen, oldCap, num, et)
305 if goexperiment.RuntimeFreegc && oldPtr != nil && oldPtr != s.array {
306 if gp := getg(); uintptr(oldPtr) < gp.stack.lo || gp.stack.hi <= uintptr(oldPtr) {
307
308
309
310
311
312
313
314
315
316
317
318 noscan := !et.Pointers()
319 freegc(oldPtr, uintptr(oldCap)*et.Size_, noscan)
320 }
321 }
322 return s
323 }
324
325
326 func nextslicecap(newLen, oldCap int) int {
327 newcap := oldCap
328 doublecap := newcap + newcap
329 if newLen > doublecap {
330 return newLen
331 }
332
333 const threshold = 256
334 if oldCap < threshold {
335 return doublecap
336 }
337 for {
338
339
340
341 newcap += (newcap + 3*threshold) >> 2
342
343
344
345
346
347 if uint(newcap) >= uint(newLen) {
348 break
349 }
350 }
351
352
353
354 if newcap <= 0 {
355 return newLen
356 }
357 return newcap
358 }
359
360
361
362
363
364
365
366
367
368
369 func reflect_growslice(et *_type, old slice, num int) slice {
370
371
372 num -= old.cap - old.len
373 new := growslice(old.array, old.cap+num, old.cap, num, et)
374
375
376
377
378 if !et.Pointers() {
379 oldcapmem := uintptr(old.cap) * et.Size_
380 newlenmem := uintptr(new.len) * et.Size_
381 memclrNoHeapPointers(add(new.array, oldcapmem), newlenmem-oldcapmem)
382 }
383 new.len = old.len
384 return new
385 }
386
387 func isPowerOfTwo(x uintptr) bool {
388 return x&(x-1) == 0
389 }
390
391
392 func slicecopy(toPtr unsafe.Pointer, toLen int, fromPtr unsafe.Pointer, fromLen int, width uintptr) int {
393 if fromLen == 0 || toLen == 0 {
394 return 0
395 }
396
397 n := fromLen
398 if toLen < n {
399 n = toLen
400 }
401
402 if width == 0 {
403 return n
404 }
405
406 size := uintptr(n) * width
407 if raceenabled {
408 callerpc := sys.GetCallerPC()
409 pc := abi.FuncPCABIInternal(slicecopy)
410 racereadrangepc(fromPtr, size, callerpc, pc)
411 racewriterangepc(toPtr, size, callerpc, pc)
412 }
413 if msanenabled {
414 msanread(fromPtr, size)
415 msanwrite(toPtr, size)
416 }
417 if asanenabled {
418 asanread(fromPtr, size)
419 asanwrite(toPtr, size)
420 }
421
422 if size == 1 {
423
424 *(*byte)(toPtr) = *(*byte)(fromPtr)
425 } else {
426 memmove(toPtr, fromPtr, size)
427 }
428 return n
429 }
430
431
432 func bytealg_MakeNoZero(len int) []byte {
433 if uintptr(len) > maxAlloc {
434 panicmakeslicelen()
435 }
436 cap := roundupsize(uintptr(len), true)
437 return unsafe.Slice((*byte)(mallocgc(cap, nil, false)), cap)[:len]
438 }
439
440
441
442 func moveSlice(et *_type, old unsafe.Pointer, len, cap int) (unsafe.Pointer, int, int) {
443 if cap == 0 {
444 if old != nil {
445 old = unsafe.Pointer(&zerobase)
446 }
447 return old, 0, 0
448 }
449 capmem := uintptr(cap) * et.Size_
450 new := mallocgc(capmem, et, true)
451 bulkBarrierPreWriteSrcOnly(uintptr(new), uintptr(old), capmem, et)
452 memmove(new, old, capmem)
453 return new, len, cap
454 }
455
456
457
458 func moveSliceNoScan(elemSize uintptr, old unsafe.Pointer, len, cap int) (unsafe.Pointer, int, int) {
459 if cap == 0 {
460 if old != nil {
461 old = unsafe.Pointer(&zerobase)
462 }
463 return old, 0, 0
464 }
465 capmem := uintptr(cap) * elemSize
466 new := mallocgc(capmem, nil, false)
467 memmove(new, old, capmem)
468 return new, len, cap
469 }
470
471
472
473
474 func moveSliceNoCap(et *_type, old unsafe.Pointer, len int) (unsafe.Pointer, int, int) {
475 if len == 0 {
476 if old != nil {
477 old = unsafe.Pointer(&zerobase)
478 }
479 return old, 0, 0
480 }
481 lenmem := uintptr(len) * et.Size_
482 capmem := roundupsize(lenmem, false)
483 new := mallocgc(capmem, et, true)
484 bulkBarrierPreWriteSrcOnly(uintptr(new), uintptr(old), lenmem, et)
485 memmove(new, old, lenmem)
486 return new, len, int(capmem / et.Size_)
487 }
488
489
490 func moveSliceNoCapNoScan(elemSize uintptr, old unsafe.Pointer, len int) (unsafe.Pointer, int, int) {
491 if len == 0 {
492 if old != nil {
493 old = unsafe.Pointer(&zerobase)
494 }
495 return old, 0, 0
496 }
497 lenmem := uintptr(len) * elemSize
498 capmem := roundupsize(lenmem, true)
499 new := mallocgc(capmem, nil, false)
500 memmove(new, old, lenmem)
501 if capmem > lenmem {
502 memclrNoHeapPointers(add(new, lenmem), capmem-lenmem)
503 }
504 return new, len, int(capmem / elemSize)
505 }
506
507
508
509 func growsliceBuf(oldPtr unsafe.Pointer, newLen, oldCap, num int, et *_type, bufPtr unsafe.Pointer, bufLen int) slice {
510 if newLen > bufLen {
511
512 return growslice(oldPtr, newLen, oldCap, num, et)
513 }
514 oldLen := newLen - num
515 if oldPtr != bufPtr && oldLen != 0 {
516
517
518 memmove(bufPtr, oldPtr, uintptr(oldLen)*et.Size_)
519 }
520
521
522
523
524
525
526
527
528
529
530 newCap := int(roundupsize(uintptr(newLen)*et.Size_, !et.Pointers()) / et.Size_)
531
532
533
534
535
536
537 if newLen < newCap {
538 memclrNoHeapPointers(add(bufPtr, uintptr(newLen)*et.Size_), uintptr(newCap-newLen)*et.Size_)
539 }
540
541 return slice{bufPtr, newLen, newCap}
542 }
543
544
545
546 func growsliceBufNoAlias(oldPtr unsafe.Pointer, newLen, oldCap, num int, et *_type, bufPtr unsafe.Pointer, bufLen int) slice {
547 s := growsliceBuf(oldPtr, newLen, oldCap, num, et, bufPtr, bufLen)
548 if goexperiment.RuntimeFreegc && oldPtr != bufPtr && oldPtr != nil && oldPtr != s.array {
549
550
551
552
553
554
555
556
557 noscan := !et.Pointers()
558 freegc(oldPtr, uintptr(oldCap)*et.Size_, noscan)
559 }
560 return s
561 }
562
View as plain text