This source file includes following definitions.
- gc_color_name
- gc_trace_ref
- gc_remove_from_roots
- root_buffer_dtor
- gc_globals_ctor_ex
- gc_globals_ctor
- gc_globals_dtor
- gc_reset
- gc_init
- gc_possible_root
- gc_remove_from_buffer
- gc_scan_black
- gc_mark_grey
- gc_mark_roots
- gc_scan
- gc_scan_roots
- gc_add_garbage
- gc_collect_white
- gc_collect_roots
- gc_remove_nested_data_from_buffer
- zend_gc_collect_cycles
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 #include "zend.h"
23 #include "zend_API.h"
24
25
26 #define GC_ROOT_BUFFER_MAX_ENTRIES 10001
27
28 #define GC_FAKE_BUFFER_FLAG 0x80
29 #define GC_TYPE_MASK 0x7f
30
31 #define GC_HAS_DESTRUCTORS (1<<0)
32
33 #ifndef ZEND_GC_DEBUG
34 # define ZEND_GC_DEBUG 0
35 #endif
36
37 #define GC_NUM_ADDITIONAL_ENTRIES \
38 ((4096 - ZEND_MM_OVERHEAD - sizeof(void*) * 2) / sizeof(gc_root_buffer))
39
40 typedef struct _gc_addtional_bufer gc_additional_buffer;
41
42 struct _gc_addtional_bufer {
43 uint32_t used;
44 gc_additional_buffer *next;
45 gc_root_buffer buf[GC_NUM_ADDITIONAL_ENTRIES];
46 };
47
48 #ifdef ZTS
49 ZEND_API int gc_globals_id;
50 #else
51 ZEND_API zend_gc_globals gc_globals;
52 #endif
53
54 ZEND_API int (*gc_collect_cycles)(void);
55
56 #define GC_REMOVE_FROM_ROOTS(current) \
57 gc_remove_from_roots((current))
58
59 #if ZEND_GC_DEBUG > 1
60 # define GC_TRACE(format, ...) fprintf(stderr, format "\n", ##__VA_ARGS__);
61 # define GC_TRACE_REF(ref, format, ...) \
62 do { \
63 gc_trace_ref((zend_refcounted *) ref); \
64 fprintf(stderr, format "\n", ##__VA_ARGS__); \
65 } while (0)
66 # define GC_TRACE_SET_COLOR(ref, color) \
67 GC_TRACE_REF(ref, "->%s", gc_color_name(color))
68 #else
69 # define GC_TRACE_REF(ref, format, ...)
70 # define GC_TRACE_SET_COLOR(ref, new_color)
71 # define GC_TRACE(str)
72 #endif
73
74 #define GC_REF_SET_ADDRESS(ref, a) \
75 GC_INFO_SET_ADDRESS(GC_INFO(ref), a)
76 #define GC_REF_GET_COLOR(ref) \
77 GC_INFO_GET_COLOR(GC_INFO(ref))
78 #define GC_REF_SET_COLOR(ref, c) \
79 do { GC_TRACE_SET_COLOR(ref, c); GC_INFO_SET_COLOR(GC_INFO(ref), c); } while (0)
80 #define GC_REF_SET_BLACK(ref) \
81 do { GC_TRACE_SET_COLOR(ref, GC_BLACK); GC_INFO_SET_BLACK(GC_INFO(ref)); } while (0)
82 #define GC_REF_SET_PURPLE(ref) \
83 do { GC_TRACE_SET_COLOR(ref, GC_PURPLE); GC_INFO_SET_PURPLE(GC_INFO(ref)); } while (0)
84
85 #if ZEND_GC_DEBUG > 1
86 static const char *gc_color_name(uint32_t color) {
87 switch (color) {
88 case GC_BLACK: return "black";
89 case GC_WHITE: return "white";
90 case GC_GREY: return "grey";
91 case GC_PURPLE: return "purple";
92 default: return "unknown";
93 }
94 }
95 static void gc_trace_ref(zend_refcounted *ref) {
96 if (GC_TYPE(ref) == IS_OBJECT) {
97 zend_object *obj = (zend_object *) ref;
98 fprintf(stderr, "[%p] rc=%d addr=%d %s object(%s)#%d ",
99 ref, GC_REFCOUNT(ref), GC_ADDRESS(GC_INFO(ref)),
100 gc_color_name(GC_REF_GET_COLOR(ref)),
101 obj->ce->name->val, obj->handle);
102 } else if (GC_TYPE(ref) == IS_ARRAY) {
103 zend_array *arr = (zend_array *) ref;
104 fprintf(stderr, "[%p] rc=%d addr=%d %s array(%d) ",
105 ref, GC_REFCOUNT(ref), GC_ADDRESS(GC_INFO(ref)),
106 gc_color_name(GC_REF_GET_COLOR(ref)),
107 zend_hash_num_elements(arr));
108 } else {
109 fprintf(stderr, "[%p] rc=%d addr=%d %s %s ",
110 ref, GC_REFCOUNT(ref), GC_ADDRESS(GC_INFO(ref)),
111 gc_color_name(GC_REF_GET_COLOR(ref)),
112 zend_get_type_by_const(GC_TYPE(ref)));
113 }
114 }
115 #endif
116
117 static zend_always_inline void gc_remove_from_roots(gc_root_buffer *root)
118 {
119 root->next->prev = root->prev;
120 root->prev->next = root->next;
121 root->prev = GC_G(unused);
122 GC_G(unused) = root;
123 GC_BENCH_DEC(root_buf_length);
124 }
125
126 static void root_buffer_dtor(zend_gc_globals *gc_globals)
127 {
128 if (gc_globals->buf) {
129 free(gc_globals->buf);
130 gc_globals->buf = NULL;
131 }
132 }
133
134 static void gc_globals_ctor_ex(zend_gc_globals *gc_globals)
135 {
136 gc_globals->gc_enabled = 0;
137 gc_globals->gc_active = 0;
138
139 gc_globals->buf = NULL;
140
141 gc_globals->roots.next = &gc_globals->roots;
142 gc_globals->roots.prev = &gc_globals->roots;
143 gc_globals->unused = NULL;
144 gc_globals->next_to_free = NULL;
145
146 gc_globals->to_free.next = &gc_globals->to_free;
147 gc_globals->to_free.prev = &gc_globals->to_free;
148
149 gc_globals->gc_runs = 0;
150 gc_globals->collected = 0;
151
152 #if GC_BENCH
153 gc_globals->root_buf_length = 0;
154 gc_globals->root_buf_peak = 0;
155 gc_globals->zval_possible_root = 0;
156 gc_globals->zval_buffered = 0;
157 gc_globals->zval_remove_from_buffer = 0;
158 gc_globals->zval_marked_grey = 0;
159 #endif
160 }
161
162 ZEND_API void gc_globals_ctor(void)
163 {
164 #ifdef ZTS
165 ts_allocate_id(&gc_globals_id, sizeof(zend_gc_globals), (ts_allocate_ctor) gc_globals_ctor_ex, (ts_allocate_dtor) root_buffer_dtor);
166 #else
167 gc_globals_ctor_ex(&gc_globals);
168 #endif
169 }
170
171 ZEND_API void gc_globals_dtor(void)
172 {
173 #ifndef ZTS
174 root_buffer_dtor(&gc_globals);
175 #endif
176 }
177
178 ZEND_API void gc_reset(void)
179 {
180 GC_G(gc_runs) = 0;
181 GC_G(collected) = 0;
182 GC_G(gc_full) = 0;
183
184 #if GC_BENCH
185 GC_G(root_buf_length) = 0;
186 GC_G(root_buf_peak) = 0;
187 GC_G(zval_possible_root) = 0;
188 GC_G(zval_buffered) = 0;
189 GC_G(zval_remove_from_buffer) = 0;
190 GC_G(zval_marked_grey) = 0;
191 #endif
192
193 GC_G(roots).next = &GC_G(roots);
194 GC_G(roots).prev = &GC_G(roots);
195
196 GC_G(to_free).next = &GC_G(to_free);
197 GC_G(to_free).prev = &GC_G(to_free);
198
199 if (GC_G(buf)) {
200 GC_G(unused) = NULL;
201 GC_G(first_unused) = GC_G(buf) + 1;
202 } else {
203 GC_G(unused) = NULL;
204 GC_G(first_unused) = NULL;
205 GC_G(last_unused) = NULL;
206 }
207 }
208
209 ZEND_API void gc_init(void)
210 {
211 if (GC_G(buf) == NULL && GC_G(gc_enabled)) {
212 GC_G(buf) = (gc_root_buffer*) malloc(sizeof(gc_root_buffer) * GC_ROOT_BUFFER_MAX_ENTRIES);
213 GC_G(last_unused) = &GC_G(buf)[GC_ROOT_BUFFER_MAX_ENTRIES];
214 gc_reset();
215 }
216 }
217
218 ZEND_API void ZEND_FASTCALL gc_possible_root(zend_refcounted *ref)
219 {
220 gc_root_buffer *newRoot;
221
222 if (UNEXPECTED(CG(unclean_shutdown)) || UNEXPECTED(GC_G(gc_active))) {
223 return;
224 }
225
226 ZEND_ASSERT(GC_TYPE(ref) == IS_ARRAY || GC_TYPE(ref) == IS_OBJECT);
227 ZEND_ASSERT(EXPECTED(GC_REF_GET_COLOR(ref) == GC_BLACK));
228 ZEND_ASSERT(!GC_ADDRESS(GC_INFO(ref)));
229
230 GC_BENCH_INC(zval_possible_root);
231
232 newRoot = GC_G(unused);
233 if (newRoot) {
234 GC_G(unused) = newRoot->prev;
235 } else if (GC_G(first_unused) != GC_G(last_unused)) {
236 newRoot = GC_G(first_unused);
237 GC_G(first_unused)++;
238 } else {
239 if (!GC_G(gc_enabled)) {
240 return;
241 }
242 GC_REFCOUNT(ref)++;
243 gc_collect_cycles();
244 GC_REFCOUNT(ref)--;
245 if (UNEXPECTED(GC_REFCOUNT(ref)) == 0) {
246 zval_dtor_func_for_ptr(ref);
247 return;
248 }
249 if (UNEXPECTED(GC_INFO(ref))) {
250 return;
251 }
252 newRoot = GC_G(unused);
253 if (!newRoot) {
254 #if ZEND_GC_DEBUG
255 if (!GC_G(gc_full)) {
256 fprintf(stderr, "GC: no space to record new root candidate\n");
257 GC_G(gc_full) = 1;
258 }
259 #endif
260 return;
261 }
262 GC_G(unused) = newRoot->prev;
263 }
264
265 GC_TRACE_SET_COLOR(ref, GC_PURPLE);
266 GC_INFO(ref) = (newRoot - GC_G(buf)) | GC_PURPLE;
267 newRoot->ref = ref;
268
269 newRoot->next = GC_G(roots).next;
270 newRoot->prev = &GC_G(roots);
271 GC_G(roots).next->prev = newRoot;
272 GC_G(roots).next = newRoot;
273
274 GC_BENCH_INC(zval_buffered);
275 GC_BENCH_INC(root_buf_length);
276 GC_BENCH_PEAK(root_buf_peak, root_buf_length);
277 }
278
279 ZEND_API void ZEND_FASTCALL gc_remove_from_buffer(zend_refcounted *ref)
280 {
281 gc_root_buffer *root;
282
283 ZEND_ASSERT(GC_ADDRESS(GC_INFO(ref)));
284 ZEND_ASSERT(GC_ADDRESS(GC_INFO(ref)) < GC_ROOT_BUFFER_MAX_ENTRIES);
285
286 GC_BENCH_INC(zval_remove_from_buffer);
287
288 root = GC_G(buf) + GC_ADDRESS(GC_INFO(ref));
289 if (GC_REF_GET_COLOR(ref) != GC_BLACK) {
290 GC_TRACE_SET_COLOR(ref, GC_PURPLE);
291 }
292 GC_INFO(ref) = 0;
293 GC_REMOVE_FROM_ROOTS(root);
294
295
296 if (GC_G(next_to_free) == root) {
297 GC_G(next_to_free) = root->next;
298 }
299 }
300
301 static void gc_scan_black(zend_refcounted *ref)
302 {
303 HashTable *ht;
304 Bucket *p, *end;
305 zval *zv;
306
307 tail_call:
308 ht = NULL;
309 GC_REF_SET_BLACK(ref);
310
311 if (GC_TYPE(ref) == IS_OBJECT && !(GC_FLAGS(ref) & IS_OBJ_FREE_CALLED)) {
312 zend_object_get_gc_t get_gc;
313 zend_object *obj = (zend_object*)ref;
314
315 if (EXPECTED(IS_OBJ_VALID(EG(objects_store).object_buckets[obj->handle]) &&
316 (get_gc = obj->handlers->get_gc) != NULL)) {
317 int n;
318 zval *zv, *end;
319 zval tmp;
320
321 ZVAL_OBJ(&tmp, obj);
322 ht = get_gc(&tmp, &zv, &n);
323 end = zv + n;
324 if (EXPECTED(!ht)) {
325 if (!n) return;
326 while (!Z_REFCOUNTED_P(--end)) {
327 if (zv == end) return;
328 }
329 }
330 while (zv != end) {
331 if (Z_REFCOUNTED_P(zv)) {
332 ref = Z_COUNTED_P(zv);
333 GC_REFCOUNT(ref)++;
334 if (GC_REF_GET_COLOR(ref) != GC_BLACK) {
335 gc_scan_black(ref);
336 }
337 }
338 zv++;
339 }
340 if (EXPECTED(!ht)) {
341 ref = Z_COUNTED_P(zv);
342 GC_REFCOUNT(ref)++;
343 if (GC_REF_GET_COLOR(ref) != GC_BLACK) {
344 goto tail_call;
345 }
346 return;
347 }
348 } else {
349 return;
350 }
351 } else if (GC_TYPE(ref) == IS_ARRAY) {
352 if ((zend_array*)ref != &EG(symbol_table)) {
353 ht = (zend_array*)ref;
354 } else {
355 return;
356 }
357 } else if (GC_TYPE(ref) == IS_REFERENCE) {
358 if (Z_REFCOUNTED(((zend_reference*)ref)->val)) {
359 ref = Z_COUNTED(((zend_reference*)ref)->val);
360 GC_REFCOUNT(ref)++;
361 if (GC_REF_GET_COLOR(ref) != GC_BLACK) {
362 goto tail_call;
363 }
364 }
365 return;
366 } else {
367 return;
368 }
369
370 if (!ht->nNumUsed) return;
371 p = ht->arData;
372 end = p + ht->nNumUsed;
373 while (1) {
374 end--;
375 zv = &end->val;
376 if (Z_TYPE_P(zv) == IS_INDIRECT) {
377 zv = Z_INDIRECT_P(zv);
378 }
379 if (Z_REFCOUNTED_P(zv)) {
380 break;
381 }
382 if (p == end) return;
383 }
384 while (p != end) {
385 zv = &p->val;
386 if (Z_TYPE_P(zv) == IS_INDIRECT) {
387 zv = Z_INDIRECT_P(zv);
388 }
389 if (Z_REFCOUNTED_P(zv)) {
390 ref = Z_COUNTED_P(zv);
391 GC_REFCOUNT(ref)++;
392 if (GC_REF_GET_COLOR(ref) != GC_BLACK) {
393 gc_scan_black(ref);
394 }
395 }
396 p++;
397 }
398 zv = &p->val;
399 if (Z_TYPE_P(zv) == IS_INDIRECT) {
400 zv = Z_INDIRECT_P(zv);
401 }
402 ref = Z_COUNTED_P(zv);
403 GC_REFCOUNT(ref)++;
404 if (GC_REF_GET_COLOR(ref) != GC_BLACK) {
405 goto tail_call;
406 }
407 }
408
409 static void gc_mark_grey(zend_refcounted *ref)
410 {
411 HashTable *ht;
412 Bucket *p, *end;
413 zval *zv;
414
415 tail_call:
416 if (GC_REF_GET_COLOR(ref) != GC_GREY) {
417 ht = NULL;
418 GC_BENCH_INC(zval_marked_grey);
419 GC_REF_SET_COLOR(ref, GC_GREY);
420
421 if (GC_TYPE(ref) == IS_OBJECT && !(GC_FLAGS(ref) & IS_OBJ_FREE_CALLED)) {
422 zend_object_get_gc_t get_gc;
423 zend_object *obj = (zend_object*)ref;
424
425 if (EXPECTED(IS_OBJ_VALID(EG(objects_store).object_buckets[obj->handle]) &&
426 (get_gc = obj->handlers->get_gc) != NULL)) {
427 int n;
428 zval *zv, *end;
429 zval tmp;
430
431 ZVAL_OBJ(&tmp, obj);
432 ht = get_gc(&tmp, &zv, &n);
433 end = zv + n;
434 if (EXPECTED(!ht)) {
435 if (!n) return;
436 while (!Z_REFCOUNTED_P(--end)) {
437 if (zv == end) return;
438 }
439 }
440 while (zv != end) {
441 if (Z_REFCOUNTED_P(zv)) {
442 ref = Z_COUNTED_P(zv);
443 GC_REFCOUNT(ref)--;
444 gc_mark_grey(ref);
445 }
446 zv++;
447 }
448 if (EXPECTED(!ht)) {
449 ref = Z_COUNTED_P(zv);
450 GC_REFCOUNT(ref)--;
451 goto tail_call;
452 }
453 } else {
454 return;
455 }
456 } else if (GC_TYPE(ref) == IS_ARRAY) {
457 if (((zend_array*)ref) == &EG(symbol_table)) {
458 GC_REF_SET_BLACK(ref);
459 return;
460 } else {
461 ht = (zend_array*)ref;
462 }
463 } else if (GC_TYPE(ref) == IS_REFERENCE) {
464 if (Z_REFCOUNTED(((zend_reference*)ref)->val)) {
465 if (UNEXPECTED(!EG(objects_store).object_buckets) &&
466 Z_TYPE(((zend_reference*)ref)->val) == IS_OBJECT) {
467 Z_TYPE_INFO(((zend_reference*)ref)->val) = IS_NULL;
468 return;
469 }
470 ref = Z_COUNTED(((zend_reference*)ref)->val);
471 GC_REFCOUNT(ref)--;
472 goto tail_call;
473 }
474 return;
475 } else {
476 return;
477 }
478
479 if (!ht->nNumUsed) return;
480 p = ht->arData;
481 end = p + ht->nNumUsed;
482 while (1) {
483 end--;
484 zv = &end->val;
485 if (Z_TYPE_P(zv) == IS_INDIRECT) {
486 zv = Z_INDIRECT_P(zv);
487 }
488 if (Z_REFCOUNTED_P(zv)) {
489 break;
490 }
491 if (p == end) return;
492 }
493 while (p != end) {
494 zv = &p->val;
495 if (Z_TYPE_P(zv) == IS_INDIRECT) {
496 zv = Z_INDIRECT_P(zv);
497 }
498 if (Z_REFCOUNTED_P(zv)) {
499 if (Z_TYPE_P(zv) == IS_OBJECT &&
500 UNEXPECTED(!EG(objects_store).object_buckets)) {
501 Z_TYPE_INFO_P(zv) = IS_NULL;
502 } else {
503 ref = Z_COUNTED_P(zv);
504 GC_REFCOUNT(ref)--;
505 gc_mark_grey(ref);
506 }
507 }
508 p++;
509 }
510 zv = &p->val;
511 if (Z_TYPE_P(zv) == IS_INDIRECT) {
512 zv = Z_INDIRECT_P(zv);
513 }
514 if (Z_TYPE_P(zv) == IS_OBJECT &&
515 UNEXPECTED(!EG(objects_store).object_buckets)) {
516 Z_TYPE_INFO_P(zv) = IS_NULL;
517 } else {
518 ref = Z_COUNTED_P(zv);
519 GC_REFCOUNT(ref)--;
520 goto tail_call;
521 }
522 }
523 }
524
525 static void gc_mark_roots(void)
526 {
527 gc_root_buffer *current = GC_G(roots).next;
528
529 while (current != &GC_G(roots)) {
530 if (GC_REF_GET_COLOR(current->ref) == GC_PURPLE) {
531 gc_mark_grey(current->ref);
532 }
533 current = current->next;
534 }
535 }
536
537 static void gc_scan(zend_refcounted *ref)
538 {
539 HashTable *ht;
540 Bucket *p, *end;
541 zval *zv;
542
543 tail_call:
544 if (GC_REF_GET_COLOR(ref) == GC_GREY) {
545 if (GC_REFCOUNT(ref) > 0) {
546 gc_scan_black(ref);
547 } else {
548 GC_REF_SET_COLOR(ref, GC_WHITE);
549 if (GC_TYPE(ref) == IS_OBJECT && !(GC_FLAGS(ref) & IS_OBJ_FREE_CALLED)) {
550 zend_object_get_gc_t get_gc;
551 zend_object *obj = (zend_object*)ref;
552
553 if (EXPECTED(IS_OBJ_VALID(EG(objects_store).object_buckets[obj->handle]) &&
554 (get_gc = obj->handlers->get_gc) != NULL)) {
555 int n;
556 zval *zv, *end;
557 zval tmp;
558
559 ZVAL_OBJ(&tmp, obj);
560 ht = get_gc(&tmp, &zv, &n);
561 end = zv + n;
562 if (EXPECTED(!ht)) {
563 if (!n) return;
564 while (!Z_REFCOUNTED_P(--end)) {
565 if (zv == end) return;
566 }
567 }
568 while (zv != end) {
569 if (Z_REFCOUNTED_P(zv)) {
570 ref = Z_COUNTED_P(zv);
571 gc_scan(ref);
572 }
573 zv++;
574 }
575 if (EXPECTED(!ht)) {
576 ref = Z_COUNTED_P(zv);
577 goto tail_call;
578 }
579 } else {
580 return;
581 }
582 } else if (GC_TYPE(ref) == IS_ARRAY) {
583 if ((zend_array*)ref == &EG(symbol_table)) {
584 GC_REF_SET_BLACK(ref);
585 return;
586 } else {
587 ht = (zend_array*)ref;
588 }
589 } else if (GC_TYPE(ref) == IS_REFERENCE) {
590 if (Z_REFCOUNTED(((zend_reference*)ref)->val)) {
591 ref = Z_COUNTED(((zend_reference*)ref)->val);
592 goto tail_call;
593 }
594 return;
595 } else {
596 return;
597 }
598
599 if (!ht->nNumUsed) return;
600 p = ht->arData;
601 end = p + ht->nNumUsed;
602 while (1) {
603 end--;
604 zv = &end->val;
605 if (Z_TYPE_P(zv) == IS_INDIRECT) {
606 zv = Z_INDIRECT_P(zv);
607 }
608 if (Z_REFCOUNTED_P(zv)) {
609 break;
610 }
611 if (p == end) return;
612 }
613 while (p != end) {
614 zv = &p->val;
615 if (Z_TYPE_P(zv) == IS_INDIRECT) {
616 zv = Z_INDIRECT_P(zv);
617 }
618 if (Z_REFCOUNTED_P(zv)) {
619 ref = Z_COUNTED_P(zv);
620 gc_scan(ref);
621 }
622 p++;
623 }
624 zv = &p->val;
625 if (Z_TYPE_P(zv) == IS_INDIRECT) {
626 zv = Z_INDIRECT_P(zv);
627 }
628 ref = Z_COUNTED_P(zv);
629 goto tail_call;
630 }
631 }
632 }
633
634 static void gc_scan_roots(void)
635 {
636 gc_root_buffer *current = GC_G(roots).next;
637
638 while (current != &GC_G(roots)) {
639 gc_scan(current->ref);
640 current = current->next;
641 }
642 }
643
644 static void gc_add_garbage(zend_refcounted *ref, gc_additional_buffer **additional_buffer)
645 {
646 gc_root_buffer *buf = GC_G(unused);
647
648 if (buf) {
649 GC_G(unused) = buf->prev;
650 #if 1
651
652 GC_INFO(ref) = buf - GC_G(buf);
653 #else
654 GC_REF_SET_ADDRESS(ref, buf - GC_G(buf));
655 #endif
656 } else if (GC_G(first_unused) != GC_G(last_unused)) {
657 buf = GC_G(first_unused);
658 GC_G(first_unused)++;
659 #if 1
660
661 GC_INFO(ref) = buf - GC_G(buf);
662 #else
663 GC_REF_SET_ADDRESS(ref, buf - GC_G(buf));
664 #endif
665 } else {
666
667
668
669
670 if (!*additional_buffer || (*additional_buffer)->used == GC_NUM_ADDITIONAL_ENTRIES) {
671 gc_additional_buffer *new_buffer = emalloc(sizeof(gc_additional_buffer));
672 new_buffer->used = 0;
673 new_buffer->next = *additional_buffer;
674 *additional_buffer = new_buffer;
675 }
676 buf = (*additional_buffer)->buf + (*additional_buffer)->used;
677 (*additional_buffer)->used++;
678 #if 1
679
680 GC_INFO(ref) = GC_ROOT_BUFFER_MAX_ENTRIES;
681 #else
682 GC_REF_SET_ADDRESS(ref, GC_ROOT_BUFFER_MAX_ENTRIES);
683 #endif
684
685 GC_TYPE(ref) |= GC_FAKE_BUFFER_FLAG;
686 }
687 if (buf) {
688 GC_REFCOUNT(ref)++;
689 buf->ref = ref;
690 buf->next = GC_G(roots).next;
691 buf->prev = &GC_G(roots);
692 GC_G(roots).next->prev = buf;
693 GC_G(roots).next = buf;
694 }
695 }
696
697 static int gc_collect_white(zend_refcounted *ref, uint32_t *flags, gc_additional_buffer **additional_buffer)
698 {
699 int count = 0;
700 HashTable *ht;
701 Bucket *p, *end;
702 zval *zv;
703
704 tail_call:
705 if (GC_REF_GET_COLOR(ref) == GC_WHITE) {
706 ht = NULL;
707 GC_REF_SET_BLACK(ref);
708
709
710 if (GC_TYPE(ref) != IS_REFERENCE) {
711 count++;
712 }
713
714 if (GC_TYPE(ref) == IS_OBJECT && !(GC_FLAGS(ref) & IS_OBJ_FREE_CALLED)) {
715 zend_object_get_gc_t get_gc;
716 zend_object *obj = (zend_object*)ref;
717
718 if (EXPECTED(IS_OBJ_VALID(EG(objects_store).object_buckets[obj->handle]) &&
719 (get_gc = obj->handlers->get_gc) != NULL)) {
720 int n;
721 zval *zv, *end;
722 zval tmp;
723
724 #if 1
725
726 if (!GC_INFO(ref)) {
727 #else
728 if (!GC_ADDRESS(GC_INFO(ref))) {
729 #endif
730 gc_add_garbage(ref, additional_buffer);
731 }
732 if (obj->handlers->dtor_obj &&
733 ((obj->handlers->dtor_obj != zend_objects_destroy_object) ||
734 (obj->ce->destructor != NULL))) {
735 *flags |= GC_HAS_DESTRUCTORS;
736 }
737 ZVAL_OBJ(&tmp, obj);
738 ht = get_gc(&tmp, &zv, &n);
739 end = zv + n;
740 if (EXPECTED(!ht)) {
741 if (!n) return count;
742 while (!Z_REFCOUNTED_P(--end)) {
743
744 if (Z_TYPE_P(zv) != IS_UNDEF) {
745 count++;
746 }
747 if (zv == end) return count;
748 }
749 }
750 while (zv != end) {
751 if (Z_REFCOUNTED_P(zv)) {
752 ref = Z_COUNTED_P(zv);
753 GC_REFCOUNT(ref)++;
754 count += gc_collect_white(ref, flags, additional_buffer);
755
756 } else if (Z_TYPE_P(zv) != IS_UNDEF) {
757 count++;
758 }
759 zv++;
760 }
761 if (EXPECTED(!ht)) {
762 ref = Z_COUNTED_P(zv);
763 GC_REFCOUNT(ref)++;
764 goto tail_call;
765 }
766 } else {
767 return count;
768 }
769 } else if (GC_TYPE(ref) == IS_ARRAY) {
770 #if 1
771
772 if (!GC_INFO(ref)) {
773 #else
774 if (!GC_ADDRESS(GC_INFO(ref))) {
775 #endif
776 gc_add_garbage(ref, additional_buffer);
777 }
778 ht = (zend_array*)ref;
779 } else if (GC_TYPE(ref) == IS_REFERENCE) {
780 if (Z_REFCOUNTED(((zend_reference*)ref)->val)) {
781 ref = Z_COUNTED(((zend_reference*)ref)->val);
782 GC_REFCOUNT(ref)++;
783 goto tail_call;
784 }
785 return count;
786 } else {
787 return count;
788 }
789
790 if (!ht->nNumUsed) return count;
791 p = ht->arData;
792 end = p + ht->nNumUsed;
793 while (1) {
794 end--;
795 zv = &end->val;
796 if (Z_TYPE_P(zv) == IS_INDIRECT) {
797 zv = Z_INDIRECT_P(zv);
798 }
799 if (Z_REFCOUNTED_P(zv)) {
800 break;
801 }
802
803 if (Z_TYPE_P(zv) != IS_UNDEF) {
804 count++;
805 }
806 if (p == end) return count;
807 }
808 while (p != end) {
809 zv = &p->val;
810 if (Z_TYPE_P(zv) == IS_INDIRECT) {
811 zv = Z_INDIRECT_P(zv);
812 }
813 if (Z_REFCOUNTED_P(zv)) {
814 ref = Z_COUNTED_P(zv);
815 GC_REFCOUNT(ref)++;
816 count += gc_collect_white(ref, flags, additional_buffer);
817
818 } else if (Z_TYPE_P(zv) != IS_UNDEF) {
819 count++;
820 }
821 p++;
822 }
823 zv = &p->val;
824 if (Z_TYPE_P(zv) == IS_INDIRECT) {
825 zv = Z_INDIRECT_P(zv);
826 }
827 ref = Z_COUNTED_P(zv);
828 GC_REFCOUNT(ref)++;
829 goto tail_call;
830 }
831 return count;
832 }
833
834 static int gc_collect_roots(uint32_t *flags, gc_additional_buffer **additional_buffer)
835 {
836 int count = 0;
837 gc_root_buffer *current = GC_G(roots).next;
838
839
840 while (current != &GC_G(roots)) {
841 gc_root_buffer *next = current->next;
842 if (GC_REF_GET_COLOR(current->ref) == GC_BLACK) {
843 GC_INFO(current->ref) = 0;
844 GC_REMOVE_FROM_ROOTS(current);
845 }
846 current = next;
847 }
848
849 current = GC_G(roots).next;
850 while (current != &GC_G(roots)) {
851 GC_REFCOUNT(current->ref)++;
852 if (GC_REF_GET_COLOR(current->ref) == GC_WHITE) {
853 count += gc_collect_white(current->ref, flags, additional_buffer);
854 }
855 current = current->next;
856 }
857
858
859 if (GC_G(roots).next != &GC_G(roots)) {
860 if (GC_G(to_free).next == &GC_G(to_free)) {
861
862 GC_G(to_free).next = GC_G(roots).next;
863 GC_G(to_free).prev = GC_G(roots).prev;
864 GC_G(to_free).next->prev = &GC_G(to_free);
865 GC_G(to_free).prev->next = &GC_G(to_free);
866 } else {
867
868 GC_G(to_free).prev->next = GC_G(roots).next;
869 GC_G(roots).next->prev = GC_G(to_free).prev;
870 GC_G(roots).prev->next = &GC_G(to_free);
871 GC_G(to_free).prev = GC_G(roots).prev;
872 }
873
874 GC_G(roots).next = &GC_G(roots);
875 GC_G(roots).prev = &GC_G(roots);
876 }
877 return count;
878 }
879
880 static void gc_remove_nested_data_from_buffer(zend_refcounted *ref, gc_root_buffer *root)
881 {
882 HashTable *ht = NULL;
883 Bucket *p, *end;
884 zval *zv;
885
886 tail_call:
887 if (root ||
888 (GC_ADDRESS(GC_INFO(ref)) != 0 &&
889 GC_REF_GET_COLOR(ref) == GC_BLACK &&
890 GC_ADDRESS(GC_INFO(ref)) != GC_ROOT_BUFFER_MAX_ENTRIES)) {
891 GC_TRACE_REF(ref, "removing from buffer");
892 GC_REFCOUNT(ref)--;
893 if (root) {
894 GC_INFO(ref) = 0;
895 GC_REMOVE_FROM_ROOTS(root);
896 root = NULL;
897 } else {
898 GC_REMOVE_FROM_BUFFER(ref);
899 }
900
901 if (GC_TYPE(ref) == IS_OBJECT && !(GC_FLAGS(ref) & IS_OBJ_FREE_CALLED)) {
902 zend_object_get_gc_t get_gc;
903 zend_object *obj = (zend_object*)ref;
904
905 if (EXPECTED(IS_OBJ_VALID(EG(objects_store).object_buckets[obj->handle]) &&
906 (get_gc = obj->handlers->get_gc) != NULL)) {
907 int n;
908 zval *zv, *end;
909 zval tmp;
910
911 ZVAL_OBJ(&tmp, obj);
912 ht = get_gc(&tmp, &zv, &n);
913 end = zv + n;
914 if (EXPECTED(!ht)) {
915 if (!n) return;
916 while (!Z_REFCOUNTED_P(--end)) {
917 if (zv == end) return;
918 }
919 }
920 while (zv != end) {
921 if (Z_REFCOUNTED_P(zv)) {
922 ref = Z_COUNTED_P(zv);
923 gc_remove_nested_data_from_buffer(ref, NULL);
924 }
925 zv++;
926 }
927 if (EXPECTED(!ht)) {
928 ref = Z_COUNTED_P(zv);
929 goto tail_call;
930 }
931 } else {
932 return;
933 }
934 } else if (GC_TYPE(ref) == IS_ARRAY) {
935 ht = (zend_array*)ref;
936 } else if (GC_TYPE(ref) == IS_REFERENCE) {
937 if (Z_REFCOUNTED(((zend_reference*)ref)->val)) {
938 ref = Z_COUNTED(((zend_reference*)ref)->val);
939 goto tail_call;
940 }
941 return;
942 } else {
943 return;
944 }
945
946 if (!ht->nNumUsed) return;
947 p = ht->arData;
948 end = p + ht->nNumUsed;
949 while (1) {
950 end--;
951 zv = &end->val;
952 if (Z_TYPE_P(zv) == IS_INDIRECT) {
953 zv = Z_INDIRECT_P(zv);
954 }
955 if (Z_REFCOUNTED_P(zv)) {
956 break;
957 }
958 if (p == end) return;
959 }
960 while (p != end) {
961 zv = &p->val;
962 if (Z_TYPE_P(zv) == IS_INDIRECT) {
963 zv = Z_INDIRECT_P(zv);
964 }
965 if (Z_REFCOUNTED_P(zv)) {
966 ref = Z_COUNTED_P(zv);
967 gc_remove_nested_data_from_buffer(ref, NULL);
968 }
969 p++;
970 }
971 zv = &p->val;
972 if (Z_TYPE_P(zv) == IS_INDIRECT) {
973 zv = Z_INDIRECT_P(zv);
974 }
975 ref = Z_COUNTED_P(zv);
976 goto tail_call;
977 }
978 }
979
980 ZEND_API int zend_gc_collect_cycles(void)
981 {
982 int count = 0;
983
984 if (GC_G(roots).next != &GC_G(roots)) {
985 gc_root_buffer *current, *next, *orig_next_to_free;
986 zend_refcounted *p;
987 gc_root_buffer to_free;
988 uint32_t gc_flags = 0;
989 gc_additional_buffer *additional_buffer;
990 #if ZEND_GC_DEBUG
991 zend_bool orig_gc_full;
992 #endif
993
994 if (GC_G(gc_active)) {
995 return 0;
996 }
997
998 GC_TRACE("Collecting cycles");
999 GC_G(gc_runs)++;
1000 GC_G(gc_active) = 1;
1001
1002 GC_TRACE("Marking roots");
1003 gc_mark_roots();
1004 GC_TRACE("Scanning roots");
1005 gc_scan_roots();
1006
1007 #if ZEND_GC_DEBUG
1008 orig_gc_full = GC_G(gc_full);
1009 GC_G(gc_full) = 0;
1010 #endif
1011
1012 GC_TRACE("Collecting roots");
1013 additional_buffer = NULL;
1014 count = gc_collect_roots(&gc_flags, &additional_buffer);
1015 #if ZEND_GC_DEBUG
1016 GC_G(gc_full) = orig_gc_full;
1017 #endif
1018 GC_G(gc_active) = 0;
1019
1020 if (GC_G(to_free).next == &GC_G(to_free)) {
1021
1022 GC_TRACE("Nothing to free");
1023 return 0;
1024 }
1025
1026
1027 to_free.next = GC_G(to_free).next;
1028 to_free.prev = GC_G(to_free).prev;
1029 to_free.next->prev = &to_free;
1030 to_free.prev->next = &to_free;
1031
1032
1033 GC_G(to_free).next = &GC_G(to_free);
1034 GC_G(to_free).prev = &GC_G(to_free);
1035
1036 orig_next_to_free = GC_G(next_to_free);
1037
1038 #if ZEND_GC_DEBUG
1039 orig_gc_full = GC_G(gc_full);
1040 GC_G(gc_full) = 0;
1041 #endif
1042
1043 if (gc_flags & GC_HAS_DESTRUCTORS) {
1044 GC_TRACE("Calling destructors");
1045 if (EG(objects_store).object_buckets) {
1046
1047 current = to_free.next;
1048 while (current != &to_free) {
1049 current->refcount = GC_REFCOUNT(current->ref);
1050 current = current->next;
1051 }
1052
1053
1054 current = to_free.next;
1055 while (current != &to_free) {
1056 p = current->ref;
1057 GC_G(next_to_free) = current->next;
1058 if ((GC_TYPE(p) & GC_TYPE_MASK) == IS_OBJECT) {
1059 zend_object *obj = (zend_object*)p;
1060
1061 if (IS_OBJ_VALID(EG(objects_store).object_buckets[obj->handle]) &&
1062 !(GC_FLAGS(obj) & IS_OBJ_DESTRUCTOR_CALLED)) {
1063 GC_TRACE_REF(obj, "calling destructor");
1064 GC_FLAGS(obj) |= IS_OBJ_DESTRUCTOR_CALLED;
1065 if (obj->handlers->dtor_obj) {
1066 GC_REFCOUNT(obj)++;
1067 obj->handlers->dtor_obj(obj);
1068 GC_REFCOUNT(obj)--;
1069 }
1070 }
1071 }
1072 current = GC_G(next_to_free);
1073 }
1074
1075
1076 current = to_free.next;
1077 while (current != &to_free) {
1078 GC_G(next_to_free) = current->next;
1079 if (GC_REFCOUNT(current->ref) > current->refcount) {
1080 gc_remove_nested_data_from_buffer(current->ref, current);
1081 }
1082 current = GC_G(next_to_free);
1083 }
1084 }
1085 }
1086
1087
1088 GC_TRACE("Destroying zvals");
1089 GC_G(gc_active) = 1;
1090 current = to_free.next;
1091 while (current != &to_free) {
1092 p = current->ref;
1093 GC_G(next_to_free) = current->next;
1094 GC_TRACE_REF(p, "destroying");
1095 if ((GC_TYPE(p) & GC_TYPE_MASK) == IS_OBJECT) {
1096 zend_object *obj = (zend_object*)p;
1097
1098 if (EG(objects_store).object_buckets &&
1099 IS_OBJ_VALID(EG(objects_store).object_buckets[obj->handle])) {
1100 GC_TYPE(obj) = IS_NULL;
1101 if (!(GC_FLAGS(obj) & IS_OBJ_FREE_CALLED)) {
1102 GC_FLAGS(obj) |= IS_OBJ_FREE_CALLED;
1103 if (obj->handlers->free_obj) {
1104 GC_REFCOUNT(obj)++;
1105 obj->handlers->free_obj(obj);
1106 GC_REFCOUNT(obj)--;
1107 }
1108 }
1109 SET_OBJ_BUCKET_NUMBER(EG(objects_store).object_buckets[obj->handle], EG(objects_store).free_list_head);
1110 EG(objects_store).free_list_head = obj->handle;
1111 p = current->ref = (zend_refcounted*)(((char*)obj) - obj->handlers->offset);
1112 }
1113 } else if ((GC_TYPE(p) & GC_TYPE_MASK) == IS_ARRAY) {
1114 zend_array *arr = (zend_array*)p;
1115
1116 GC_TYPE(arr) = IS_NULL;
1117 zend_hash_destroy(arr);
1118 }
1119 current = GC_G(next_to_free);
1120 }
1121
1122
1123 current = to_free.next;
1124 while (current != &to_free) {
1125 next = current->next;
1126 p = current->ref;
1127 if (EXPECTED(current >= GC_G(buf) && current < GC_G(buf) + GC_ROOT_BUFFER_MAX_ENTRIES)) {
1128 current->prev = GC_G(unused);
1129 GC_G(unused) = current;
1130 }
1131 efree(p);
1132 current = next;
1133 }
1134
1135 while (additional_buffer != NULL) {
1136 gc_additional_buffer *next = additional_buffer->next;
1137 efree(additional_buffer);
1138 additional_buffer = next;
1139 }
1140
1141 GC_TRACE("Collection finished");
1142 GC_G(collected) += count;
1143 GC_G(next_to_free) = orig_next_to_free;
1144 #if ZEND_GC_DEBUG
1145 GC_G(gc_full) = orig_gc_full;
1146 #endif
1147 GC_G(gc_active) = 0;
1148 }
1149
1150 return count;
1151 }
1152
1153
1154
1155
1156
1157
1158
1159