Generational behavior for the garbage collector

base/base.jl

@@ -154,7 +154,7 @@ end
 
 finalize(o::ANY) = ccall(:jl_finalize, Void, (Any,), o)
 
-gc() = ccall(:jl_gc_collect, Void, ())
+gc(full = true) = ccall(:jl_gc_collect, Void, (Int,), full ? 1 : 0)
 gc_enable() = ccall(:jl_gc_enable, Void, ())
 gc_disable() = ccall(:jl_gc_disable, Void, ())
 

base/util.jl

@@ -12,6 +12,9 @@ gc_time_ns() = ccall(:jl_gc_total_hrtime, UInt64, ())
 # total number of bytes allocated so far
 gc_bytes() = ccall(:jl_gc_total_bytes, Int64, ())
 
+gc_num_pause() = ccall(:jl_gc_num_pause, Int64, ())
+gc_num_full_sweep() = ccall(:jl_gc_num_full_sweep, Int64, ())
+
 function tic()
     t0 = time_ns()
     task_local_storage(:TIMERS, (t0, get(task_local_storage(), :TIMERS, ())))
@@ -36,12 +39,17 @@ function toc()
 end
 
 # print elapsed time, return expression value
-
-function time_print(t, b, g)
+const _units = ["bytes", "kB", "MB"]
+function time_print(t, b, g, np, nfs)
+    i = 1
+    while b > 1024 && i < length(_units)
+        b = div(b, 1024)
+        i += 1
+    end
     if 0 < g
-        @printf("elapsed time: %s seconds (%d bytes allocated, %.2f%% gc time)\n", t/1e9, b, 100*g/t)
+        @printf("elapsed time: %s seconds (%d %s allocated, %.2f%% gc time in %d pauses with %d full sweep)\n", t/1e9, b, _units[i], 100*g/t, np, nfs)
     else
-        @printf("elapsed time: %s seconds (%d bytes allocated)\n", t/1e9, b)
+        @printf("elapsed time: %s seconds (%d %s allocated)\n", t/1e9, b, _units[i])
     end
 end
 
@@ -50,11 +58,15 @@ macro time(ex)
         local b0 = gc_bytes()
         local t0 = time_ns()
         local g0 = gc_time_ns()
+        local n0 = gc_num_pause()
+        local nfs0 = gc_num_full_sweep()
         local val = $(esc(ex))
+        local nfs1 = gc_num_full_sweep()
+        local n1 = gc_num_pause()
         local g1 = gc_time_ns()
         local t1 = time_ns()
         local b1 = gc_bytes()
-        time_print(t1-t0, b1-b0, g1-g0)
+        time_print(t1-t0, b1-b0, g1-g0, n1-n0, nfs1-nfs0)
         val
     end
 end

doc/manual/embedding.rst

@@ -107,6 +107,23 @@ Several Julia values can be pushed at once using the ``JL_GC_PUSH2`` , ``JL_GC_P
     // Do something with args (e.g. call jl_... functions)
     JL_GC_POP();
 
+The garbage collector also operates under the assumption that it is aware of every old-generation object pointing to a young-generation one. Any time a pointer is updated breaking that assumption, it must be signaled to the collector with the ``gc_wb`` (write barrier) function like so::
+
+    jl_value_t *parent = some_old_value, *child = some_young_value;
+    ((some_specific_type*)parent)->field = child;
+    gc_wb(parent, child);
+
+It is in general impossible to predict which values will be old at runtime, so the write barrier must be inserted after all explicit stores. One notable exception is if the ``parent`` object was just allocated and garbage collection was not run since then. Remember that most ``jl_...`` functions can sometimes invoke garbage collection.
+
+The write barrier is also necessary for arrays of pointers when updating their data directly. For example::
+
+    jl_array_t *some_array = ...; // e.g. a Vector{Any}
+    void **data = (void**)jl_array_data(some_array);
+    jl_value_t *some_value = ...;
+    data[0] = some_value;
+    gc_wb(some_array, some_value);
+
+
 Manipulating the Garbage Collector
 ---------------------------------------------------
 

src/alloc.c

@@ -278,6 +278,7 @@ void jl_set_nth_field(jl_value_t *v, size_t i, jl_value_t *rhs)
     size_t offs = jl_field_offset(st,i) + sizeof(void*);
     if (st->fields[i].isptr) {
         *(jl_value_t**)((char*)v + offs) = rhs;
+        if(rhs != NULL) gc_wb(v, rhs);
     }
     else {
         jl_assign_bits((char*)v + offs, rhs);
@@ -521,17 +522,18 @@ static jl_sym_t *mk_symbol(const char *str)
 static void unmark_symbols_(jl_sym_t *root)
 {
     while (root != NULL) {
-        root->type = (jl_value_t*)(((uptrint_t)root->type)&~1UL);
+        root->type = (jl_value_t*)(((uptrint_t)root->type)&~3UL);
         unmark_symbols_(root->left);
         root = root->right;
     }
 }
 
 void jl_unmark_symbols(void) { unmark_symbols_(symtab); }
 
-static jl_sym_t **symtab_lookup(jl_sym_t **ptree, const char *str)
+static jl_sym_t **symtab_lookup(jl_sym_t **ptree, const char *str, jl_sym_t **parent)
 {
     int x;
+    if (parent != NULL) *parent = NULL;
     uptrint_t h = hash_symbol(str, strlen(str));
 
     // Tree nodes sorted by major key of (int(hash)) and minor key o (str).
@@ -542,6 +544,7 @@ static jl_sym_t **symtab_lookup(jl_sym_t **ptree, const char *str)
             if (x == 0)
                 return ptree;
         }
+        if (parent != NULL) *parent = *ptree;
         if (x < 0)
             ptree = &(*ptree)->left;
         else
@@ -553,16 +556,19 @@ static jl_sym_t **symtab_lookup(jl_sym_t **ptree, const char *str)
 jl_sym_t *jl_symbol(const char *str)
 {
     jl_sym_t **pnode;
-
-    pnode = symtab_lookup(&symtab, str);
-    if (*pnode == NULL)
+    jl_sym_t *parent;
+    pnode = symtab_lookup(&symtab, str, &parent);
+    if (*pnode == NULL) {
         *pnode = mk_symbol(str);
+        if (parent != NULL)
+            gc_wb(parent, *pnode);
+    }
     return *pnode;
 }
 
 jl_sym_t *jl_symbol_lookup(const char *str)
 {
-    return *symtab_lookup(&symtab, str);
+    return *symtab_lookup(&symtab, str, NULL);
 }
 
 DLLEXPORT jl_sym_t *jl_symbol_n(const char *str, int32_t len)
@@ -696,12 +702,15 @@ jl_datatype_t *jl_new_datatype(jl_sym_t *name, jl_datatype_t *super,
         t = jl_new_uninitialized_datatype(jl_tuple_len(fnames));
     else
         tn = t->name;
-
     // init before possibly calling jl_new_typename
     t->super = super;
+    if(super != NULL) gc_wb(t, t->super);
     t->parameters = parameters;
+    gc_wb(t, t->parameters);
     t->names = fnames;
+    gc_wb(t, t->names);
     t->types = ftypes;
+    if(ftypes != NULL) gc_wb(t, t->types);
     t->abstract = abstract;
     t->mutabl = mutabl;
     t->pointerfree = 0;
@@ -718,10 +727,13 @@ jl_datatype_t *jl_new_datatype(jl_sym_t *name, jl_datatype_t *super,
         else
             tn = jl_new_typename((jl_sym_t*)name);
         t->name = tn;
+        gc_wb(t, t->name);
     }
 
-    if (t->name->primary == NULL)
+    if (t->name->primary == NULL) {
         t->name->primary = (jl_value_t*)t;
+        gc_wb(t->name, t);
+    }
 
     if (abstract || jl_tuple_len(parameters) > 0) {
         t->uid = 0;

src/array.c

@@ -99,6 +99,7 @@ static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims,
             memset(data, 0, tot);
         JL_GC_POP();
     }
+    a->pooled = tsz <= 2048;
 
     a->data = data;
     if (elsz == 1) ((char*)data)[tot-1] = '\0';
@@ -147,8 +148,10 @@ jl_array_t *jl_reshape_array(jl_value_t *atype, jl_array_t *data, jl_tuple_t *di
     size_t ndims = jl_tuple_len(dims);
 
     int ndimwords = jl_array_ndimwords(ndims);
-    a = (jl_array_t*)allocobj((sizeof(jl_array_t) + sizeof(void*) + ndimwords*sizeof(size_t) + 15)&-16);
+    int tsz = (sizeof(jl_array_t) + sizeof(void*) + ndimwords*sizeof(size_t) + 15)&-16;
+    a = (jl_array_t*)allocobj(tsz);
     a->type = atype;
+    a->pooled = tsz <= 2048;
     a->ndims = ndims;
     a->offset = 0;
     a->data = NULL;
@@ -211,8 +214,9 @@ jl_array_t *jl_ptr_to_array_1d(jl_value_t *atype, void *data, size_t nel,
         elsz = jl_datatype_size(el_type);
     else
         elsz = sizeof(void*);
-
-    a = (jl_array_t*)allocobj((sizeof(jl_array_t)+jl_array_ndimwords(1)*sizeof(size_t)+15)&-16);
+    int tsz = (sizeof(jl_array_t)+jl_array_ndimwords(1)*sizeof(size_t)+15)&-16;
+    a = (jl_array_t*)allocobj(tsz);
+    a->pooled = tsz <= 2048;
     a->type = atype;
     a->data = data;
 #ifdef STORE_ARRAY_LEN
@@ -226,6 +230,7 @@ jl_array_t *jl_ptr_to_array_1d(jl_value_t *atype, void *data, size_t nel,
     if (own_buffer) {
         a->how = 2;
         jl_gc_track_malloced_array(a);
+        jl_gc_count_allocd(nel*elsz + (elsz == 1 ? 1 : 0));
     }
     else {
         a->how = 0;
@@ -260,7 +265,9 @@ jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data, jl_tuple_t *dims,
         elsz = sizeof(void*);
 
     int ndimwords = jl_array_ndimwords(ndims);
-    a = (jl_array_t*)allocobj((sizeof(jl_array_t) + ndimwords*sizeof(size_t)+15)&-16);
+    int tsz = (sizeof(jl_array_t) + ndimwords*sizeof(size_t)+15)&-16;
+    a = (jl_array_t*)allocobj(tsz);
+    a->pooled = tsz <= 2048;
     a->type = atype;
     a->data = data;
 #ifdef STORE_ARRAY_LEN
@@ -275,6 +282,7 @@ jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data, jl_tuple_t *dims,
     if (own_buffer) {
         a->how = 2;
         jl_gc_track_malloced_array(a);
+        jl_gc_count_allocd(nel*elsz + (elsz == 1 ? 1 : 0));
     }
     else {
         a->how = 0;
@@ -502,6 +510,7 @@ void jl_arrayset(jl_array_t *a, jl_value_t *rhs, size_t i)
     }
     else {
         ((jl_value_t**)a->data)[i] = rhs;
+        gc_wb(a, rhs);
     }
 }
 
@@ -545,7 +554,7 @@ static void array_resize_buffer(jl_array_t *a, size_t newlen, size_t oldlen, siz
     if (a->how == 2) {
         // already malloc'd - use realloc
         newdata = (char*)jl_gc_managed_realloc((char*)a->data - oldoffsnb, nbytes,
-                                               oldnbytes+oldoffsnb, a->isaligned);
+                                               oldnbytes+oldoffsnb, a->isaligned, (jl_value_t*)a);
         if (offs != a->offset) {
             memmove(&newdata[offsnb], &newdata[oldoffsnb], oldnbytes);
         }
@@ -574,6 +583,8 @@ static void array_resize_buffer(jl_array_t *a, size_t newlen, size_t oldlen, siz
     a->isshared = 0;
     if (a->ptrarray || es==1)
         memset(newdata+offsnb+oldnbytes, 0, nbytes-oldnbytes-offsnb);
+    if (a->how == 1)
+        gc_wb_buf(a, newdata);
     a->maxsize = newlen;
 }
 

src/ast.c

@@ -731,6 +731,7 @@ static jl_value_t *copy_ast(jl_value_t *expr, jl_tuple_t *sp, int do_sp)
         // of a top-level thunk that gets type inferred.
         li->def = li;
         li->ast = jl_prepare_ast(li, li->sparams);
+        gc_wb(li, li->ast);
         JL_GC_POP();
         return (jl_value_t*)li;
     }
@@ -749,17 +750,18 @@ static jl_value_t *copy_ast(jl_value_t *expr, jl_tuple_t *sp, int do_sp)
         jl_expr_t *ne = jl_exprn(e->head, jl_array_len(e->args));
         JL_GC_PUSH1(&ne);
         if (e->head == lambda_sym) {
-            jl_exprarg(ne, 0) = copy_ast(jl_exprarg(e,0), sp, 0);
-            jl_exprarg(ne, 1) = copy_ast(jl_exprarg(e,1), sp, 0);
-            jl_exprarg(ne, 2) = copy_ast(jl_exprarg(e,2), sp, 1);
+            jl_exprargset(ne, 0, copy_ast(jl_exprarg(e,0), sp, 0));
+            jl_exprargset(ne, 1, copy_ast(jl_exprarg(e,1), sp, 0));
+            jl_exprargset(ne, 2, copy_ast(jl_exprarg(e,2), sp, 1));
         }
         else if (e->head == assign_sym) {
-            jl_exprarg(ne, 0) = copy_ast(jl_exprarg(e,0), sp, 0);
-            jl_exprarg(ne, 1) = copy_ast(jl_exprarg(e,1), sp, 1);
+            jl_exprargset(ne, 0, copy_ast(jl_exprarg(e,0), sp, 0));
+            jl_exprargset(ne, 1, copy_ast(jl_exprarg(e,1), sp, 1));
         }
         else {
-            for(size_t i=0; i < jl_array_len(e->args); i++)
-                jl_exprarg(ne, i) = copy_ast(jl_exprarg(e,i), sp, 1);
+            for(size_t i=0; i < jl_array_len(e->args); i++) {
+                jl_exprargset(ne, i, copy_ast(jl_exprarg(e,i), sp, 1));
+            }
         }
         JL_GC_POP();
         return (jl_value_t*)ne;
@@ -780,10 +782,12 @@ DLLEXPORT jl_value_t *jl_copy_ast(jl_value_t *expr)
         ne = jl_exprn(e->head, l);
         if (l == 0) {
             ne->args = jl_alloc_cell_1d(0);
+            gc_wb(ne, ne->args);
         }
         else {
-            for(i=0; i < l; i++)
-                jl_exprarg(ne, i) = jl_copy_ast(jl_exprarg(e,i));
+            for(i=0; i < l; i++) {
+                jl_exprargset(ne, i, jl_copy_ast(jl_exprarg(e,i)));
+            }
         }
         JL_GC_POP();
         return (jl_value_t*)ne;
@@ -820,17 +824,18 @@ static jl_value_t *dont_copy_ast(jl_value_t *expr, jl_tuple_t *sp, int do_sp)
     else if (jl_is_expr(expr)) {
         jl_expr_t *e = (jl_expr_t*)expr;
         if (e->head == lambda_sym) {
-            jl_exprarg(e, 0) = dont_copy_ast(jl_exprarg(e,0), sp, 0);
-            jl_exprarg(e, 1) = dont_copy_ast(jl_exprarg(e,1), sp, 0);
-            jl_exprarg(e, 2) = dont_copy_ast(jl_exprarg(e,2), sp, 1);
+            jl_exprargset(e, 0, dont_copy_ast(jl_exprarg(e,0), sp, 0));
+            jl_exprargset(e, 1, dont_copy_ast(jl_exprarg(e,1), sp, 0));
+            jl_exprargset(e, 2, dont_copy_ast(jl_exprarg(e,2), sp, 1));
         }
         else if (e->head == assign_sym) {
-            jl_exprarg(e, 0) = dont_copy_ast(jl_exprarg(e,0), sp, 0);
-            jl_exprarg(e, 1) = dont_copy_ast(jl_exprarg(e,1), sp, 1);
+            jl_exprargset(e, 0, dont_copy_ast(jl_exprarg(e,0), sp, 0));
+            jl_exprargset(e, 1, dont_copy_ast(jl_exprarg(e,1), sp, 1));
         }
         else {
-            for(size_t i=0; i < jl_array_len(e->args); i++)
-                jl_exprarg(e, i) = dont_copy_ast(jl_exprarg(e,i), sp, 1);
+            for(size_t i=0; i < jl_array_len(e->args); i++) {
+                jl_exprargset(e, i, dont_copy_ast(jl_exprarg(e,i), sp, 1));
+            }
         }
         return (jl_value_t*)e;
     }