fix performance regression in parallel loops

also generally improve type serialization

It turns out this requires the serializer to preserve the identity of
TypeVars, since otherwise TypeName.primary doesn't have the right
structure and field types don't get populated correctly when the
type is instantiated. Since TypeVar identity does actually matter,
we have to consider them mutable for now.

base/multi.jl

@@ -1608,42 +1608,43 @@ function splitrange(N::Int, np::Int)
     return chunks
 end
 
-function preduce(reducer, f, N::Int)
+function preduce(reducer, f, R)
+    N = length(R)
     chunks = splitrange(N, nworkers())
     all_w = workers()[1:length(chunks)]
 
     w_exec = Task[]
     for (idx,pid) in enumerate(all_w)
-        t = Task(()->remotecall_fetch(f, pid, first(chunks[idx]), last(chunks[idx])))
+        t = Task(()->remotecall_fetch(f, pid, reducer, R, first(chunks[idx]), last(chunks[idx])))
         schedule(t)
         push!(w_exec, t)
     end
     reduce(reducer, [wait(t) for t in w_exec])
 end
 
-function pfor(f, N::Int)
-    [@spawn f(first(c), last(c)) for c in splitrange(N, nworkers())]
+function pfor(f, R)
+    [@spawn f(R, first(c), last(c)) for c in splitrange(length(R), nworkers())]
 end
 
-function make_preduce_body(reducer, var, body, R)
+function make_preduce_body(var, body)
     quote
-        function (lo::Int, hi::Int)
-            $(esc(var)) = ($R)[lo]
+        function (reducer, R, lo::Int, hi::Int)
+            $(esc(var)) = R[lo]
             ac = $(esc(body))
             if lo != hi
-                for $(esc(var)) in ($R)[(lo+1):hi]
-                    ac = ($reducer)(ac, $(esc(body)))
+                for $(esc(var)) in R[(lo+1):hi]
+                    ac = reducer(ac, $(esc(body)))
                 end
             end
             ac
         end
     end
 end
 
-function make_pfor_body(var, body, R)
+function make_pfor_body(var, body)
     quote
-        function (lo::Int, hi::Int)
-            for $(esc(var)) in ($R)[lo:hi]
+        function (R, lo::Int, hi::Int)
+            for $(esc(var)) in R[lo:hi]
                 $(esc(body))
             end
         end
@@ -1667,13 +1668,11 @@ macro parallel(args...)
     r = loop.args[1].args[2]
     body = loop.args[2]
     if na==1
-        thecall = :(pfor($(make_pfor_body(var, body, :therange)), length(therange)))
-        localize_vars(quote therange = $(esc(r)); $thecall; end)
+        thecall = :(pfor($(make_pfor_body(var, body)), $(esc(r))))
     else
-        thecall = :(preduce(thereducer,
-                            $(make_preduce_body(:thereducer, var, body, :therange)), length(therange)))
-        localize_vars(quote thereducer = $(esc(reducer)); therange = $(esc(r)); $thecall; end)
+        thecall = :(preduce($(esc(reducer)), $(make_preduce_body(var, body)), $(esc(r))))
     end
+    localize_vars(thecall)
 end
 
 

base/serialize.jl

@@ -358,7 +358,14 @@ function serialize(s::SerializationState, t::TypeName)
     serialize(s, t.primary.mutable)
     serialize(s, t.primary.ninitialized)
     if isdefined(t, :mt)
-        serialize(s, t.mt)
+        serialize(s, t.mt.name)
+        serialize(s, t.mt.defs)
+        serialize(s, t.mt.max_args)
+        if isdefined(t.mt, :kwsorter)
+            serialize(s, t.mt.kwsorter)
+        else
+            writetag(s.io, UNDEFREF_TAG)
+        end
     else
         writetag(s.io, UNDEFREF_TAG)
     end
@@ -633,14 +640,27 @@ function deserialize(s::SerializationState, ::Type{Union})
     Union{types...}
 end
 
+module __deserialized_types__
+end
+
 function deserialize(s::SerializationState, ::Type{TypeName})
     number = deserialize(s)
     name = deserialize(s)
     mod = deserialize(s)
     if haskey(known_object_data, number)
         tn = known_object_data[number]::TypeName
+        name = tn.name
+        mod = tn.module
+        makenew = false
+    elseif isdefined(mod, name)
+        tn = getfield(mod, name).name
+        # TODO: confirm somehow that the types match
+        name = tn.name
+        mod = tn.module
         makenew = false
     else
+        name = gensym()
+        mod = __deserialized_types__
         tn = ccall(:jl_new_typename_in, Any, (Any, Any), name, mod)
         makenew = true
     end
@@ -661,41 +681,42 @@ function deserialize(s::SerializationState, ::Type{TypeName})
                            tn, super, parameters, names, types,
                            abstr, mutable, ninitialized)
         known_object_data[number] = tn
+        ty = tn.primary
+        ccall(:jl_set_const, Void, (Any, Any, Any), mod, name, ty)
+        if !isdefined(ty,:instance)
+            if isempty(parameters) && !abstr && size == 0 && (!mutable || isempty(names))
+                setfield!(ty, :instance, ccall(:jl_new_struct, Any, (Any,Any...), ty))
+            end
+        end
     end
     tag = Int32(read(s.io, UInt8)::UInt8)
     if tag != UNDEFREF_TAG
-        mt = handle_deserialize(s, tag)
+        mtname = handle_deserialize(s, tag)
+        defs = deserialize(s)
+        maxa = deserialize(s)
         if makenew
-            tn.mt = mt
+            tn.mt = ccall(:jl_new_method_table, Any, (Any, Any), name, mod)
+            tn.mt.name = mtname
+            tn.mt.defs = defs
+            tn.mt.max_args = maxa
+        end
+        tag = Int32(read(s.io, UInt8)::UInt8)
+        if tag != UNDEFREF_TAG
+            kws = handle_deserialize(s, tag)
+            if makenew
+                tn.mt.kwsorter = kws
+            end
         end
     end
 
     return tn
 end
 
-module __deserialized_types__
-end
-
 function deserialize_datatype(s::SerializationState)
     form = read(s.io, UInt8)::UInt8
     if (form&2) != 0
         tname = deserialize(s)::TypeName
-        if isdefined(tname.module, tname.name)
-            # TODO: confirm somehow that the types match
-            ty = getfield(tname.module, tname.name)
-        else
-            ty = tname.primary
-            # assign to a new name inside a special module, reset tname.module and tname.name
-            newname = gensym()
-            tname.module = __deserialized_types__
-            tname.name = newname
-            ccall(:jl_set_const, Void, (Any, Any, Any), __deserialized_types__, newname, ty)
-            if !isdefined(ty,:instance)
-                if isempty(ty.parameters) && !ty.abstract && ty.size == 0 && (!ty.mutable || isempty(tname.names))
-                    setfield!(ty, :instance, ccall(:jl_new_struct, Any, (Any,Any...), ty))
-                end
-            end
-        end
+        ty = tname.primary
     else
         name = deserialize(s)::Symbol
         mod = deserialize(s)::Module

src/alloc.c

@@ -535,7 +535,7 @@ jl_sym_t *jl_demangle_typename(jl_sym_t *s)
     return jl_symbol_n(&n[1], len);
 }
 
-jl_methtable_t *jl_new_method_table(jl_sym_t *name, jl_module_t *module)
+JL_DLLEXPORT jl_methtable_t *jl_new_method_table(jl_sym_t *name, jl_module_t *module)
 {
     jl_methtable_t *mt = (jl_methtable_t*)jl_gc_allocobj(sizeof(jl_methtable_t));
     jl_set_typeof(mt, jl_methtable_type);

src/macroexpand.scm

@@ -271,7 +271,7 @@
 
            ((localize)
             (let ((expr (cadr e))
-                  (lvars (cddr e)))
+                  (lvars (map unescape (cddr e))))
               (let ((vs (delete-duplicates
 			 (expr-find-all (lambda (v)
 					  (and (symbol? v) (or (memq v lvars)