implementing remote parallel load, closes #81

old load() is now called include()
this also makes it easy to work on #7
array constructor tweaks

j/base.j

@@ -135,43 +135,26 @@ end
 # we share Array with Base so we can add definitions to it
 const Array = eval(Base, :Array)
 
-Array{T}  (::Type{T}, d::(Integer,)) =
-    ccall(:jl_alloc_array_1d, Array{T,1}, (Any,Int), Array{T,1},
-          int(d[1]))
-Array{T}  (::Type{T}, d::(Integer,Integer)) =
-    ccall(:jl_alloc_array_2d, Array{T,2}, (Any,Int,Int), Array{T,2},
-          int(d[1]), int(d[2]))
-
-Array{T}  (::Type{T}, d::(Int,Int,Int)) =
-    ccall(:jl_new_array, Array{T,3}, (Any,Any), Array{T,3}, d)
-Array{T}  (::Type{T}, d::(Integer,Integer,Integer)) =
-    ccall(:jl_new_arrayv, Array{T,3}, (Any,Int...), Array{T,3},
-          int(d[1]), int(d[2]), int(d[3]))
-Array{T}  (::Type{T}, d::(Int,Int,Int,Int)) =
-    ccall(:jl_new_array, Array{T,4}, (Any,Any), Array{T,4}, d)
-Array{T}  (::Type{T}, d::(Integer,Integer,Integer,Integer)) =
-    ccall(:jl_new_arrayv, Array{T,4}, (Any,Int...), Array{T,4},
-          int(d[1]), int(d[2]), int(d[3]), int(d[4]))
-
-Array{T,N}(::Type{T}, d::NTuple{N,Integer}) =
-    ccall(:jl_new_array, Array{T,N}, (Any,Any), Array{T,N},
-          convert((Int...), d))
-Array{N}(T, d::NTuple{N,Integer}) =
-    ccall(:jl_new_array, Array{T,N}, (Any,Any), Array{T,N},
-          convert((Int...), d))
+Array{T,N}(::Type{T}, d::NTuple{N,Int}) =
+    ccall(:jl_new_array, Array{T,N}, (Any,Any), Array{T,N}, d)
+Array{N}(T, d::NTuple{N,Int}) =
+    ccall(:jl_new_array, Array{T,N}, (Any,Any), Array{T,N}, d)
+
+Array{T}(::Type{T}, m::Int) =
+    ccall(:jl_alloc_array_1d, Array{T,1}, (Any,Int), Array{T,1}, m)
+Array{T}(::Type{T}, m::Int,n::Int) =
+    ccall(:jl_alloc_array_2d, Array{T,2}, (Any,Int,Int), Array{T,2}, m,n)
+Array{T}(::Type{T}, m::Int,n::Int,o::Int) =
+    ccall(:jl_alloc_array_3d, Array{T,3}, (Any,Int,Int,Int), Array{T,3}, m,n,o)
+
+Array(T, d::Int...) = Array(T, d)
 
 Array{T}(::Type{T}, m::Integer) =
     ccall(:jl_alloc_array_1d, Array{T,1}, (Any,Int), Array{T,1},
           int(m))
 Array{T}(::Type{T}, m::Integer,n::Integer) =
     ccall(:jl_alloc_array_2d, Array{T,2}, (Any,Int,Int), Array{T,2},
           int(m), int(n))
-
-Array{T}(::Type{T}, m::Integer, n::Integer, o::Integer) =
-    ccall(:jl_new_arrayv, Array{T,3}, (Any,Int...), Array{T,3},
+Array{T}(::Type{T}, m::Integer,n::Integer,o::Integer) =
+    ccall(:jl_alloc_array_3d, Array{T,3}, (Any,Int,Int,Int), Array{T,3},
           int(m), int(n), int(o))
-Array{T}(::Type{T}, m::Integer, n::Integer, o::Integer, p::Integer) =
-    ccall(:jl_new_arrayv, Array{T,4}, (Any,Int...), Array{T,4},
-          int(m), int(n), int(o), int(p))
-
-Array(T, d::Integer...) = Array(T, d)

j/client.j

@@ -139,7 +139,7 @@ function process_options(args::Array{Any,1})
             eval(parse_input_line(args[i]))
         elseif args[i]=="-L"
             i+=1
-            load(args[i])
+            include(args[i])
         elseif args[i]=="-p"
             i+=1
             np = int32(args[i])
@@ -160,7 +160,7 @@ function process_options(args::Array{Any,1})
             global ARGS
             # remove julia's arguments
             ARGS = ARGS[i:end]
-            load(args[i])
+            include(args[i])
             exit(0)
         else
             error("unknown option: ", args[i])
@@ -194,8 +194,8 @@ function _start()
         global const VARIABLES = {}
 
         # Load customized startup
-        try load(strcat(getcwd(),"/startup.j")) end
-        try load(strcat(ENV["HOME"],"/.juliarc")) end
+        try include(strcat(getcwd(),"/startup.j")) end
+        try include(strcat(ENV["HOME"],"/.juliarc")) end
 
         (quiet,repl) = process_options(ARGS)
         if repl

j/multi.j

@@ -39,7 +39,7 @@
 ## @parallel (r) for i=1:n ... end -
 ##     parallel loop. the results from each iteration are reduced using (r).
 ##
-## @everywhere expr - run expr everywhere. useful for load().
+## @everywhere expr - run expr everywhere.
 
 # todo:
 # - more indexing
@@ -1083,7 +1083,7 @@ end
 addprocs_sge(n) = add_workers(PGRP, start_sge_workers(n))
 SGE(n) = addprocs_sge(n)
 
-#load("vcloud.j")
+#include("vcloud.j")
 
 ## global objects and collective operations ##
 

j/pcre.j

@@ -1,6 +1,6 @@
 ## low-level pcre interface ##
 
-load("pcre_h.j")
+include("pcre_h.j")
 
 _jl_libpcre = dlopen("libpcre")
 

j/regex.j

@@ -1,4 +1,4 @@
-load("pcre.j")
+include("pcre.j")
 
 ## object-oriented Regex interface ##
 

j/stage0.j

@@ -27,7 +27,7 @@ if true
     end
 end
 
-load("sysimg.j")
+include("sysimg.j")
 
 ccall(:jl_save_system_image, Void, (Ptr{Uint8},Ptr{Uint8}),
       cstring("sys0.ji"), cstring("j/start_image.j"))

j/stage1.j

@@ -1,6 +1,6 @@
 module NewSystem
 
-load("sysimg.j")
+include("sysimg.j")
 
 # invoke type inference, running the existing inference code on the new
 # inference code to cache an optimized version of it.

j/sysimg.j

@@ -1,99 +1,99 @@
 ## Load essential files and libraries
 
-load("base.j")
+include("base.j")
 
 # core operations & types
-load("range.j")
-load("tuple.j")
-load("cell.j")
-load("expr.j")
-load("error.j")
+include("range.j")
+include("tuple.j")
+include("cell.j")
+include("expr.j")
+include("error.j")
 
 # core numeric operations & types
-load("bool.j")
-load("number.j")
-load("int.j")
-load("float.j")
-load("pointer.j")
-load("char.j")
-load("operators.j")
-load("promotion.j")
-load("reduce.j")
-load("complex.j")
-load("rational.j")
+include("bool.j")
+include("number.j")
+include("int.j")
+include("float.j")
+include("pointer.j")
+include("char.j")
+include("operators.j")
+include("promotion.j")
+include("reduce.j")
+include("complex.j")
+include("rational.j")
 
 # core data structures (used by type inference)
-load("abstractarray.j")
-load("subarray.j")
-load("array.j")
-load("intset.j")
-load("table.j")
-load("set.j")
+include("abstractarray.j")
+include("subarray.j")
+include("array.j")
+include("intset.j")
+include("table.j")
+include("set.j")
 
 # compiler
-load("inference.j")
+include("inference.j")
 
 # I/O, strings & printing
-load("io.j")
+include("io.j")
 #set_current_output_stream(make_stdout_stream()) # for error reporting
-load("string.j")
-load("ascii.j")
-load("utf8.j")
-load("regex.j")
-load("show.j")
-load("grisu.j")
-load("printf.j")
+include("string.j")
+include("ascii.j")
+include("utf8.j")
+include("regex.j")
+include("show.j")
+include("grisu.j")
+include("printf.j")
 
 # system & environment
-load("libc.j")
-load("env.j")
-load("errno_h.j")
+include("libc.j")
+include("env.j")
+include("errno_h.j")
 
 # concurrency and parallelism
-load("iterator.j")
-load("task.j")
-load("process.j")
-load("serialize.j")
-load("multi.j")
+include("iterator.j")
+include("task.j")
+include("process.j")
+include("serialize.j")
+include("multi.j")
 
 # front end
-load("client.j")
+include("client.j")
 
 # core math functions
-load("intfuncs.j")
-load("floatfuncs.j")
-load("math.j")
-load("math_libm.j")
-load("sort.j")
-load("combinatorics.j")
-load("statistics.j")
+include("intfuncs.j")
+include("floatfuncs.j")
+include("math.j")
+include("math_libm.j")
+include("sort.j")
+include("combinatorics.j")
+include("statistics.j")
 
 # random number generation
-load("random.j")
+include("random.j")
 
 # sparse matrices
-load("sparse.j")
+include("sparse.j")
 
 # distributed arrays
-load("darray.j")
+include("darray.j")
 
 # utilities - version, timing, help, edit
-load("version.j")
-load("util.j")
-load("datafmt.j")
+include("version.j")
+include("util.j")
+include("datafmt.j")
 
 ## Load optional external libraries
 
 # linear algebra
-load("linalg.j")
-load("linalg_blas.j")
-load("linalg_lapack.j")
-load("linalg_arpack.j")
-load("linalg_suitesparse.j")
+include("linalg.j")
+include("linalg_blas.j")
+include("linalg_lapack.j")
+include("linalg_arpack.j")
+include("linalg_suitesparse.j")
 
 # signal processing
-load("signal.j")
-load("signal_fftw.j")
+include("signal.j")
+include("signal_fftw.j")
 
 
 # prime method cache with some things we know we'll need right after startup

j/util.j

@@ -123,6 +123,90 @@ edit(f::Function, t) = edit(function_loc(f,t)...)
 less(f::Function)    = less(function_loc(f)...)
 less(f::Function, t) = less(function_loc(f,t)...)
 
+# remote/parallel load
+
+function is_file_readable(path)
+    local f
+    try
+        f = open(cstring(path))
+    catch
+        return false
+    end
+    close(f)
+    return true
+end
+
+function find_in_path(fname)
+    if fname[1] == '/'
+        return fname
+    end
+    loadpath = { "", "$JULIA_HOME/", "$JULIA_HOME/j/" }
+    for pfx in loadpath
+        pfxd = strcat(pfx,fname)
+        if is_file_readable(pfxd)
+            return pfxd
+        end
+    end
+    return fname
+end
+
+begin
+local in_load = false
+local in_remote_load = false
+local load_dict = {}
+global load, remote_load
+
+load(fname::String) = load(cstring(fname))
+function load(fname::ByteString)
+    if in_load
+        path = find_in_path(fname)
+        push(load_dict, fname)
+        f = open(path)
+        push(load_dict, readall(f))
+        close(f)
+        include(path)
+        return
+    elseif in_remote_load
+        for i=1:2:length(load_dict)
+            if load_dict[i] == fname
+                return include_string(load_dict[i+1])
+            end
+        end
+    else
+        in_load = true
+        iserr, err = false, ()
+        try
+            load(fname)
+            for p = 1:nprocs()
+                if p != myid()
+                    remote_do(p, remote_load, load_dict)
+                end
+            end
+        catch e
+            iserr, err = true, e
+        end
+        load_dict = {}
+        in_load = false
+        if iserr throw(err); end
+    end
+end
+
+function remote_load(dict)
+    load_dict = dict
+    in_remote_load = true
+    try
+        load(dict[1])
+    catch e
+        in_remote_load = false
+        throw(e)
+    end
+    in_remote_load = false
+    nothing
+end
+end
+
+# help
+
 function parse_help(stream)
     helpdb = HashTable()
     for l = each_line(stream)