change lowering of ccall cconvert arguments

ccall(:a, B, (C,), d) is now lowered as:
  d#1 = cconvert_gcroot(C, d)
  Expr(:call, :ccall, :a, B, (C,), cconvert(C, d#1), d#1)

This allows implementation of the special-case ccall lowering logic in Julia
and still create a gcroot for it as needed

the Ref{T} type is the new supertype of Ptr{T}. the ccall emission
turns a Ref{T} into a Ptr{T}, as if the &addressOf operator had been
applied. the specific rules for this will be moved from the current
gist to the documentation at a later point.

.gitignore

@@ -14,6 +14,7 @@
 *.do
 *.o
 *.obj
+*.so
 *.dylib
 *.dSYM
 *.jl.cov

base/abstractarray.jl

@@ -47,6 +47,17 @@ else
     end
 end
 
+# convert Arrays to pointer arrays for ccall
+cconvert_gcroot{P<:Ptr}(::Type{Ptr{P}}, a::Array{P}) = a
+function cconvert_gcroot{P<:Ptr}(::Type{Ptr{P}}, a::Array)
+    ptrs = Array(P, length(a)+1)
+    for i = 1:length(a)
+        ptrs[i] = convert(P, a[i])
+    end
+    ptrs[length(a)+1] = C_NULL
+    return ptrs
+end
+
 size{T,n}(t::AbstractArray{T,n}, d) = d <= n ? size(t)[d] : 1
 size(x, d1::Integer, d2::Integer, dx::Integer...) = tuple(size(x, d1), size(x, d2, dx...)...)
 eltype{T}(::Type{AbstractArray{T}}) = T

base/base.jl

@@ -52,16 +52,12 @@ convert{T}(::Type{(T...)}, x::Tuple) = cnvt_all(T, x...)
 cnvt_all(T) = ()
 cnvt_all(T, x, rest...) = tuple(convert(T,x), cnvt_all(T, rest...)...)
 
-
-ptr_arg_convert{T}(::Type{Ptr{T}}, x) = convert(T, x)
-ptr_arg_convert(::Type{Ptr{Void}}, x) = x
-
-# conversion used by ccall
-cconvert(T, x) = convert(T, x)
-# use the code in ccall.cpp to safely allocate temporary pointer arrays
-cconvert{T}(::Type{Ptr{Ptr{T}}}, a::Array) = a
-# convert strings to ByteString to pass as pointers
-cconvert{P<:Union(Int8,UInt8)}(::Type{Ptr{P}}, s::AbstractString) = bytestring(s)
+# conversions used by ccall
+ptr_arg_cconvert_gcroot{T}(::Type{Ptr{T}}, x) = cconvert_gcroot(T, x)
+ptr_arg_cconvert{T}(::Type{Ptr{T}}, x) = cconvert(T, x)
+ptr_arg_cconvert(::Type{Ptr{Void}}, x) = x
+cconvert_gcroot(T, x) = x
+cconvert(T, x) = convert(T,x)
 
 reinterpret{T,S}(::Type{T}, x::S) = box(T,unbox(S,x))
 

base/boot.jl

@@ -73,7 +73,8 @@
 #    contents::T
 #end
 
-#bitstype {32|64} Ptr{T}
+#abstract Ref{T}
+#bitstype {32|64} Ptr{T} <: Ref{T}
 
 # types for the front end
 
@@ -125,7 +126,7 @@ export
     Module, Symbol, Task, Array, GenSym,
     # numeric types
     Bool, FloatingPoint, Float16, Float32, Float64, Number, Integer, Int, Int8, Int16,
-    Int32, Int64, Int128, Ptr, Real, Signed, UInt, UInt8, UInt16, UInt32,
+    Int32, Int64, Int128, Ref, Ptr, Real, Signed, UInt, UInt8, UInt16, UInt32,
     UInt64, UInt128, Unsigned,
     # string types
     Char, ASCIIString, ByteString, DirectIndexString, AbstractString, UTF8String,

base/inference.jl

@@ -112,7 +112,20 @@ t_func[fpiseq] = (2, 2, cmp_tfunc)
 t_func[fpislt] = (2, 2, cmp_tfunc)
 t_func[nan_dom_err] = (2, 2, (a, b)->a)
 t_func[eval(Core.Intrinsics,:ccall)] =
-    (3, Inf, (fptr, rt, at, a...)->(isType(rt) ? rt.parameters[1] : Any))
+    (3, Inf, function(fptr, rt, at, a...)
+        if !isType(rt)
+            return Any
+        end
+        t = rt.parameters[1]
+        if isa(t,DataType) && is((t::DataType).name,Ref.name)
+            t = t.parameters[1]
+            if is(t,Any)
+                return Union() # a return type of Box{Any} is invalid
+            end
+            return t
+        end
+        return t
+    end)
 t_func[eval(Core.Intrinsics,:llvmcall)] =
     (3, Inf, (fptr, rt, at, a...)->(isType(rt) ? rt.parameters[1] :
                                     isa(rt,Tuple) ? map(x->x.parameters[1],rt) : Any))

base/pcre.jl

@@ -57,7 +57,7 @@ function info{T}(
     regex::Ptr{Void},
     extra::Ptr{Void}, what::Integer, ::Type{T}
 )
-    buf = Array(UInt8,sizeof(T))
+    buf = zeros(UInt8,sizeof(T))
     ret = ccall((:pcre_fullinfo, :libpcre), Int32,
                 (Ptr{Void}, Ptr{Void}, Int32, Ptr{UInt8}),
                 regex, extra, what, buf)
@@ -71,7 +71,7 @@ function info{T}(
 end
 
 function config{T}(what::Integer, ::Type{T})
-    buf = Array(UInt8, sizeof(T))
+    buf = zeros(UInt8, sizeof(T))
     ret = ccall((:pcre_config, :libpcre), Int32,
                 (Int32, Ptr{UInt8}),
                 what, buf)
@@ -84,7 +84,8 @@ end
 
 function compile(pattern::AbstractString, options::Integer)
     errstr = Array(Ptr{UInt8},1)
-    erroff = Array(Int32,1)
+    errstr[1] = C_NULL
+    erroff = zeros(Int32,1)
     re_ptr = ccall((:pcre_compile, :libpcre), Ptr{Void},
                     (Ptr{UInt8}, Int32, Ptr{Ptr{UInt8}}, Ptr{Int32}, Ptr{UInt8}),
                     pattern, options, errstr, erroff, C_NULL)
@@ -100,6 +101,7 @@ end
 function study(regex::Ptr{Void}, options::Integer)
     # NOTE: options should always be zero in current PCRE
     errstr = Array(Ptr{UInt8},1)
+    errstr[1] = C_NULL
     extra = ccall((:pcre_study, :libpcre), Ptr{Void},
                   (Ptr{Void}, Int32, Ptr{Ptr{UInt8}}),
                   regex, options, errstr)

base/pointer.jl

@@ -19,16 +19,13 @@ convert(::Type{Ptr{UInt8}}, x::Symbol) = ccall(:jl_symbol_name, Ptr{UInt8}, (Any
 convert(::Type{Ptr{Int8}}, x::Symbol) = ccall(:jl_symbol_name, Ptr{Int8}, (Any,), x)
 convert(::Type{Ptr{UInt8}}, s::ByteString) = convert(Ptr{UInt8}, s.data)
 convert(::Type{Ptr{Int8}}, s::ByteString) = convert(Ptr{Int8}, s.data)
+# convert strings to ByteString to pass as pointers
+cconvert_gcroot(::Type{Ptr{UInt8}}, s::AbstractString) = bytestring(s)
+cconvert_gcroot(::Type{Ptr{Int8}}, s::AbstractString) = bytestring(s)
 
 convert{T}(::Type{Ptr{T}}, a::Array{T}) = ccall(:jl_array_ptr, Ptr{T}, (Any,), a)
 convert(::Type{Ptr{Void}}, a::Array) = ccall(:jl_array_ptr, Ptr{Void}, (Any,), a)
 
-# note: these definitions don't mean any AbstractArray is convertible to
-# pointer. they just map the array element type to the pointer type for
-# convenience in cases that work.
-pointer{T}(x::AbstractArray{T}) = convert(Ptr{T},x)
-pointer{T}(x::AbstractArray{T}, i::Integer) = convert(Ptr{T},x) + (i-1)*elsize(x)
-
 # unsafe pointer to array conversions
 pointer_to_array(p, d::Integer, own=false) = pointer_to_array(p, (d,), own)
 function pointer_to_array{T,N}(p::Ptr{T}, dims::NTuple{N,Int}, own::Bool=false)
@@ -53,7 +50,8 @@ unsafe_store!{T}(p::Ptr{T}, x) = pointerset(p, convert(T,x), 1)
 
 # convert a raw Ptr to an object reference, and vice-versa
 unsafe_pointer_to_objref(x::Ptr) = ccall(:jl_value_ptr, Any, (Ptr{Void},), x)
-pointer_from_objref(x::Any) = ccall(:jl_value_ptr, Ptr{Void}, (Any,), x)
+pointer_from_objref(x::ANY) = ccall(:jl_value_ptr, Ptr{Void}, (Any,), x)
+data_pointer_from_objref(x::ANY) = pointer_from_objref(x)::Ptr{Void}+Core.sizeof(Int)
 
 integer(x::Ptr) = convert(UInt, x)
 unsigned(x::Ptr) = convert(UInt, x)

base/refpointer.jl

@@ -0,0 +1,121 @@
+### General Methods for Ref{T} type
+
+Base.eltype{T}(x::Type{Ref{T}}) = T
+Base.convert{T}(::Type{Ref{T}}, x::Ref{T}) = x
+
+# create Ref objects for general object conversion
+@inline Base.cconvert_gcroot{T}(::Type{Ref{T}}, x) = convert(Ref{T}, x)
+@inline Base.cconvert{T}(::Type{Ref{T}}, x) = convert(Ptr{T}, x)
+
+### Methods for a Ref object that can store a single value
+
+type RefValue{T} <: Ref{T}
+    x::T
+    RefValue() = new()
+    RefValue(x) = new(x)
+end
+Base.convert{T}(::Type{Ref{T}}, x) = RefValue{T}(x)
+Base.call{T}(::Type{Ref{T}}) = RefValue{T}()
+
+Ref(x::Ref) = x
+Ref{T}(x::T) = RefValue{T}(x)
+Ref{T}(x::Ptr{T}, i::Integer=1) = x + (i-1)*Core.sizeof(T)
+Ref(x, i::Integer) = (i != 1 && error("Object only has one element"); Ref(x))
+
+@inline function Base.convert{T}(P::Type{Ptr{T}}, b::RefValue{T})
+    if isbits(T)
+        return convert(P, data_pointer_from_objref(b))
+    else
+        return convert(P, data_pointer_from_objref(b.x))
+    end
+end
+@inline function Base.convert(P::Type{Ptr{Any}}, b::RefValue{Any})
+    return convert(P, data_pointer_from_objref(b))
+end
+@inline Base.convert{T}(::Type{Ptr{Void}}, b::RefValue{T}) = Base.convert(Ptr{Void}, Base.convert(Ptr{T}, b))
+
+### Methods for a Ref object that is backed by an array at index i
+
+# note: the following type definitions don't mean any AbstractArray is convertible to
+# a data Ref. they just map the array element type to the pointer type for
+# convenience in cases that work.
+pointer{T}(x::AbstractArray{T}) = convert(Ptr{T},x)
+pointer{T}(x::AbstractArray{T}, i::Integer) = convert(Ptr{T},x) + (i-1)*elsize(x)
+
+immutable RefArray{T, A<:AbstractArray} <: Ref{T}
+    x::A
+    i::Int
+    RefArray(x,i) = (@assert(eltype(A) == T); new(x,i))
+end
+convert{T}(::Type{Ref{T}}, x::AbstractArray{T}) = RefArray{T,typeof(x)}(x, 1)
+Ref{T}(x::AbstractArray{T}, i::Integer=1) = RefArray{T,typeof(x)}(x, i)
+
+@inline function Base.convert{T}(P::Type{Ptr{T}}, b::RefArray{T})
+    if isbits(T)
+        convert(P, pointer(b.x, b.i))
+    else
+        convert(P, data_pointer_from_objref(b.x[b.i]))
+    end
+end
+@inline function Base.convert(P::Type{Ptr{Any}}, b::RefArray{Any})
+    return convert(P, pointer(b.x, b.i))
+end
+@inline Base.convert{T}(::Type{Ptr{Void}}, b::RefArray{T}) = Base.convert(Ptr{Void}, Base.convert(Ptr{T}, b))
+
+### Methods for a Ref object that is backed by an array at index (i...)
+
+immutable RefArrayND{T, A<:AbstractArray} <: Ref{T}
+    x::A
+    i::(Int...)
+    RefArrayND(x,i) = (@assert(eltype(A) == T); new(x,i))
+end
+Ref{A<:AbstractArray}(x::A, i::Integer...) = RefArrayND{eltype(A),A}(x, i)
+Ref{A<:AbstractArray}(x::A, i::(Integer...)) = RefArrayND{eltype(A),A}(x, i)
+
+@inline function Base.convert{T}(P::Type{Ptr{T}}, b::RefArrayND{T})
+    if isbits(T)
+        convert(P, pointer(b.x, b.i...))
+    else
+        convert(P, data_pointer_from_objref(b.x[b.i...]))
+    end
+end
+@inline function Base.convert(P::Type{Ptr{Any}}, b::RefArrayND{Any})
+    return convert(P, pointer(b.x, b.i...))
+end
+@inline Base.convert{T}(::Type{Ptr{Void}}, b::RefArrayND{T}) = Base.convert(Ptr{Void}, Base.convert(Ptr{T}, b))
+
+### Methods for a Ref object that is backed by an array at index (Any...)
+
+immutable RefArrayI{T} <: Ref{T}
+    x::AbstractArray{T}
+    i::Tuple
+    RefArrayI(x,i::ANY) = (@assert(eltype(A) == T); new(x,i))
+end
+Ref{T}(x::AbstractArray{T}, i...) = RefArrayI{T}(x, i)
+Ref{T}(x::AbstractArray{T}, i::Tuple) = RefArrayI{T}(x, i)
+
+@inline function Base.convert{T}(P::Type{Ptr{T}}, b::RefArrayI{T})
+    if isbits(T)
+        convert(P, pointer(b.x, b.i...))
+    else
+        convert(P, data_pointer_from_objref(b.x[b.i...]))
+    end
+end
+@inline function Base.convert(P::Type{Ptr{Any}}, b::RefArrayI{Any})
+    return convert(P, pointer(b.x, b.i...))
+end
+@inline Base.convert{T}(::Type{Ptr{Void}}, b::RefArrayI{T}) = Base.convert(Ptr{Void}, Base.convert(Ptr{T}, b))
+
+###
+
+Base.getindex(b::RefValue) = b.x
+Base.getindex(b::RefArray) = b.x[b.i]
+Base.getindex(b::RefArrayND) = b.x[b.i...]
+Base.getindex(b::RefArrayI) = b.x[b.i...]
+
+Base.setindex!(b::RefValue, x) = (b.x = x; b)
+Base.setindex!(b::RefArray, x) = (b.x[b.i] = x; b)
+Base.setindex!(b::RefArrayND, x) = (b.x[b.i...] = x; b)
+Base.setindex!(b::RefArrayI, x) = (b.x[b.i...] = x; b)
+
+###

base/socket.jl

@@ -593,7 +593,8 @@ const _sizeof_uv_interface_address = ccall(:jl_uv_sizeof_interface_address,Int32
 
 function getipaddr()
     addr = Array(Ptr{UInt8},1)
-    count = Array(Int32,1)
+    addr[1] = C_NULL
+    count = zeros(Int32,1)
     lo_present = false
     err = ccall(:jl_uv_interface_addresses,Int32,(Ptr{Ptr{UInt8}},Ptr{Int32}),addr,count)
     addr, count = addr[1],count[1]

base/sysimg.jl

@@ -67,6 +67,7 @@ include("dict.jl")
 include("set.jl")
 include("hashing.jl")
 include("iterator.jl")
+include("refpointer.jl") #XXX: move this earlier when we don't need @inline annotations anymore
 
 # SIMD loops
 include("simdloop.jl")

src/alloc.c

@@ -54,6 +54,7 @@ jl_datatype_t *jl_typeerror_type;
 jl_datatype_t *jl_methoderror_type;
 jl_datatype_t *jl_loaderror_type;
 jl_datatype_t *jl_undefvarerror_type;
+jl_datatype_t *jl_ref_type;
 jl_datatype_t *jl_pointer_type;
 jl_datatype_t *jl_void_type;
 jl_datatype_t *jl_voidpointer_type;

src/builtins.c

@@ -1156,6 +1156,7 @@ void jl_init_primitives(void)
     add_builtin("IntrinsicFunction", (jl_value_t*)jl_intrinsic_type);
     add_builtin("Function", (jl_value_t*)jl_function_type);
     add_builtin("LambdaStaticData", (jl_value_t*)jl_lambda_info_type);
+    add_builtin("Ref", (jl_value_t*)jl_ref_type);
     add_builtin("Ptr", (jl_value_t*)jl_pointer_type);
     add_builtin("Box", (jl_value_t*)jl_box_type);
     add_builtin("Task", (jl_value_t*)jl_task_type);