RFC: change lowering of ccall cconvert arguments and add Ref{T} type

.gitignore

@@ -14,6 +14,7 @@
 *.do
 *.o
 *.obj
+*.so
 *.dylib
 *.dSYM
 *.jl.cov

base/REPL.jl

@@ -259,7 +259,7 @@ end
 
 immutable LatexCompletions <: CompletionProvider; end
 
-bytestring_beforecursor(buf::IOBuffer) = bytestring(pointer(buf.data), buf.ptr-1)
+bytestring_beforecursor(buf::IOBuffer) = bytestring(buf.data[1:buf.ptr-1])
 
 function complete_line(c::REPLCompletionProvider, s)
     partial = bytestring_beforecursor(s.input_buffer)

base/array.jl

@@ -18,6 +18,29 @@ call{T}(::Type{Matrix{T}}, m::Integer, n::Integer) = Array{T}(m, n)
 
 ## Basic functions ##
 
+# convert Arrays to pointer arrays for ccall
+function call{P<:Ptr,T<:Ptr}(::Type{Ref{P}}, a::Array{T}) # Ref{P<:Ptr}(a::Array{T<:Ptr})
+    return RefArray(a) # effectively a no-op
+end
+function call{P<:Ptr,T}(::Type{Ref{P}}, a::Array{T}) # Ref{P<:Ptr}(a::Array)
+    if (!isbits(T) && T <: eltype(P))
+        # this Array already has the right memory layout for the requested Ref
+        return RefArray(a,1,false) # root something, so that this function is type-stable
+    else
+        ptrs = Array(P, length(a)+1)
+        roots = Array(Any, length(a))
+        for i = 1:length(a)
+            root = cconvert(P, a[i])
+            ptrs[i] = unsafe_convert(P, root)::P
+            roots[i] = root
+        end
+        ptrs[length(a)+1] = C_NULL
+        return RefArray(ptrs,1,roots)
+    end
+end
+cconvert{P<:Ptr,T<:Ptr}(::Union(Type{Ptr{P}},Type{Ref{P}}), a::Array{T}) = a
+cconvert{P<:Ptr}(::Union(Type{Ptr{P}},Type{Ref{P}}), a::Array) = Ref{P}(a)
+
 size(a::Array) = arraysize(a)
 size(a::Array, d) = arraysize(a, d)
 size(a::Matrix) = (arraysize(a,1), arraysize(a,2))

base/base.jl

@@ -52,16 +52,15 @@ convert{T}(::Type{(T...)}, x::Tuple) = cnvt_all(T, x...)
 cnvt_all(T) = ()
 cnvt_all(T, x, rest...) = tuple(convert(T,x), cnvt_all(T, rest...)...)
 
-
-ptr_arg_convert{T}(::Type{Ptr{T}}, x) = convert(T, x)
-ptr_arg_convert(::Type{Ptr{Void}}, x) = x
-
-# conversion used by ccall
-cconvert(T, x) = convert(T, x)
-# use the code in ccall.cpp to safely allocate temporary pointer arrays
-cconvert{T}(::Type{Ptr{Ptr{T}}}, a::Array) = a
-# convert strings to ByteString to pass as pointers
-cconvert{P<:Union(Int8,UInt8)}(::Type{Ptr{P}}, s::AbstractString) = bytestring(s)
+# conversions used by ccall
+ptr_arg_cconvert{T}(::Type{Ptr{T}}, x) = cconvert(T, x)
+ptr_arg_unsafe_convert{T}(::Type{Ptr{T}}, x) = unsafe_convert(T, x)
+ptr_arg_unsafe_convert(::Type{Ptr{Void}}, x) = x
+
+cconvert(T::Type, x) = convert(T, x) # do the conversion eagerly in most cases
+cconvert{P<:Ptr}(::Type{P}, x) = x # but defer the conversion to Ptr to unsafe_convert
+unsafe_convert{T}(::Type{T}, x::T) = x # unsafe_convert (like convert) defaults to assuming the convert occurred
+unsafe_convert{P<:Ptr}(::Type{P}, x::Ptr) = convert(P, x)
 
 reinterpret{T,S}(::Type{T}, x::S) = box(T,unbox(S,x))
 

base/boot.jl

@@ -73,7 +73,8 @@
 #    contents::T
 #end
 
-#bitstype {32|64} Ptr{T}
+#abstract Ref{T}
+#bitstype {32|64} Ptr{T} <: Ref{T}
 
 # types for the front end
 
@@ -125,7 +126,7 @@ export
     Module, Symbol, Task, Array, GenSym,
     # numeric types
     Bool, FloatingPoint, Float16, Float32, Float64, Number, Integer, Int, Int8, Int16,
-    Int32, Int64, Int128, Ptr, Real, Signed, UInt, UInt8, UInt16, UInt32,
+    Int32, Int64, Int128, Ref, Ptr, Real, Signed, UInt, UInt8, UInt16, UInt32,
     UInt64, UInt128, Unsigned,
     # string types
     Char, ASCIIString, ByteString, DirectIndexString, AbstractString, UTF8String,

base/deprecated.jl

@@ -301,6 +301,21 @@ function subtypetree(x::DataType, level=-1)
     (level == 0 ? (x, []) : (x, Any[subtypetree(y, level-1) for y in subtypes(x)]))
 end
 
+function unsafe_convert{P}(::Type{P}, x)
+    P<:Ptr || throw(MethodError(unsafe_convert, (Type{P}, x)))
+    depwarn("convert(::Type{Ptr}, ::$(typeof(x))) methods should be converted to be methods of unsafe_convert", :unsafe_convert)
+    return convert(P, x)
+end
+
+function convert{T}(::Type{Ptr{T}}, x::Integer)
+    depwarn("converting integers to pointers is discontinued", :convert)
+    box(Ptr{T},unbox(UInt,UInt(x)))
+end
+function convert{T}(::Type{Ptr{T}}, x::Signed)
+    depwarn("converting signed numbers to pointers is discontinued", :convert)
+    box(Ptr{T},unbox(Int,Int(x)))
+end
+
 # 8898
 @deprecate precision(x::DateTime) eps(x)
 @deprecate precision(x::Date) eps(x)

base/fs.jl

@@ -63,8 +63,9 @@ end
 
 isopen(f::Union(File,AsyncFile)) = f.open
 
+# Not actually a pointer, but that's how we pass it through the C API so it's fine
+uvhandle(file::File) = convert(Ptr{Void}, file.handle % UInt)
 uvtype(::File) = Base.UV_RAW_FD
-uvhandle(file::File) = file.handle
 
 _uv_fs_result(req) = ccall(:jl_uv_fs_result,Int32,(Ptr{Void},),req)
 

base/gmp.jl

@@ -82,7 +82,7 @@ function Base.parseint_nocheck(::Type{BigInt}, s::AbstractString, base::Int)
     z = BigInt()
     err = ccall((:__gmpz_set_str, :libgmp),
                Int32, (Ptr{BigInt}, Ptr{UInt8}, Int32),
-               &z, convert(Ptr{UInt8},SubString(s,i)), base)
+               &z, SubString(s,i), base)
     err == 0 || throw(ArgumentError("invalid BigInt: $(repr(s))"))
     return sgn < 0 ? -z : z
 end

base/inference.jl

@@ -112,7 +112,20 @@ t_func[fpiseq] = (2, 2, cmp_tfunc)
 t_func[fpislt] = (2, 2, cmp_tfunc)
 t_func[nan_dom_err] = (2, 2, (a, b)->a)
 t_func[eval(Core.Intrinsics,:ccall)] =
-    (3, Inf, (fptr, rt, at, a...)->(isType(rt) ? rt.parameters[1] : Any))
+    (3, Inf, function(fptr, rt, at, a...)
+        if !isType(rt)
+            return Any
+        end
+        t = rt.parameters[1]
+        if isa(t,DataType) && is((t::DataType).name,Ref.name)
+            t = t.parameters[1]
+            if is(t,Any)
+                return Union() # a return type of Box{Any} is invalid
+            end
+            return t
+        end
+        return t
+    end)
 t_func[eval(Core.Intrinsics,:llvmcall)] =
     (3, Inf, (fptr, rt, at, a...)->(isType(rt) ? rt.parameters[1] :
                                     isa(rt,Tuple) ? map(x->x.parameters[1],rt) : Any))
@@ -2119,8 +2132,7 @@ function is_pure_builtin(f)
              f === Intrinsics.pointerset || # this one is never effect-free
              f === Intrinsics.ccall ||      # this one is never effect-free
              f === Intrinsics.llvmcall ||   # this one is never effect-free
-             f === Intrinsics.jl_alloca ||  # this one is volatile, TODO: possibly also effect-free?
-             f === Intrinsics.pointertoref) # this one is volatile
+             f === Intrinsics.jl_alloca)
             return true
         end
     end
@@ -2364,7 +2376,7 @@ function inlineable(f::ANY, e::Expr, atypes::Tuple, sv::StaticVarInfo, enclosing
     if incompletematch
         cost *= 4
     end
-    if is(f, next) || is(f, done)
+    if is(f, next) || is(f, done) || is(f, unsafe_convert) || is(f, cconvert)
         cost /= 4
     end
     if !inline_worthy(body, cost)

base/io.jl

@@ -96,7 +96,7 @@ function write(s::IO, p::Ptr, n::Integer)
 end
 
 function write(io::IO, s::Symbol)
-    pname = convert(Ptr{UInt8}, s)
+    pname = unsafe_convert(Ptr{UInt8}, s)
     write(io, pname, int(ccall(:strlen, Csize_t, (Ptr{UInt8},), pname)))
 end
 

base/iostream.jl

@@ -22,7 +22,7 @@ function IOStream(name::AbstractString, finalize::Bool)
 end
 IOStream(name::AbstractString) = IOStream(name, true)
 
-convert(T::Type{Ptr{Void}}, s::IOStream) = convert(T, s.ios)
+unsafe_convert(T::Type{Ptr{Void}}, s::IOStream) = convert(T, pointer(s.ios))
 show(io::IO, s::IOStream) = print(io, "IOStream(", s.name, ")")
 fd(s::IOStream) = int(ccall(:jl_ios_fd, Clong, (Ptr{Void},), s.ios))
 stat(s::IOStream) = stat(fd(s))

base/libc.jl

@@ -28,7 +28,7 @@ type TmStruct
         t = floor(t)
         tm = TmStruct()
         # TODO: add support for UTC via gmtime_r()
-        ccall(:localtime_r, Ptr{Void}, (Ptr{Int}, Ptr{Void}), &t, &tm)
+        ccall(:localtime_r, Ptr{TmStruct}, (Ptr{Int}, Ptr{TmStruct}), &t, &tm)
         return tm
     end
 end
@@ -37,18 +37,18 @@ strftime(t) = strftime("%c", t)
 strftime(fmt::AbstractString, t::Real) = strftime(fmt, TmStruct(t))
 function strftime(fmt::AbstractString, tm::TmStruct)
     timestr = Array(UInt8, 128)
-    n = ccall(:strftime, Int, (Ptr{UInt8}, Int, Ptr{UInt8}, Ptr{Void}),
+    n = ccall(:strftime, Int, (Ptr{UInt8}, Int, Ptr{UInt8}, Ptr{TmStruct}),
               timestr, length(timestr), fmt, &tm)
     if n == 0
         return ""
     end
-    bytestring(convert(Ptr{UInt8},timestr))
+    bytestring(pointer(timestr), n)
 end
 
 strptime(timestr::AbstractString) = strptime("%c", timestr)
 function strptime(fmt::AbstractString, timestr::AbstractString)
     tm = TmStruct()
-    r = ccall(:strptime, Ptr{UInt8}, (Ptr{UInt8}, Ptr{UInt8}, Ptr{Void}),
+    r = ccall(:strptime, Ptr{UInt8}, (Ptr{UInt8}, Ptr{UInt8}, Ptr{TmStruct}),
               timestr, fmt, &tm)
     # the following would tell mktime() that this is a local time, and that
     # it should try to guess the timezone. not sure if/how this should be
@@ -62,13 +62,13 @@ function strptime(fmt::AbstractString, timestr::AbstractString)
         # if we didn't explicitly parse the weekday or year day, use mktime
         # to fill them in automatically.
         if !ismatch(r"([^%]|^)%(a|A|j|w|Ow)", fmt)
-            ccall(:mktime, Int, (Ptr{Void},), &tm)
+            ccall(:mktime, Int, (Ptr{TmStruct},), &tm)
         end
     end
     tm
 end
 
-time(tm::TmStruct) = float64(ccall(:mktime, Int, (Ptr{Void},), &tm))
+time(tm::TmStruct) = float64(ccall(:mktime, Int, (Ptr{TmStruct},), &tm))
 
 ## process-related functions ##
 
@@ -81,7 +81,7 @@ function gethostname()
     @unix_only err=ccall(:gethostname, Int32, (Ptr{UInt8}, UInt), hn, length(hn))
     @windows_only err=ccall(:gethostname, stdcall, Int32, (Ptr{UInt8}, UInt32), hn, length(hn))
     systemerror("gethostname", err != 0)
-    bytestring(convert(Ptr{UInt8},hn))
+    bytestring(pointer(hn))
 end
 
 ## Memory related ##

base/linalg/tridiag.jl

@@ -127,7 +127,7 @@ setindex!(a::ZeroOffsetVector, x, i) = a.data[i+1]=x
 #Implements the inverse using the recurrence relation between principal minors
 # a, b, c are assumed to be the subdiagonal, diagonal, and superdiagonal of
 # a tridiagonal matrix.
-#Ref:
+#Reference:
 #    R. Usmani, "Inversion of a tridiagonal Jacobi matrix",
 #    Linear Algebra and its Applications 212-213 (1994), pp.413-414
 #    doi:10.1016/0024-3795(94)90414-6

base/multi.jl

@@ -722,7 +722,7 @@ fetch_ref(rid) = wait_full(lookup_ref(rid))
 fetch(r::RemoteRef) = call_on_owner(fetch_ref, r)
 fetch(x::ANY) = x
 
-# storing a value to a Ref
+# storing a value to a RemoteRef
 function put!(rv::RemoteValue, val::ANY)
     wait_empty(rv)
     rv.result = val
@@ -772,7 +772,7 @@ function deliver_result(sock::IO, msg, oid, value)
     end
 end
 
-# notify waiters that a certain job has finished or Ref has been emptied
+# notify waiters that a certain job has finished or RemoteRef has been emptied
 notify_full (rv::RemoteValue) = notify(rv.full, work_result(rv))
 notify_empty(rv::RemoteValue) = notify(rv.empty)
 

base/pcre.jl

@@ -57,7 +57,7 @@ function info{T}(
     regex::Ptr{Void},
     extra::Ptr{Void}, what::Integer, ::Type{T}
 )
-    buf = Array(UInt8,sizeof(T))
+    buf = zeros(UInt8,sizeof(T))
     ret = ccall((:pcre_fullinfo, :libpcre), Int32,
                 (Ptr{Void}, Ptr{Void}, Int32, Ptr{UInt8}),
                 regex, extra, what, buf)
@@ -71,7 +71,7 @@ function info{T}(
 end
 
 function config{T}(what::Integer, ::Type{T})
-    buf = Array(UInt8, sizeof(T))
+    buf = zeros(UInt8, sizeof(T))
     ret = ccall((:pcre_config, :libpcre), Int32,
                 (Int32, Ptr{UInt8}),
                 what, buf)
@@ -84,7 +84,8 @@ end
 
 function compile(pattern::AbstractString, options::Integer)
     errstr = Array(Ptr{UInt8},1)
-    erroff = Array(Int32,1)
+    errstr[1] = C_NULL
+    erroff = zeros(Int32,1)
     re_ptr = ccall((:pcre_compile, :libpcre), Ptr{Void},
                     (Ptr{UInt8}, Int32, Ptr{Ptr{UInt8}}, Ptr{Int32}, Ptr{UInt8}),
                     pattern, options, errstr, erroff, C_NULL)
@@ -100,6 +101,7 @@ end
 function study(regex::Ptr{Void}, options::Integer)
     # NOTE: options should always be zero in current PCRE
     errstr = Array(Ptr{UInt8},1)
+    errstr[1] = C_NULL
     extra = ccall((:pcre_study, :libpcre), Ptr{Void},
                   (Ptr{Void}, Int32, Ptr{Ptr{UInt8}}),
                   regex, options, errstr)

base/pointer.jl

@@ -7,27 +7,24 @@ convert{T<:Union(Int,UInt)}(::Type{T}, x::Ptr) = box(T, unbox(Ptr,x))
 convert{T<:Integer}(::Type{T}, x::Ptr) = convert(T,unsigned(x))
 
 # integer to pointer
-convert{T}(::Type{Ptr{T}}, x::Integer) = box(Ptr{T},unbox(UInt,UInt(x)))
-convert{T}(::Type{Ptr{T}}, x::Signed) = box(Ptr{T},unbox(Int,Int(x)))
+convert{T}(::Type{Ptr{T}}, x::UInt) = box(Ptr{T},unbox(UInt,UInt(x)))
+convert{T}(::Type{Ptr{T}}, x::Int) = box(Ptr{T},unbox(Int,Int(x)))
 
 # pointer to pointer
 convert{T}(::Type{Ptr{T}}, p::Ptr{T}) = p
 convert{T}(::Type{Ptr{T}}, p::Ptr) = box(Ptr{T}, unbox(Ptr,p))
 
-# object to pointer
-convert(::Type{Ptr{UInt8}}, x::Symbol) = ccall(:jl_symbol_name, Ptr{UInt8}, (Any,), x)
-convert(::Type{Ptr{Int8}}, x::Symbol) = ccall(:jl_symbol_name, Ptr{Int8}, (Any,), x)
-convert(::Type{Ptr{UInt8}}, s::ByteString) = convert(Ptr{UInt8}, s.data)
-convert(::Type{Ptr{Int8}}, s::ByteString) = convert(Ptr{Int8}, s.data)
+# object to pointer (when used with ccall)
+unsafe_convert(::Type{Ptr{UInt8}}, x::Symbol) = ccall(:jl_symbol_name, Ptr{UInt8}, (Any,), x)
+unsafe_convert(::Type{Ptr{Int8}}, x::Symbol) = ccall(:jl_symbol_name, Ptr{Int8}, (Any,), x)
+unsafe_convert(::Type{Ptr{UInt8}}, s::ByteString) = unsafe_convert(Ptr{UInt8}, s.data)
+unsafe_convert(::Type{Ptr{Int8}}, s::ByteString) = unsafe_convert(Ptr{Int8}, s.data)
+# convert strings to ByteString to pass as pointers
+cconvert(::Type{Ptr{UInt8}}, s::AbstractString) = bytestring(s)
+cconvert(::Type{Ptr{Int8}}, s::AbstractString) = bytestring(s)
 
-convert{T}(::Type{Ptr{T}}, a::Array{T}) = ccall(:jl_array_ptr, Ptr{T}, (Any,), a)
-convert(::Type{Ptr{Void}}, a::Array) = ccall(:jl_array_ptr, Ptr{Void}, (Any,), a)
-
-# note: these definitions don't mean any AbstractArray is convertible to
-# pointer. they just map the array element type to the pointer type for
-# convenience in cases that work.
-pointer{T}(x::AbstractArray{T}) = convert(Ptr{T},x)
-pointer{T}(x::AbstractArray{T}, i::Integer) = convert(Ptr{T},x) + (i-1)*elsize(x)
+unsafe_convert{T}(::Type{Ptr{T}}, a::Array{T}) = ccall(:jl_array_ptr, Ptr{T}, (Any,), a)
+unsafe_convert(::Type{Ptr{Void}}, a::Array) = ccall(:jl_array_ptr, Ptr{Void}, (Any,), a)
 
 # unsafe pointer to array conversions
 pointer_to_array(p, d::Integer, own=false) = pointer_to_array(p, (d,), own)
@@ -52,8 +49,9 @@ unsafe_store!{T}(p::Ptr{T}, x, i::Integer) = pointerset(p, convert(T,x), Int(i))
 unsafe_store!{T}(p::Ptr{T}, x) = pointerset(p, convert(T,x), 1)
 
 # convert a raw Ptr to an object reference, and vice-versa
-unsafe_pointer_to_objref(p::Ptr) = pointertoref(unbox(Ptr{Void},p))
-pointer_from_objref(x::Any) = ccall(:jl_value_ptr, Ptr{Void}, (Any,), x)
+unsafe_pointer_to_objref(x::Ptr) = ccall(:jl_value_ptr, Any, (Ptr{Void},), x)
+pointer_from_objref(x::ANY) = ccall(:jl_value_ptr, Ptr{Void}, (Any,), x)
+data_pointer_from_objref(x::ANY) = pointer_from_objref(x)::Ptr{Void}+Core.sizeof(Int)
 
 integer(x::Ptr) = convert(UInt, x)
 unsigned(x::Ptr) = convert(UInt, x)