work towards updating to DataFrames v0.11+ framework

i haven't been able to fix everything due to slight inconsistencies
between PooledDataArrays and CategoricalArrays behavior. See
JuliaData/CategoricalArrays.jl#117 for
details. Working towards a fix for GiovineItalia#1065

REQUIRE

@@ -4,8 +4,9 @@ Compat 0.18.0
 Compose 0.5.2
 Contour 0.1.1
 CoupledFields
-DataFrames 0.4.2
-DataArrays
+DataFrames 0.11.0
+DataArrays 0.7.0
+CategoricalArrays
 DataStructures
 Distributions
 Hexagons

src/Gadfly.jl

@@ -6,6 +6,7 @@ using Colors
 using Compat
 using Compose
 using DataArrays
+using CategoricalArrays
 using DataFrames
 using DataStructures
 using JSON

src/aesthetics.jl

@@ -1,8 +1,8 @@
 const NumericalOrCategoricalAesthetic =
-    Union{(Void), Vector, DataArray, PooledDataArray}
+    Union{(Void), Vector, DataArray, CategoricalArray}
 
 const CategoricalAesthetic =
-    Union{(Void), PooledDataArray}
+    Union{(Void), CategoricalArray}
 
 const NumericalAesthetic =
     Union{(Void), Matrix, Vector, DataArray}
@@ -287,18 +287,8 @@ function cat_aes_var!{T, U}(a::AbstractArray{T}, b::AbstractArray{U})
     return ab
 end
 
-function cat_aes_var!{T}(xs::PooledDataVector{T}, ys::PooledDataVector{T})
-    newpool = T[x for x in union(Set(xs.pool), Set(ys.pool))]
-    newdata = vcat(T[x for x in xs], T[y for y in ys])
-    PooledDataArray(newdata, newpool, [false for _ in newdata])
-end
-
-function cat_aes_var!{T, U}(xs::PooledDataVector{T}, ys::PooledDataVector{U})
-    V = promote_type(T, U)
-    newpool = V[x for x in union(Set(xs.pool), Set(ys.pool))]
-    newdata = vcat(V[x for x in xs], V[y for y in ys])
-    PooledDataArray(newdata, newpool, [false for _ in newdata])
-end
+cat_aes_var!{T}(xs::CategoricalArray{T}, ys::CategoricalArray{T}) = vcat(xs, ys)
+cat_aes_var!{T, U}(xs::CategoricalArray{T}, ys::CategoricalArray{U}) = vcat(xs, ys)
 
 
 # Summarizing aesthetics
@@ -333,11 +323,6 @@ function by_xy_group{T <: Union{Data, Aesthetics}}(aes::T, xgroup, ygroup,
 
     xgroup === nothing && ygroup === nothing && return aes_grid
 
-    make_pooled_data_array{T,U,V}(::Type{PooledDataArray{T,U,V}}, arr::AbstractArray) =
-            PooledDataArray(convert(Array{T}, arr))
-    make_pooled_data_array{T,U,V}(::Type{PooledDataArray{T,U,V}},
-            arr::PooledDataArray{T,U,V}) = arr
-
     for var in fieldnames(T)
         # Skipped aesthetics. Don't try to partition aesthetics for which it
         # makes no sense to pass on to subplots.
@@ -366,9 +351,8 @@ function by_xy_group{T <: Union{Data, Aesthetics}}(aes::T, xgroup, ygroup,
             end
 
             for i in 1:n, j in 1:m
-                if typeof(vals) <: PooledDataArray
-                    setfield!(aes_grid[i, j], var,
-                              make_pooled_data_array(typeof(vals), staging[i, j]))
+                if typeof(vals) <: CategoricalArray
+                    setfield!(aes_grid[i, j], var, CategoricalArray(staging[i, j]))
                 else
                     if !applicable(convert, typeof(vals), staging[i, j])
                         T2 = eltype(vals)

src/geom/bar.jl

@@ -76,7 +76,7 @@ function render_stacked_bar(geom::BarGeometry,
                                      aes::Gadfly.Aesthetics,
                                      orientation::Symbol)
     # preserve factor orders of pooled data arrays
-    if isa(aes.color, PooledDataArray)
+    if isa(aes.color, CategoricalArray)
         idxs = sortperm(aes.color.refs, rev=true)
     else
         idxs = 1:length(aes.color)
@@ -152,7 +152,7 @@ function render_dodged_bar(geom::BarGeometry,
                                     aes::Gadfly.Aesthetics,
                                     orientation::Symbol)
     # preserve factor orders of pooled data arrays
-    if isa(aes.color, PooledDataArray)
+    if isa(aes.color, CategoricalArray)
         idxs = sortperm(aes.color.refs, rev=true)
     else
         idxs = 1:length(aes.color)

src/geom/beeswarm.jl

@@ -16,7 +16,7 @@ function render(geom::BeeswarmGeometry, theme::Gadfly.Theme, aes::Gadfly.Aesthet
     Gadfly.assert_aesthetics_equal_length("Geom.point", aes,
                                           element_aesthetics(geom)...)
     default_aes = Gadfly.Aesthetics()
-    default_aes.color = PooledDataArray(RGBA{Float32}[theme.default_color])
+    default_aes.color = CategoricalArray(RGBA{Float32}[theme.default_color])
     default_aes.size = Measure[theme.point_size]
     aes = inherit(aes, default_aes)
     padding = 1.0mm

src/geom/boxplot.jl

@@ -27,7 +27,7 @@ function render(geom::BoxplotGeometry, theme::Gadfly.Theme, aes::Gadfly.Aestheti
                                      :upper_hinge, :upper_fence, :outliers)
 
     default_aes = Gadfly.Aesthetics()
-    default_aes.color = PooledDataArray(RGB{Float32}[theme.default_color])
+    default_aes.color = CategoricalArray(RGB{Float32}[theme.default_color])
     default_aes.x = Float64[0.5]
     aes = inherit(aes, default_aes)
 

src/geom/errorbar.jl

@@ -54,7 +54,7 @@ function render(geom::YErrorBarGeometry, theme::Gadfly.Theme, aes::Gadfly.Aesthe
                                           element_aesthetics(geom)...)
 
     default_aes = Gadfly.Aesthetics()
-    default_aes.color = PooledDataArray(RGB{Float32}[theme.default_color])
+    default_aes.color = CategoricalArray(RGB{Float32}[theme.default_color])
     aes = inherit(aes, default_aes)
     caplen = theme.errorbar_cap_length/2
     ttc, teb, tbc = subtags(geom.tag, :top_cap, :error_bar, :bottom_cap)
@@ -90,7 +90,7 @@ function render(geom::XErrorBarGeometry, theme::Gadfly.Theme, aes::Gadfly.Aesthe
 
     colored = aes.color != nothing
     default_aes = Gadfly.Aesthetics()
-    default_aes.color = PooledDataArray(RGB{Float32}[theme.default_color])
+    default_aes.color = CategoricalArray(RGB{Float32}[theme.default_color])
     aes = inherit(aes, default_aes)
     caplen = theme.errorbar_cap_length/2
     tlc, teb, trc = subtags(geom.tag, :left_cap, :error_bar, :right_cap)

src/geom/hexbin.jl

@@ -16,7 +16,7 @@ element_aesthetics(geom::HexagonalBinGeometry) = [:x, :y, :xsize, :ysize, :color
 
 function render(geom::HexagonalBinGeometry, theme::Gadfly.Theme, aes::Gadfly.Aesthetics)
     default_aes = Gadfly.Aesthetics()
-    default_aes.color = PooledDataArray(RGB{Float32}[theme.default_color])
+    default_aes.color = CategoricalArray(RGB{Float32}[theme.default_color])
     default_aes.xsize = [1.0]
     default_aes.ysize = [1.0]
     aes = inherit(aes, default_aes)
@@ -25,7 +25,7 @@ function render(geom::HexagonalBinGeometry, theme::Gadfly.Theme, aes::Gadfly.Aes
     Gadfly.assert_aesthetics_equal_length("Geom.hexbin", aes, :x, :y)
 
     n = length(aes.x)
-    visibility = Bool[!isna(c) for c in takestrict(cycle(aes.color), n)]
+    visibility = Bool[!ismissing(c) for c in takestrict(cycle(aes.color), n)]
     xs = aes.x[visibility]
     ys = aes.y[visibility]
     xsizes = collect(eltype(aes.xsize), takestrict(cycle(aes.xsize), n))[visibility]

src/geom/line.jl

@@ -58,7 +58,7 @@ function render(geom::LineGeometry, theme::Gadfly.Theme, aes::Gadfly.Aesthetics)
                                           element_aesthetics(geom)...)
 
     default_aes = Gadfly.Aesthetics()
-    default_aes.color = PooledDataArray(RGBA{Float32}[theme.default_color])
+    default_aes.color = CategoricalArray(RGBA{Float32}[theme.default_color])
     aes = inherit(aes, default_aes)
 
     ctx = context(order=geom.order)
@@ -131,7 +131,7 @@ function render(geom::LineGeometry, theme::Gadfly.Theme, aes::Gadfly.Aesthetics)
                       svgclass(classes))
 
     elseif length(aes.color) == 1 &&
-            !(isa(aes.color, PooledDataArray) && length(levels(aes.color)) > 1)
+            !(isa(aes.color, CategoricalArray) && length(levels(aes.color)) > 1)
         T = (Tuple{eltype(aes.x), eltype(aes.y)})
         points = T[(x, y) for (x, y) in zip(aes.x, aes.y)]
         geom.preserve_order || sort!(points, by=first)

src/geom/point.jl

@@ -25,7 +25,7 @@ function render(geom::PointGeometry, theme::Gadfly.Theme, aes::Gadfly.Aesthetics
 
     default_aes = Gadfly.Aesthetics()
     default_aes.shape = Function[Shape.circle]
-    default_aes.color = PooledDataArray(RGBA{Float32}[theme.default_color])
+    default_aes.color = CategoricalArray(RGBA{Float32}[theme.default_color])
     default_aes.size = Measure[theme.point_size]
     aes = inherit(aes, default_aes)
 

src/geom/polygon.jl

@@ -30,7 +30,7 @@ function render(geom::PolygonGeometry, theme::Gadfly.Theme,
     Gadfly.assert_aesthetics_defined("Geom.polygon", aes, :x, :y)
 
     default_aes = Gadfly.Aesthetics()
-    default_aes.color = PooledDataArray(RGBA{Float32}[theme.default_color])
+    default_aes.color = CategoricalArray(RGBA{Float32}[theme.default_color])
     aes = inherit(aes, default_aes)
 
     ctx = context(order=geom.order)
@@ -56,7 +56,7 @@ function render(geom::PolygonGeometry, theme::Gadfly.Theme,
             compose!(ctx, fill(nothing), stroke(cs))
         end
     elseif length(aes.color) == 1 &&
-            !(isa(aes.color, PooledDataArray) && length(levels(aes.color)) > 1)
+            !(isa(aes.color, CategoricalArray) && length(levels(aes.color)) > 1)
         compose!(ctx, Compose.polygon(polygon_points(aes.x, aes.y, geom.preserve_order), geom.tag))
         if geom.fill
             compose!(ctx, fill(aes.color[1]),

src/geom/rectbin.jl

@@ -42,7 +42,7 @@ element_aesthetics(::RectangularBinGeometry) =
 function render(geom::RectangularBinGeometry, theme::Gadfly.Theme, aes::Gadfly.Aesthetics)
 
     default_aes = Gadfly.Aesthetics()
-    default_aes.color = PooledDataArray(RGBA{Float32}[theme.default_color])
+    default_aes.color = CategoricalArray(RGBA{Float32}[theme.default_color])
     aes = inherit(aes, default_aes)
 
     Gadfly.assert_aesthetics_defined("RectangularBinGeometry", aes, :xmin, :xmax, :ymin, :ymax)

src/geom/ribbon.jl

@@ -17,13 +17,13 @@ function render(geom::RibbonGeometry, theme::Gadfly.Theme, aes::Gadfly.Aesthetic
                                           element_aesthetics(geom)...)
 
     default_aes = Gadfly.Aesthetics()
-    default_aes.color = PooledDataArray(RGB{Float32}[theme.default_color])
+    default_aes.color = CategoricalArray(RGB{Float32}[theme.default_color])
     aes = inherit(aes, default_aes)
 
     aes_x, aes_ymin, aes_ymax = concretize(aes.x, aes.ymin, aes.ymax)
 
     if length(aes.color) == 1 &&
-        !(isa(aes.color, PooledDataArray) && length(levels(aes.color)) > 1)
+        !(isa(aes.color, CategoricalArray) && length(levels(aes.color)) > 1)
         max_points = collect(zip(aes_x, aes_ymax))
         sort!(max_points, by=first)
 

src/geometry.jl

@@ -4,6 +4,7 @@ using Colors
 using Compat
 using Compose
 using DataArrays
+using CategoricalArrays
 using DataStructures
 using Gadfly
 using Measures

src/misc.jl

@@ -1,13 +1,13 @@
 #Is this usable data?
-isconcrete{T<:Number}(x::T) = !isna(x) && isfinite(x)
+isconcrete{T<:Number}(x::T) = !ismissing(x) && isfinite(x)
 isconcrete(x::(Irrational)) = true
-isconcrete(x) = !isna(x)
+isconcrete(x) = !ismissing(x)
 
 hasna(xs) = false
 
 function hasna(xs::AbstractDataArray)
     for x in xs
-        if isna(x)
+        if ismissing(x)
             return true
         end
     end

src/poetry.jl

@@ -155,7 +155,7 @@ A plot object.
 """
 
 function spy(M::AbstractMatrix, elements::ElementOrFunction...; mapping...)
-    is, js, values = _findnz(x->!isnan(x), M)
+    is, js, values = _findnz(x->!ismissingn(x), M)
     n,m = size(M)
     df = DataFrames.DataFrame(i=is, j=js, value=values)
     plot(df, x=:j, y=:i, color=:value,

src/scale.jl

@@ -4,6 +4,7 @@ using Colors
 using Compat
 using Compose
 using DataArrays
+using CategoricalArrays
 using DataStructures
 using Gadfly
 using Showoff
@@ -250,67 +251,26 @@ function apply_scale_typed!(ds, field, scale::ContinuousScale)
     end
 end
 
-# Reorder the levels of a pooled data array
-function reorder_levels(da::PooledDataArray, order::AbstractVector)
+# Reorder the levels of a categorical array
+function reorder_levels(da::CategoricalArray, order::AbstractVector)
     level_values = levels(da)
     length(order) != length(level_values) &&
             error("Discrete scale order is not of the same length as the data's levels.")
     permute!(level_values, order)
-    return PooledDataArray(da, level_values)
-end
-
-function discretize_make_pda(values::Vector, levels=nothing)
-    if levels == nothing
-        return PooledDataArray(values)
-    else
-        return PooledDataArray(convert(Vector{eltype(levels)}, values), levels)
-    end
-end
-
-function discretize_make_pda(values::DataArray, levels=nothing)
-    if levels == nothing
-        return PooledDataArray(values)
-    else
-        return PooledDataArray(convert(DataArray{eltype(levels)}, values), levels)
-    end
-end
-
-function discretize_make_pda(values::Range, levels=nothing)
-    if levels == nothing
-        return PooledDataArray(collect(values))
-    else
-        return PooledDataArray(collect(values), levels)
-    end
-end
-
-function discretize_make_pda(values::PooledDataArray, levels=nothing)
-    if levels == nothing
-        return values
-    else
-        return PooledDataArray(values, convert(Vector{eltype(values)}, levels))
-    end
+    levels!(da, level_values)
 end
 
 function discretize(values, levels=nothing, order=nothing, preserve_order=true)
-    if levels == nothing
-        if preserve_order
-            levels = OrderedSet()
-            for value in values
-                push!(levels, value)
-            end
-            da = discretize_make_pda(values, collect(eltype(values), levels))
-        else
-            da = discretize_make_pda(values)
-        end
+    if preserve_order
+        da = CategoricalArray(values, ordered=true)
     else
-        da = discretize_make_pda(values, levels)
+        da = CategoricalArray(values)
     end
 
     if order != nothing
-        return reorder_levels(da, order)
-    else
-        return da
+        reorder_levels(da, order)
     end
+    return da
 end
 
 
@@ -364,7 +324,7 @@ function apply_scale(scale::DiscreteScale, aess::Vector{Gadfly.Aesthetics}, data
             getfield(data, var) === nothing && continue
 
             disc_data = discretize(getfield(data, var), scale.levels, scale.order)
-            setfield!(aes, var, PooledDataArray([round(Int64,x) for x in disc_data.refs]))
+            setfield!(aes, var, CategoricalArray([round(Int64,x) for x in disc_data.refs]))
 
             # The leveler for discrete scales is a closure over the discretized data.
             if scale.labels === nothing
@@ -418,7 +378,7 @@ function apply_scale(scale::IdentityColorScale,
                      aess::Vector{Gadfly.Aesthetics}, datas::Gadfly.Data...)
     for (aes, data) in zip(aess, datas)
         data.color === nothing && continue
-        aes.color = PooledDataArray(data.color)
+        aes.color = CategoricalArray(data.color)
         aes.color_key_colors = Dict()
     end
 end
@@ -488,7 +448,7 @@ function apply_scale(scale::DiscreteColorScale,
     for (aes, data) in zip(aess, datas)
         data.color === nothing && continue
         for d in data.color
-            isna(d) || push!(levelset, d)
+            ismissing(d) || push!(levelset, d)
         end
     end
 
@@ -517,7 +477,7 @@ function apply_scale(scale::DiscreteColorScale,
             end
         end
 
-        colored_ds = PooledDataArray(colorvals, colors)
+        colored_ds = CategoricalArray(colorvals)
         aes.color = colored_ds
 
         aes.color_label = labeler