diff --git a/src/PooledArrays.jl b/src/PooledArrays.jl
index b524b1d..cee51f8 100644
--- a/src/PooledArrays.jl
+++ b/src/PooledArrays.jl
@@ -445,7 +445,10 @@ function Base.vcat(a::PooledArray{T, <:Integer, 1}, b::PooledArray{S, <:Integer,
     return PooledArray(RefArray(newrefs), convert(Dict{U, refT}, newlabels))
 end
 
-function fast_sortable(y::PooledArray)
+fast_sortable(y::PooledArray) = _fast_sortable(y)
+fast_sortable(y::PooledArray{T}) where {T<:Integer} = isbitstype(T) ? y : _fast_sortable(y)
+
+function _fast_sortable(y::PooledArray)
     poolranks = invperm(sortperm(y.pool))
     newpool = Dict(j=>convert(eltype(y.refs), i) for (i,j) in enumerate(poolranks))
     PooledArray(RefArray(y.refs), newpool)
diff --git a/test/runtests.jl b/test/runtests.jl
index af967b5..c901ee7 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -94,4 +94,20 @@ using DataAPI: refarray, refvalue, refpool
         @test ys === pushfirst!(ys, -100)
         @test ys == [-100, 10, 20, 30]
     end
+
+    v1 = PooledArray([1, 3, 2, 4])
+    v2 = PooledArray(BigInt.([1, 3, 2, 4]))
+    v3 = PooledArray(["a", "c", "b", "d"])
+
+    @test PooledArrays.fast_sortable(v1) === v1
+    @test isbitstype(eltype(PooledArrays.fast_sortable(v1)))
+    Base.Order.Perm(Base.Order.Forward, v1).data === v1
+
+    @test PooledArrays.fast_sortable(v2) == PooledArray([1, 3, 2, 4])
+    @test isbitstype(eltype(PooledArrays.fast_sortable(v2)))
+    Base.Order.Perm(Base.Order.Forward, v2).data == PooledArray([1, 3, 2, 4])
+
+    @test PooledArrays.fast_sortable(v3) == PooledArray([1, 3, 2, 4])
+    @test isbitstype(eltype(PooledArrays.fast_sortable(v3)))
+    Base.Order.Perm(Base.Order.Forward, v3).data == PooledArray([1, 3, 2, 4])
 end