Fix #9618: BigInt exponentiation for large exponents

If y in (x::BigInt)^y exceeds typemax of Uint and |x|>1, the result will
overflow. In this case, return OverflowError. Otherwise retain the
current behavior of returning a DomainError, unless if x==0, in which
case just return 0.

base/gmp.jl

@@ -363,7 +363,7 @@ function isqrt(x::BigInt)
     return z
 end
 
-function ^(x::BigInt, y::UInt)
+function ^(x::BigInt, y::Culong)
     z = BigInt()
     ccall((:__gmpz_pow_ui, :libgmp), Void, (Ptr{BigInt}, Ptr{BigInt}, Culong), &z, &x, y)
     return z
@@ -373,8 +373,20 @@ function bigint_pow(x::BigInt, y::Integer)
     if y<0; throw(DomainError()); end
     if x== 1; return x; end
     if x==-1; return isodd(y) ? x : -x; end
-    if y>typemax(UInt); throw(DomainError()); end
-    return x^uint(y)
+    if y>typemax(Culong)
+       x==0 && return x
+
+       #At this point, x is not 1, 0 or -1 and it is not possible to use
+       #gmpz_pow_ui to compute the answer. Note that the magnitude of the
+       #answer is:
+       #- at least 2^(2^32-1) ≈ 10^(1.3e9) (if Culong === UInt32).
+       #- at least 2^(2^64-1) ≈ 10^(5.5e18) (if Culong === UInt64).
+       #
+       #Assume that the answer will definitely overflow.
+
+       throw(OverflowError())
+    end
+    return x^convert(Culong, y)
 end
 
 ^(x::BigInt , y::BigInt ) = bigint_pow(x, y)

test/numbers.jl

@@ -2086,3 +2086,8 @@ end
 let x = big(-0.0)
     @test signbit(x) && !signbit(abs(x))
 end
+
+# Issue #9618: errors thrown by large exponentiations
+@test_throws DomainError big(2)^-(big(typemax(Uint))+1)
+@test_throws OverflowError big(2)^(big(typemax(Uint))+1)
+