From 16f5e9e956bf9df75cbeef2b54e9b1d09ce9f043 Mon Sep 17 00:00:00 2001 From: Douglas Young Date: Sun, 16 Feb 2014 19:11:39 +0000 Subject: [PATCH] Add support for non-decimal floating point literals. Issue #1433. The syntax chosen was requiring an 0b, 0x, or 0o after the dot. If the literal is hexadecimal, an exponent is required. --- src/doc/rust.md | 33 +++-- src/libsyntax/parse/lexer.rs | 127 +++++++++++++----- src/test/compile-fail/lex-bad-fp-base-1.rs | 13 -- src/test/compile-fail/lex-bad-fp-base-2.rs | 13 -- src/test/compile-fail/lex-bad-fp-base-3.rs | 13 -- src/test/compile-fail/lex-bad-fp-base-4.rs | 13 -- src/test/compile-fail/lex-bad-fp-base-5.rs | 13 -- src/test/compile-fail/lex-bad-fp-base-6.rs | 13 -- src/test/compile-fail/lex-bad-fp-base-7.rs | 13 -- src/test/compile-fail/lex-bad-fp-base-8.rs | 13 -- src/test/compile-fail/lex-bad-fp-base-9.rs | 13 -- ...float-lit.rs => lex-hex-float-exponent.rs} | 2 +- ...loat-literal.rs => lex-no-mixed-floats.rs} | 10 +- src/test/compile-fail/no-hex-float-literal.rs | 17 --- .../hex-floats.rs} | 14 +- 15 files changed, 123 insertions(+), 197 deletions(-) delete mode 100644 src/test/compile-fail/lex-bad-fp-base-1.rs delete mode 100644 src/test/compile-fail/lex-bad-fp-base-2.rs delete mode 100644 src/test/compile-fail/lex-bad-fp-base-3.rs delete mode 100644 src/test/compile-fail/lex-bad-fp-base-4.rs delete mode 100644 src/test/compile-fail/lex-bad-fp-base-5.rs delete mode 100644 src/test/compile-fail/lex-bad-fp-base-6.rs delete mode 100644 src/test/compile-fail/lex-bad-fp-base-7.rs delete mode 100644 src/test/compile-fail/lex-bad-fp-base-8.rs delete mode 100644 src/test/compile-fail/lex-bad-fp-base-9.rs rename src/test/compile-fail/{lex-hex-float-lit.rs => lex-hex-float-exponent.rs} (86%) rename src/test/compile-fail/{no-oct-float-literal.rs => lex-no-mixed-floats.rs} (80%) delete mode 100644 src/test/compile-fail/no-hex-float-literal.rs rename src/test/{compile-fail/no-binary-float-literal.rs => run-pass/hex-floats.rs} (60%) diff --git a/src/doc/rust.md b/src/doc/rust.md index c605ed06ffd1f..988a1cfb65314 100644 --- a/src/doc/rust.md +++ b/src/doc/rust.md @@ -319,11 +319,13 @@ r##"foo #"# bar"##; // foo #"# bar #### Number literals ~~~~ {.ebnf .gram} -num_lit : nonzero_dec [ dec_digit | '_' ] * num_suffix ? - | '0' [ [ dec_digit | '_' ] * num_suffix ? - | 'b' [ '1' | '0' | '_' ] + int_suffix ? - | 'o' [ oct_digit | '_' ] + int_suffix ? - | 'x' [ hex_digit | '_' ] + int_suffix ? ] ; +num_lit : radix_lit num_suffix ; + +radix_lit : nonzero_dec [ dec_digit | '_' ] * + | '0' [ [ dec_digit | '_' ] * + | 'b' [ '1' | '0' | '_' ] + + | 'o' [ oct_digit | '_' ] + + | 'x' [ hex_digit | '_' ] + ] ; num_suffix : int_suffix | float_suffix ; @@ -331,9 +333,9 @@ int_suffix : 'u' int_suffix_size ? | 'i' int_suffix_size ? ; int_suffix_size : [ '8' | '1' '6' | '3' '2' | '6' '4' ] ; -float_suffix : [ exponent | '.' dec_lit exponent ? ] ? float_suffix_ty ? ; +float_suffix : [ exponent | '.' radix_lit exponent ? ] ? float_suffix_ty ? ; float_suffix_ty : 'f' [ '3' '2' | '6' '4' ] ; -exponent : ['E' | 'e'] ['-' | '+' ] ? dec_lit ; +exponent : ['E' | 'e' | 'p' | 'P'] ['-' | '+' ] ? dec_lit ; dec_lit : [ dec_digit | '_' ] + ; ~~~~ @@ -343,7 +345,7 @@ as they are differentiated by suffixes. ##### Integer literals -An _integer literal_ has one of four forms: +An _radix literal_ has one of four forms: * A _decimal literal_ starts with a *decimal digit* and continues with any mixture of *decimal digits* and _underscores_. @@ -354,9 +356,9 @@ An _integer literal_ has one of four forms: * A _binary literal_ starts with the character sequence `U+0030` `U+0062` (`0b`) and continues as any mixture binary digits and underscores. -An integer literal may be followed (immediately, without any spaces) by an -_integer suffix_, which changes the type of the literal. There are two kinds -of integer literal suffix: +An integer literal consists of a radix literal and may be followed +(immediately, without any spaces) by an _integer suffix_, which changes the +type of the literal. There are two kinds of integer literal suffix: * The `i` and `u` suffixes give the literal type `int` or `uint`, respectively. @@ -389,10 +391,11 @@ Examples of integer literals of various forms: A _floating-point literal_ has one of two forms: -* Two _decimal literals_ separated by a period +* Two _radix literals_ separated by a period character `U+002E` (`.`), with an optional _exponent_ trailing after the - second decimal literal. -* A single _decimal literal_ followed by an _exponent_. + second decimal literal. Both radix literals must have the same base. +* A single _radix literal_ followed by an _exponent_. +* If the float literal is hexadecimal, an _exponent_ must be supplied. By default, a floating-point literal has a generic type, but will fall back to `f64`. A floating-point literal may be followed (immediately, without any @@ -406,6 +409,8 @@ Examples of floating-point literals of various forms: 123.0; // type f64 0.1; // type f64 0.1f32; // type f32 +0x4.0x432p-4_f32; // type f32 +0b1.0b10111011011000; // type f64 12E+99_f64; // type f64 ~~~~ diff --git a/src/libsyntax/parse/lexer.rs b/src/libsyntax/parse/lexer.rs index b711e95bc943b..037e79e8373f4 100644 --- a/src/libsyntax/parse/lexer.rs +++ b/src/libsyntax/parse/lexer.rs @@ -443,8 +443,7 @@ fn scan_exponent(rdr: &StringReader, start_bpos: BytePos) -> Option<~str> { // \x00 hits the `return None` case immediately, so this is fine. let mut c = rdr.curr.get().unwrap_or('\x00'); let mut rslt = ~""; - if c == 'e' || c == 'E' { - rslt.push_char(c); + if c == 'e' || c == 'E' || c == 'p' || c == 'P' { bump(rdr); c = rdr.curr.get().unwrap_or('\x00'); if c == '-' || c == '+' { @@ -476,40 +475,32 @@ fn scan_digits(rdr: &StringReader, radix: uint) -> ~str { }; } -fn check_float_base(rdr: &StringReader, start_bpos: BytePos, last_bpos: BytePos, - base: uint) { - match base { - 16u => fatal_span(rdr, start_bpos, last_bpos, - ~"hexadecimal float literal is not supported"), - 8u => fatal_span(rdr, start_bpos, last_bpos, - ~"octal float literal is not supported"), - 2u => fatal_span(rdr, start_bpos, last_bpos, - ~"binary float literal is not supported"), - _ => () - } -} - -fn scan_number(c: char, rdr: &StringReader) -> token::Token { - let mut num_str; - let mut base = 10u; - let mut c = c; - let mut n = nextch(rdr).unwrap_or('\x00'); - let start_bpos = rdr.last_pos.get(); +fn scan_radix(rdr: &StringReader) -> uint { + let c = rdr.curr.get().unwrap_or('\x00'); + let n = nextch(rdr).unwrap_or('\x00'); if c == '0' && n == 'x' { bump(rdr); bump(rdr); - base = 16u; + return 16u; } else if c == '0' && n == 'o' { bump(rdr); bump(rdr); - base = 8u; + return 8u; } else if c == '0' && n == 'b' { bump(rdr); bump(rdr); - base = 2u; + return 2u; } + return 10u; +} + +fn scan_number(rdr: &StringReader) -> token::Token { + let mut num_str; + let start_bpos = rdr.last_pos.get(); + let mut base = scan_radix(rdr); num_str = scan_digits(rdr, base); - c = rdr.curr.get().unwrap_or('\x00'); + let mut c = rdr.curr.get().unwrap_or('\x00'); + let mut n:char; nextch(rdr); if c == 'u' || c == 'i' { enum Result { Signed(ast::IntTy), Unsigned(ast::UintTy) } @@ -558,19 +549,71 @@ fn scan_number(c: char, rdr: &StringReader) -> token::Token { } } let mut is_float = false; + let mut dec_part = ~""; if rdr.curr_is('.') && !(ident_start(nextch(rdr)) || nextch_is(rdr, '.')) { is_float = true; bump(rdr); - let dec_part = scan_digits(rdr, 10u); - num_str.push_char('.'); - num_str.push_str(dec_part); + let mantissa_base = scan_radix(rdr); + if mantissa_base != base { + //The ability to switch base, while conceivably useful, is much more + //likely to be triggered by accident. + fatal_span(rdr, start_bpos, rdr.last_pos.get(), + ~"float literals must have consistent base before and after decimal point"); + } + base = mantissa_base; + dec_part = scan_digits(rdr, mantissa_base); } + let mut exp_part = ~""; match scan_exponent(rdr, start_bpos) { - Some(ref s) => { + Some(s) => { is_float = true; - num_str.push_str(*s); + exp_part = s; } - None => () + None => { + if is_float && base > 10 { + //otherwise we have ambiguity: 0x1.0xffff_f32 gets parsed as + //0x1.fffff32, which will create confusing results. + fatal_span(rdr, start_bpos, rdr.last_pos.get(), + ~"hexadecimal float literals must contain exponent"); + } + } + } + if is_float { + if base == 10 || base == 16 { + num_str.push_char('.'); + num_str.push_str( if dec_part.len() > 0 {dec_part} else {~"0"} ); + if exp_part.len() != 0 { + num_str.push_char(if base == 10 {'e'} else {'p'}); + num_str.push_str(exp_part); + } + } else { + num_str = from_str_radix::(num_str, base).unwrap().to_str_radix(16); + let mut i = 0; + let len = dec_part.len(); + let step = match base { 8 => 2, 2 => 4, _ => fail!("Impossible base for float")}; + let mut dec_str = ~""; + while i < len { + let chunk = if i + step > len { + let mut chunk = dec_part.slice_from(i).to_str(); + for _ in range(0, i + step - len) { + chunk.push_char('0'); + } + chunk + } else { + dec_part.slice(i, i + step).to_str() + }; + dec_str.push_str(from_str_radix::(chunk, base).unwrap_or(0).to_str()); + i += step; + } + num_str.push_char('.'); + num_str.push_str(dec_str); + num_str.push_char('p'); + num_str.push_str(if exp_part.len() > 0 {exp_part} else {~"0"}); + } + if base != 10 { + num_str.unshift_char('x'); + num_str.unshift_char('0'); + } } if rdr.curr_is('f') { @@ -580,12 +623,10 @@ fn scan_number(c: char, rdr: &StringReader) -> token::Token { if c == '3' && n == '2' { bump(rdr); bump(rdr); - check_float_base(rdr, start_bpos, rdr.last_pos.get(), base); return token::LIT_FLOAT(str_to_ident(num_str), ast::TyF32); } else if c == '6' && n == '4' { bump(rdr); bump(rdr); - check_float_base(rdr, start_bpos, rdr.last_pos.get(), base); return token::LIT_FLOAT(str_to_ident(num_str), ast::TyF64); /* FIXME (#2252): if this is out of range for either a 32-bit or 64-bit float, it won't be noticed till the @@ -596,7 +637,6 @@ fn scan_number(c: char, rdr: &StringReader) -> token::Token { } } if is_float { - check_float_base(rdr, start_bpos, rdr.last_pos.get(), base); return token::LIT_FLOAT_UNSUFFIXED(str_to_ident(num_str)); } else { if num_str.len() == 0u { @@ -687,7 +727,7 @@ fn next_token_inner(rdr: &StringReader) -> token::Token { }) } if is_dec_digit(c) { - return scan_number(c.unwrap(), rdr); + return scan_number(rdr); } fn binop(rdr: &StringReader, op: token::BinOp) -> token::Token { bump(rdr); @@ -1005,6 +1045,7 @@ mod test { use diagnostic; use parse::token; use parse::token::{str_to_ident}; + use ast; // represents a testing reader (incl. both reader and interner) struct Env { @@ -1139,4 +1180,20 @@ mod test { assert_eq!(tok,token::LIT_CHAR('a' as u32)); } + #[test] fn hex_floats() { + let env = setup(~"0x1.0xffffffp100_f32"); + let TokenAndSpan {tok, sp: _} = + env.string_reader.next_token(); + let id = token::str_to_ident("0x1.ffffffp100"); + assert_eq!(tok,token::LIT_FLOAT(id, ast::TyF32)); + } + + #[test] fn bin_floats() { + let env = setup(~"0b1.0b0000_0001_0010_0011_1p100_f32"); + let TokenAndSpan {tok, sp: _} = + env.string_reader.next_token(); + let id = token::str_to_ident("0x1.01238p100"); + assert_eq!(tok,token::LIT_FLOAT(id, ast::TyF32)); + } + } diff --git a/src/test/compile-fail/lex-bad-fp-base-1.rs b/src/test/compile-fail/lex-bad-fp-base-1.rs deleted file mode 100644 index 659cb5c837955..0000000000000 --- a/src/test/compile-fail/lex-bad-fp-base-1.rs +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2014 The Rust Project Developers. See the COPYRIGHT -// file at the top-level directory of this distribution and at -// http://rust-lang.org/COPYRIGHT. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -fn main() { - let a = 0o1.0; //~ ERROR: octal float literal is not supported -} diff --git a/src/test/compile-fail/lex-bad-fp-base-2.rs b/src/test/compile-fail/lex-bad-fp-base-2.rs deleted file mode 100644 index b1d45f78e4a5b..0000000000000 --- a/src/test/compile-fail/lex-bad-fp-base-2.rs +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2014 The Rust Project Developers. See the COPYRIGHT -// file at the top-level directory of this distribution and at -// http://rust-lang.org/COPYRIGHT. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -fn main() { - let b = 0o2f32; //~ ERROR: octal float literal is not supported -} diff --git a/src/test/compile-fail/lex-bad-fp-base-3.rs b/src/test/compile-fail/lex-bad-fp-base-3.rs deleted file mode 100644 index 79c42360adb2f..0000000000000 --- a/src/test/compile-fail/lex-bad-fp-base-3.rs +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2014 The Rust Project Developers. See the COPYRIGHT -// file at the top-level directory of this distribution and at -// http://rust-lang.org/COPYRIGHT. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -fn main() { - let c = 0o3.0f32; //~ ERROR: octal float literal is not supported -} diff --git a/src/test/compile-fail/lex-bad-fp-base-4.rs b/src/test/compile-fail/lex-bad-fp-base-4.rs deleted file mode 100644 index eaea61b0089af..0000000000000 --- a/src/test/compile-fail/lex-bad-fp-base-4.rs +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2014 The Rust Project Developers. See the COPYRIGHT -// file at the top-level directory of this distribution and at -// http://rust-lang.org/COPYRIGHT. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -fn main() { - let d = 0o4e4; //~ ERROR: octal float literal is not supported -} diff --git a/src/test/compile-fail/lex-bad-fp-base-5.rs b/src/test/compile-fail/lex-bad-fp-base-5.rs deleted file mode 100644 index ee25ed95639e2..0000000000000 --- a/src/test/compile-fail/lex-bad-fp-base-5.rs +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2014 The Rust Project Developers. See the COPYRIGHT -// file at the top-level directory of this distribution and at -// http://rust-lang.org/COPYRIGHT. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -fn main() { - let e = 0o5.0e5; //~ ERROR: octal float literal is not supported -} diff --git a/src/test/compile-fail/lex-bad-fp-base-6.rs b/src/test/compile-fail/lex-bad-fp-base-6.rs deleted file mode 100644 index bf08ec1eae5fe..0000000000000 --- a/src/test/compile-fail/lex-bad-fp-base-6.rs +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2014 The Rust Project Developers. See the COPYRIGHT -// file at the top-level directory of this distribution and at -// http://rust-lang.org/COPYRIGHT. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -fn main() { - let f = 0o6e6f32; //~ ERROR: octal float literal is not supported -} diff --git a/src/test/compile-fail/lex-bad-fp-base-7.rs b/src/test/compile-fail/lex-bad-fp-base-7.rs deleted file mode 100644 index 921ed8f1b69e8..0000000000000 --- a/src/test/compile-fail/lex-bad-fp-base-7.rs +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2014 The Rust Project Developers. See the COPYRIGHT -// file at the top-level directory of this distribution and at -// http://rust-lang.org/COPYRIGHT. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -fn main() { - let g = 0o7.0e7f64; //~ ERROR: octal float literal is not supported -} diff --git a/src/test/compile-fail/lex-bad-fp-base-8.rs b/src/test/compile-fail/lex-bad-fp-base-8.rs deleted file mode 100644 index 10e334ede01c2..0000000000000 --- a/src/test/compile-fail/lex-bad-fp-base-8.rs +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2014 The Rust Project Developers. See the COPYRIGHT -// file at the top-level directory of this distribution and at -// http://rust-lang.org/COPYRIGHT. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -fn main() { - let h = 0x8.0e+9; //~ ERROR: hexadecimal float literal is not supported -} diff --git a/src/test/compile-fail/lex-bad-fp-base-9.rs b/src/test/compile-fail/lex-bad-fp-base-9.rs deleted file mode 100644 index 3ea151cb9826a..0000000000000 --- a/src/test/compile-fail/lex-bad-fp-base-9.rs +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2014 The Rust Project Developers. See the COPYRIGHT -// file at the top-level directory of this distribution and at -// http://rust-lang.org/COPYRIGHT. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -fn main() { - let i = 0x9.0e-9; //~ ERROR: hexadecimal float literal is not supported -} diff --git a/src/test/compile-fail/lex-hex-float-lit.rs b/src/test/compile-fail/lex-hex-float-exponent.rs similarity index 86% rename from src/test/compile-fail/lex-hex-float-lit.rs rename to src/test/compile-fail/lex-hex-float-exponent.rs index 457c6126c44a5..3e8386c6740ce 100644 --- a/src/test/compile-fail/lex-hex-float-lit.rs +++ b/src/test/compile-fail/lex-hex-float-exponent.rs @@ -9,5 +9,5 @@ // except according to those terms. static f: float = - 0x539.0 //~ ERROR: hexadecimal float literal is not supported + 0x539.0x0 //~ ERROR: hexadecimal float literals must contain exponent ; diff --git a/src/test/compile-fail/no-oct-float-literal.rs b/src/test/compile-fail/lex-no-mixed-floats.rs similarity index 80% rename from src/test/compile-fail/no-oct-float-literal.rs rename to src/test/compile-fail/lex-no-mixed-floats.rs index 511116b1c559c..822714383a365 100644 --- a/src/test/compile-fail/no-oct-float-literal.rs +++ b/src/test/compile-fail/lex-no-mixed-floats.rs @@ -8,10 +8,6 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. -// error-pattern:octal float literal is not supported - -fn main() { - 0o123f64; - 0o123.456; - 0o123p4f; -} +static f: float = + 0x539.0 //~ ERROR: float literals must have consistent base before and after decimal point +; diff --git a/src/test/compile-fail/no-hex-float-literal.rs b/src/test/compile-fail/no-hex-float-literal.rs deleted file mode 100644 index 4abb6093b2443..0000000000000 --- a/src/test/compile-fail/no-hex-float-literal.rs +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2013 The Rust Project Developers. See the COPYRIGHT -// file at the top-level directory of this distribution and at -// http://rust-lang.org/COPYRIGHT. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -// error-pattern:hexadecimal float literal is not supported - -fn main() { - 0xABC.Df; - 0x567.89; - 0xDEAD.BEEFp-2f; -} diff --git a/src/test/compile-fail/no-binary-float-literal.rs b/src/test/run-pass/hex-floats.rs similarity index 60% rename from src/test/compile-fail/no-binary-float-literal.rs rename to src/test/run-pass/hex-floats.rs index 2e207f90d36cd..b0ed751c85992 100644 --- a/src/test/compile-fail/no-binary-float-literal.rs +++ b/src/test/run-pass/hex-floats.rs @@ -1,4 +1,4 @@ -// Copyright 2013 The Rust Project Developers. See the COPYRIGHT +// Copyright 2012 The Rust Project Developers. See the COPYRIGHT // file at the top-level directory of this distribution and at // http://rust-lang.org/COPYRIGHT. // @@ -8,10 +8,12 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. -// error-pattern:binary float literal is not supported -fn main() { - 0b101010f64; - 0b101.010; - 0b101p4f64; + +pub fn main() { + let a = 0x4.0x0123456p-10; + let b = 0b0100.0b0000_0001_0010_0011_0100_0101_0110p-10; + let c = -0x1.0xfffp-4_f32; + assert_eq!(a, b); + assert_eq!(c, -0.12498474_f32); }