Skip to content

Commit

Permalink
Rollup merge of rust-lang#137902 - nnethercote:ast-lexer-TokenKind, r…
Browse files Browse the repository at this point in the history 
…=compiler-errors

Make `ast::TokenKind` more like `lexer::TokenKind`

This is step 2 of rust-lang/compiler-team#831.

r? `@spastorino`
  • Loading branch information
@matthiaskrgr
matthiaskrgr authored Mar 3, 2025
2 parents 70b9968 + 53167c0 commit decb0c9
Show file tree
Hide file tree
Showing 24 changed files with 396 additions and 353 deletions.
296 changes: 170 additions & 126 deletions compiler/rustc_ast/src/token.rs
View file

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion compiler/rustc_ast/src/tokenstream.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -651,7 +651,7 @@ impl TokenStream {
if attr_style == AttrStyle::Inner {
vec![
TokenTree::token_joint(token::Pound, span),
TokenTree::token_joint_hidden(token::Not, span),
TokenTree::token_joint_hidden(token::Bang, span),
body,
]
} else {
Expand Down
42 changes: 21 additions & 21 deletions compiler/rustc_ast/src/util/parser.rs
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
use rustc_span::kw;

use crate::ast::{self, BinOpKind, RangeLimits};
use crate::token::{self, BinOpToken, Token};
use crate::token::{self, Token};

/// Associative operator.
#[derive(Copy, Clone, PartialEq, Debug)]
Expand Down Expand Up @@ -34,26 +34,26 @@ impl AssocOp {
use AssocOp::*;
match t.kind {
token::Eq => Some(Assign),
token::BinOp(BinOpToken::Plus) => Some(Binary(BinOpKind::Add)),
token::BinOp(BinOpToken::Minus) => Some(Binary(BinOpKind::Sub)),
token::BinOp(BinOpToken::Star) => Some(Binary(BinOpKind::Mul)),
token::BinOp(BinOpToken::Slash) => Some(Binary(BinOpKind::Div)),
token::BinOp(BinOpToken::Percent) => Some(Binary(BinOpKind::Rem)),
token::BinOp(BinOpToken::Caret) => Some(Binary(BinOpKind::BitXor)),
token::BinOp(BinOpToken::And) => Some(Binary(BinOpKind::BitAnd)),
token::BinOp(BinOpToken::Or) => Some(Binary(BinOpKind::BitOr)),
token::BinOp(BinOpToken::Shl) => Some(Binary(BinOpKind::Shl)),
token::BinOp(BinOpToken::Shr) => Some(Binary(BinOpKind::Shr)),
token::BinOpEq(BinOpToken::Plus) => Some(AssignOp(BinOpKind::Add)),
token::BinOpEq(BinOpToken::Minus) => Some(AssignOp(BinOpKind::Sub)),
token::BinOpEq(BinOpToken::Star) => Some(AssignOp(BinOpKind::Mul)),
token::BinOpEq(BinOpToken::Slash) => Some(AssignOp(BinOpKind::Div)),
token::BinOpEq(BinOpToken::Percent) => Some(AssignOp(BinOpKind::Rem)),
token::BinOpEq(BinOpToken::Caret) => Some(AssignOp(BinOpKind::BitXor)),
token::BinOpEq(BinOpToken::And) => Some(AssignOp(BinOpKind::BitAnd)),
token::BinOpEq(BinOpToken::Or) => Some(AssignOp(BinOpKind::BitOr)),
token::BinOpEq(BinOpToken::Shl) => Some(AssignOp(BinOpKind::Shl)),
token::BinOpEq(BinOpToken::Shr) => Some(AssignOp(BinOpKind::Shr)),
token::Plus => Some(Binary(BinOpKind::Add)),
token::Minus => Some(Binary(BinOpKind::Sub)),
token::Star => Some(Binary(BinOpKind::Mul)),
token::Slash => Some(Binary(BinOpKind::Div)),
token::Percent => Some(Binary(BinOpKind::Rem)),
token::Caret => Some(Binary(BinOpKind::BitXor)),
token::And => Some(Binary(BinOpKind::BitAnd)),
token::Or => Some(Binary(BinOpKind::BitOr)),
token::Shl => Some(Binary(BinOpKind::Shl)),
token::Shr => Some(Binary(BinOpKind::Shr)),
token::PlusEq => Some(AssignOp(BinOpKind::Add)),
token::MinusEq => Some(AssignOp(BinOpKind::Sub)),
token::StarEq => Some(AssignOp(BinOpKind::Mul)),
token::SlashEq => Some(AssignOp(BinOpKind::Div)),
token::PercentEq => Some(AssignOp(BinOpKind::Rem)),
token::CaretEq => Some(AssignOp(BinOpKind::BitXor)),
token::AndEq => Some(AssignOp(BinOpKind::BitAnd)),
token::OrEq => Some(AssignOp(BinOpKind::BitOr)),
token::ShlEq => Some(AssignOp(BinOpKind::Shl)),
token::ShrEq => Some(AssignOp(BinOpKind::Shr)),
token::Lt => Some(Binary(BinOpKind::Lt)),
token::Le => Some(Binary(BinOpKind::Le)),
token::Ge => Some(Binary(BinOpKind::Ge)),
Expand Down
45 changes: 23 additions & 22 deletions compiler/rustc_ast_pretty/src/pprust/state.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,7 @@ use std::sync::Arc;

use rustc_ast::attr::AttrIdGenerator;
use rustc_ast::ptr::P;
use rustc_ast::token::{
self, BinOpToken, CommentKind, Delimiter, IdentIsRaw, Nonterminal, Token, TokenKind,
};
use rustc_ast::token::{self, CommentKind, Delimiter, IdentIsRaw, Nonterminal, Token, TokenKind};
use rustc_ast::tokenstream::{Spacing, TokenStream, TokenTree};
use rustc_ast::util::classify;
use rustc_ast::util::comments::{Comment, CommentStyle};
Expand Down Expand Up @@ -319,7 +317,7 @@ fn space_between(tt1: &TokenTree, tt2: &TokenTree) -> bool {
(tt1, Tok(Token { kind: Comma | Semi | Dot, .. }, _)) if !is_punct(tt1) => false,

// IDENT + `!`: `println!()`, but `if !x { ... }` needs a space after the `if`
(Tok(Token { kind: Ident(sym, is_raw), span }, _), Tok(Token { kind: Not, .. }, _))
(Tok(Token { kind: Ident(sym, is_raw), span }, _), Tok(Token { kind: Bang, .. }, _))
if !Ident::new(*sym, *span).is_reserved() || matches!(is_raw, IdentIsRaw::Yes) =>
{
false
Expand All @@ -344,21 +342,6 @@ fn space_between(tt1: &TokenTree, tt2: &TokenTree) -> bool {
}
}

fn binop_to_string(op: BinOpToken) -> &'static str {
match op {
token::Plus => "+",
token::Minus => "-",
token::Star => "*",
token::Slash => "/",
token::Percent => "%",
token::Caret => "^",
token::And => "&",
token::Or => "|",
token::Shl => "<<",
token::Shr => ">>",
}
}

pub fn doc_comment_to_string(
comment_kind: CommentKind,
attr_style: ast::AttrStyle,
Expand Down Expand Up @@ -913,12 +896,30 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
token::Ne => "!=".into(),
token::Ge => ">=".into(),
token::Gt => ">".into(),
token::Not => "!".into(),
token::Bang => "!".into(),
token::Tilde => "~".into(),
token::OrOr => "||".into(),
token::AndAnd => "&&".into(),
token::BinOp(op) => binop_to_string(op).into(),
token::BinOpEq(op) => format!("{}=", binop_to_string(op)).into(),
token::Plus => "+".into(),
token::Minus => "-".into(),
token::Star => "*".into(),
token::Slash => "/".into(),
token::Percent => "%".into(),
token::Caret => "^".into(),
token::And => "&".into(),
token::Or => "|".into(),
token::Shl => "<<".into(),
token::Shr => ">>".into(),
token::PlusEq => "+=".into(),
token::MinusEq => "-=".into(),
token::StarEq => "*=".into(),
token::SlashEq => "/=".into(),
token::PercentEq => "%=".into(),
token::CaretEq => "^=".into(),
token::AndEq => "&=".into(),
token::OrEq => "|=".into(),
token::ShlEq => "<<=".into(),
token::ShrEq => ">>=".into(),

/* Structural symbols */
token::At => "@".into(),
Expand Down
2 changes: 1 addition & 1 deletion compiler/rustc_expand/src/config.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -328,7 +328,7 @@ impl<'a> StripUnconfigured<'a> {

// For inner attributes, we do the same thing for the `!` in `#![attr]`.
let mut trees = if cfg_attr.style == AttrStyle::Inner {
let Some(TokenTree::Token(bang_token @ Token { kind: TokenKind::Not, .. }, _)) =
let Some(TokenTree::Token(bang_token @ Token { kind: TokenKind::Bang, .. }, _)) =
orig_trees.next()
else {
panic!("Bad tokens for attribute {cfg_attr:?}");
Expand Down
2 changes: 1 addition & 1 deletion compiler/rustc_expand/src/mbe/macro_check.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -432,7 +432,7 @@ fn check_nested_occurrences(
}
(
NestedMacroState::MacroRules,
&TokenTree::Token(Token { kind: TokenKind::Not, .. }),
&TokenTree::Token(Token { kind: TokenKind::Bang, .. }),
) => {
state = NestedMacroState::MacroRulesNot;
}
Expand Down
12 changes: 6 additions & 6 deletions compiler/rustc_expand/src/mbe/macro_rules.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -690,7 +690,7 @@ fn has_compile_error_macro(rhs: &mbe::TokenTree) -> bool {
&& let TokenKind::Ident(ident, _) = ident.kind
&& ident == sym::compile_error
&& let mbe::TokenTree::Token(bang) = bang
&& let TokenKind::Not = bang.kind
&& let TokenKind::Bang = bang.kind
&& let mbe::TokenTree::Delimited(.., del) = args
&& !del.delim.skip()
{
Expand Down Expand Up @@ -1135,7 +1135,7 @@ fn check_matcher_core<'tt>(
&& matches!(kind, NonterminalKind::Pat(PatParam { inferred: true }))
&& matches!(
next_token,
TokenTree::Token(token) if *token == BinOp(token::BinOpToken::Or)
TokenTree::Token(token) if *token == token::Or
)
{
// It is suggestion to use pat_param, for example: $x:pat -> $x:pat_param.
Expand Down Expand Up @@ -1177,7 +1177,7 @@ fn check_matcher_core<'tt>(

if kind == NonterminalKind::Pat(PatWithOr)
&& sess.psess.edition.at_least_rust_2021()
&& next_token.is_token(&BinOp(token::BinOpToken::Or))
&& next_token.is_token(&token::Or)
{
let suggestion = quoted_tt_to_string(&TokenTree::MetaVarDecl(
span,
Expand Down Expand Up @@ -1296,7 +1296,7 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
const TOKENS: &[&str] = &["`=>`", "`,`", "`=`", "`|`", "`if`", "`in`"];
match tok {
TokenTree::Token(token) => match token.kind {
FatArrow | Comma | Eq | BinOp(token::Or) => IsInFollow::Yes,
FatArrow | Comma | Eq | Or => IsInFollow::Yes,
Ident(name, IdentIsRaw::No) if name == kw::If || name == kw::In => {
IsInFollow::Yes
}
Expand Down Expand Up @@ -1332,9 +1332,9 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
| Colon
| Eq
| Gt
| BinOp(token::Shr)
| Shr
| Semi
| BinOp(token::Or) => IsInFollow::Yes,
| Or => IsInFollow::Yes,
Ident(name, IdentIsRaw::No) if name == kw::As || name == kw::Where => {
IsInFollow::Yes
}
Expand Down
4 changes: 2 additions & 2 deletions compiler/rustc_expand/src/mbe/quoted.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -302,8 +302,8 @@ fn parse_tree<'a>(
/// `None`.
fn kleene_op(token: &Token) -> Option<KleeneOp> {
match token.kind {
token::BinOp(token::Star) => Some(KleeneOp::ZeroOrMore),
token::BinOp(token::Plus) => Some(KleeneOp::OneOrMore),
token::Star => Some(KleeneOp::ZeroOrMore),
token::Plus => Some(KleeneOp::OneOrMore),
token::Question => Some(KleeneOp::ZeroOrOne),
_ => None,
}
Expand Down
71 changes: 33 additions & 38 deletions compiler/rustc_expand/src/proc_macro_server.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -180,28 +180,28 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
Gt => op(">"),
AndAnd => op("&&"),
OrOr => op("||"),
Not => op("!"),
Bang => op("!"),
Tilde => op("~"),
BinOp(Plus) => op("+"),
BinOp(Minus) => op("-"),
BinOp(Star) => op("*"),
BinOp(Slash) => op("/"),
BinOp(Percent) => op("%"),
BinOp(Caret) => op("^"),
BinOp(And) => op("&"),
BinOp(Or) => op("|"),
BinOp(Shl) => op("<<"),
BinOp(Shr) => op(">>"),
BinOpEq(Plus) => op("+="),
BinOpEq(Minus) => op("-="),
BinOpEq(Star) => op("*="),
BinOpEq(Slash) => op("/="),
BinOpEq(Percent) => op("%="),
BinOpEq(Caret) => op("^="),
BinOpEq(And) => op("&="),
BinOpEq(Or) => op("|="),
BinOpEq(Shl) => op("<<="),
BinOpEq(Shr) => op(">>="),
Plus => op("+"),
Minus => op("-"),
Star => op("*"),
Slash => op("/"),
Percent => op("%"),
Caret => op("^"),
And => op("&"),
Or => op("|"),
Shl => op("<<"),
Shr => op(">>"),
PlusEq => op("+="),
MinusEq => op("-="),
StarEq => op("*="),
SlashEq => op("/="),
PercentEq => op("%="),
CaretEq => op("^="),
AndEq => op("&="),
OrEq => op("|="),
ShlEq => op("<<="),
ShrEq => op(">>="),
At => op("@"),
Dot => op("."),
DotDot => op(".."),
Expand Down Expand Up @@ -322,16 +322,16 @@ impl ToInternal<SmallVec<[tokenstream::TokenTree; 2]>>
b'=' => Eq,
b'<' => Lt,
b'>' => Gt,
b'!' => Not,
b'!' => Bang,
b'~' => Tilde,
b'+' => BinOp(Plus),
b'-' => BinOp(Minus),
b'*' => BinOp(Star),
b'/' => BinOp(Slash),
b'%' => BinOp(Percent),
b'^' => BinOp(Caret),
b'&' => BinOp(And),
b'|' => BinOp(Or),
b'+' => Plus,
b'-' => Minus,
b'*' => Star,
b'/' => Slash,
b'%' => Percent,
b'^' => Caret,
b'&' => And,
b'|' => Or,
b'@' => At,
b'.' => Dot,
b',' => Comma,
Expand Down Expand Up @@ -372,10 +372,9 @@ impl ToInternal<SmallVec<[tokenstream::TokenTree; 2]>>
suffix,
span,
}) if symbol.as_str().starts_with('-') => {
let minus = BinOp(BinOpToken::Minus);
let symbol = Symbol::intern(&symbol.as_str()[1..]);
let integer = TokenKind::lit(token::Integer, symbol, suffix);
let a = tokenstream::TokenTree::token_joint_hidden(minus, span);
let a = tokenstream::TokenTree::token_joint_hidden(Minus, span);
let b = tokenstream::TokenTree::token_alone(integer, span);
smallvec![a, b]
}
Expand All @@ -385,10 +384,9 @@ impl ToInternal<SmallVec<[tokenstream::TokenTree; 2]>>
suffix,
span,
}) if symbol.as_str().starts_with('-') => {
let minus = BinOp(BinOpToken::Minus);
let symbol = Symbol::intern(&symbol.as_str()[1..]);
let float = TokenKind::lit(token::Float, symbol, suffix);
let a = tokenstream::TokenTree::token_joint_hidden(minus, span);
let a = tokenstream::TokenTree::token_joint_hidden(Minus, span);
let b = tokenstream::TokenTree::token_alone(float, span);
smallvec![a, b]
}
Expand Down Expand Up @@ -599,10 +597,7 @@ impl server::TokenStream for Rustc<'_, '_> {
Ok(Self::TokenStream::from_iter([
// FIXME: The span of the `-` token is lost when
// parsing, so we cannot faithfully recover it here.
tokenstream::TokenTree::token_joint_hidden(
token::BinOp(token::Minus),
e.span,
),
tokenstream::TokenTree::token_joint_hidden(token::Minus, e.span),
tokenstream::TokenTree::token_alone(token::Literal(*token_lit), e.span),
]))
}
Expand Down
18 changes: 9 additions & 9 deletions compiler/rustc_parse/src/lexer/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -384,17 +384,17 @@ impl<'psess, 'src> Lexer<'psess, 'src> {
rustc_lexer::TokenKind::Colon => token::Colon,
rustc_lexer::TokenKind::Dollar => token::Dollar,
rustc_lexer::TokenKind::Eq => token::Eq,
rustc_lexer::TokenKind::Bang => token::Not,
rustc_lexer::TokenKind::Bang => token::Bang,
rustc_lexer::TokenKind::Lt => token::Lt,
rustc_lexer::TokenKind::Gt => token::Gt,
rustc_lexer::TokenKind::Minus => token::BinOp(token::Minus),
rustc_lexer::TokenKind::And => token::BinOp(token::And),
rustc_lexer::TokenKind::Or => token::BinOp(token::Or),
rustc_lexer::TokenKind::Plus => token::BinOp(token::Plus),
rustc_lexer::TokenKind::Star => token::BinOp(token::Star),
rustc_lexer::TokenKind::Slash => token::BinOp(token::Slash),
rustc_lexer::TokenKind::Caret => token::BinOp(token::Caret),
rustc_lexer::TokenKind::Percent => token::BinOp(token::Percent),
rustc_lexer::TokenKind::Minus => token::Minus,
rustc_lexer::TokenKind::And => token::And,
rustc_lexer::TokenKind::Or => token::Or,
rustc_lexer::TokenKind::Plus => token::Plus,
rustc_lexer::TokenKind::Star => token::Star,
rustc_lexer::TokenKind::Slash => token::Slash,
rustc_lexer::TokenKind::Caret => token::Caret,
rustc_lexer::TokenKind::Percent => token::Percent,

rustc_lexer::TokenKind::Unknown | rustc_lexer::TokenKind::InvalidIdent => {
// Don't emit diagnostics for sequences of the same invalid token
Expand Down
12 changes: 6 additions & 6 deletions compiler/rustc_parse/src/lexer/unicode_chars.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -308,11 +308,11 @@ pub(super) static UNICODE_ARRAY: &[(char, &str, &str)] = &[
const ASCII_ARRAY: &[(&str, &str, Option<token::TokenKind>)] = &[
(" ", "Space", None),
("_", "Underscore", Some(token::Ident(kw::Underscore, token::IdentIsRaw::No))),
("-", "Minus/Hyphen", Some(token::BinOp(token::Minus))),
("-", "Minus/Hyphen", Some(token::Minus)),
(",", "Comma", Some(token::Comma)),
(";", "Semicolon", Some(token::Semi)),
(":", "Colon", Some(token::Colon)),
("!", "Exclamation Mark", Some(token::Not)),
("!", "Exclamation Mark", Some(token::Bang)),
("?", "Question Mark", Some(token::Question)),
(".", "Period", Some(token::Dot)),
("(", "Left Parenthesis", Some(token::OpenDelim(Delimiter::Parenthesis))),
Expand All @@ -321,11 +321,11 @@ const ASCII_ARRAY: &[(&str, &str, Option<token::TokenKind>)] = &[
("]", "Right Square Bracket", Some(token::CloseDelim(Delimiter::Bracket))),
("{", "Left Curly Brace", Some(token::OpenDelim(Delimiter::Brace))),
("}", "Right Curly Brace", Some(token::CloseDelim(Delimiter::Brace))),
("*", "Asterisk", Some(token::BinOp(token::Star))),
("/", "Slash", Some(token::BinOp(token::Slash))),
("*", "Asterisk", Some(token::Star)),
("/", "Slash", Some(token::Slash)),
("\\", "Backslash", None),
("&", "Ampersand", Some(token::BinOp(token::And))),
("+", "Plus Sign", Some(token::BinOp(token::Plus))),
("&", "Ampersand", Some(token::And)),
("+", "Plus Sign", Some(token::Plus)),
("<", "Less-Than Sign", Some(token::Lt)),
("=", "Equals Sign", Some(token::Eq)),
("==", "Double Equals Sign", Some(token::EqEq)),
Expand Down
Loading

0 comments on commit decb0c9

Please sign in to comment.