ICE "Bad tokens for attribute Attribute" with cfg_attr and multiple syntax errors

[jruderman]

Found while reducing #104367

Code

struct S {
    d: [u32; {
        #![cfg_attr(not(X),Y) Z]
    }
}

ICE

thread 'rustc' panicked at 'Bad tokens for attribute Attribute { kind: Normal(NormalAttr { item: AttrItem { path: Path { span: pd.rs:5:12: 5:20 (#0), segments: [PathSegment { ident: cfg_attr#0, id: NodeId(4294967040), args: None }], tokens: None }, args: Delimited(DelimSpan { open: pd.rs:5:20: 5:21 (#0), close: pd.rs:5:29: 5:30 (#0) }, Parenthesis, TokenStream([Token(Token { kind: Ident("not", false), span: pd.rs:5:21: 5:24 (#0) }, Alone), Delimited(DelimSpan { open: pd.rs:5:24: 5:25 (#0), close: pd.rs:5:26: 5:27 (#0) }, Parenthesis, TokenStream([Token(Token { kind: Ident("X", false), span: pd.rs:5:25: 5:26 (#0) }, Alone)])), Token(Token { kind: Comma, span: pd.rs:5:27: 5:28 (#0) }, Alone), Token(Token { kind: Ident("Y", false), span: pd.rs:5:28: 5:29 (#0) }, Alone)])), tokens: None }, tokens: Some(LazyAttrTokenStream(AttrTokenStream([Token(Token { kind: Ident("cfg_attr", false), span: pd.rs:5:12: 5:20 (#0) }, Alone), Delimited(DelimSpan { open: pd.rs:5:20: 5:21 (#0), close: pd.rs:5:29: 5:30 (#0) }, Parenthesis, AttrTokenStream([Token(Token { kind: Ident("not", false), span: pd.rs:5:21: 5:24 (#0) }, Alone), Delimited(DelimSpan { open: pd.rs:5:24: 5:25 (#0), close: pd.rs:5:26: 5:27 (#0) }, Parenthesis, AttrTokenStream([Token(Token { kind: Ident("X", false), span: pd.rs:5:25: 5:26 (#0) }, Alone)])), Token(Token { kind: Comma, span: pd.rs:5:27: 5:28 (#0) }, Alone), Token(Token { kind: Ident("Y", false), span: pd.rs:5:28: 5:29 (#0) }, Alone)]))]))) }), id: AttrId(1), style: Inner, span: pd.rs:5:9: 5:30 (#0) }', compiler/rustc_expand/src/config.rs:403:13

Full output including backtrace

error: expected `]`, found `Z`
 --> pd.rs:4:8
  |
4 |     d: [u32; {
  |        ^ unclosed delimiter
5 |         #![cfg_attr(not(X),Y) Z]
  |                              -^
  |                              |
  |                              help: `]` may belong here

error: expected one of `!`, `.`, `::`, `;`, `?`, `{`, `}`, or an operator, found `]`
 --> pd.rs:5:32
  |
5 |         #![cfg_attr(not(X),Y) Z]
  |                                ^ expected one of 8 possible tokens

thread 'rustc' panicked at 'Bad tokens for attribute Attribute { kind: Normal(NormalAttr { item: AttrItem { path: Path { span: pd.rs:5:12: 5:20 (#0), segments: [PathSegment { ident: cfg_attr#0, id: NodeId(4294967040), args: None }], tokens: None }, args: Delimited(DelimSpan { open: pd.rs:5:20: 5:21 (#0), close: pd.rs:5:29: 5:30 (#0) }, Parenthesis, TokenStream([Token(Token { kind: Ident("not", false), span: pd.rs:5:21: 5:24 (#0) }, Alone), Delimited(DelimSpan { open: pd.rs:5:24: 5:25 (#0), close: pd.rs:5:26: 5:27 (#0) }, Parenthesis, TokenStream([Token(Token { kind: Ident("X", false), span: pd.rs:5:25: 5:26 (#0) }, Alone)])), Token(Token { kind: Comma, span: pd.rs:5:27: 5:28 (#0) }, Alone), Token(Token { kind: Ident("Y", false), span: pd.rs:5:28: 5:29 (#0) }, Alone)])), tokens: None }, tokens: Some(LazyAttrTokenStream(AttrTokenStream([Token(Token { kind: Ident("cfg_attr", false), span: pd.rs:5:12: 5:20 (#0) }, Alone), Delimited(DelimSpan { open: pd.rs:5:20: 5:21 (#0), close: pd.rs:5:29: 5:30 (#0) }, Parenthesis, AttrTokenStream([Token(Token { kind: Ident("not", false), span: pd.rs:5:21: 5:24 (#0) }, Alone), Delimited(DelimSpan { open: pd.rs:5:24: 5:25 (#0), close: pd.rs:5:26: 5:27 (#0) }, Parenthesis, AttrTokenStream([Token(Token { kind: Ident("X", false), span: pd.rs:5:25: 5:26 (#0) }, Alone)])), Token(Token { kind: Comma, span: pd.rs:5:27: 5:28 (#0) }, Alone), Token(Token { kind: Ident("Y", false), span: pd.rs:5:28: 5:29 (#0) }, Alone)]))]))) }), id: AttrId(1), style: Inner, span: pd.rs:5:9: 5:30 (#0) }', compiler/rustc_expand/src/config.rs:403:13
stack backtrace:
   0:        0x109025f12 - <std::sys_common::backtrace::_print::DisplayBacktrace as core::fmt::Display>::fmt::h1b594e9bcc0c6898
   1:        0x1090844ba - core::fmt::write::h7c6f83d024852aa9
   2:        0x109017fec - std::io::Write::write_fmt::h97d969d4aea1606d
   3:        0x109025cda - std::sys_common::backtrace::print::h562dadf6028256bf
   4:        0x1090290b6 - std::panicking::default_hook::{{closure}}::hee367e24075678e4
   5:        0x109028e07 - std::panicking::default_hook::hcce3553a0befadd1
   6:        0x118cdbf3d - rustc_driver[f93b8a3886cc8b65]::DEFAULT_HOOK::{closure#0}::{closure#0}
   7:        0x1090298b5 - std::panicking::rust_panic_with_hook::h4f6feaafc55c56a2
   8:        0x109029643 - std::panicking::begin_panic_handler::{{closure}}::hde5e87cf8416de78
   9:        0x1090263a8 - std::sys_common::backtrace::__rust_end_short_backtrace::h06c292cffdb0bbc3
  10:        0x10902930d - _rust_begin_unwind
  11:        0x1090b1053 - core::panicking::panic_fmt::he51d66482c616f28
  12:        0x11ce4aec4 - <rustc_expand[3365769f7608aa23]::config::StripUnconfigured>::expand_cfg_attr_item
  13:        0x11cd7f835 - <core[322641b34a183a]::iter::adapters::map::Map<alloc[dd1f1a43ef059f04]::vec::into_iter::IntoIter<(rustc_ast[e82592d0ad019575]::ast::AttrItem, rustc_span[50ad8565b2f1bbfb]::span_encoding::Span)>, <rustc_expand[3365769f7608aa23]::config::StripUnconfigured>::expand_cfg_attr::{closure#1}> as core[322641b34a183a]::iter::traits::iterator::Iterator>::fold::<(), core[322641b34a183a]::iter::traits::iterator::Iterator::for_each::call<rustc_ast[e82592d0ad019575]::ast::Attribute, <alloc[dd1f1a43ef059f04]::vec::Vec<rustc_ast[e82592d0ad019575]::ast::Attribute> as alloc[dd1f1a43ef059f04]::vec::spec_extend::SpecExtend<rustc_ast[e82592d0ad019575]::ast::Attribute, core[322641b34a183a]::iter::adapters::map::Map<alloc[dd1f1a43ef059f04]::vec::into_iter::IntoIter<(rustc_ast[e82592d0ad019575]::ast::AttrItem, rustc_span[50ad8565b2f1bbfb]::span_encoding::Span)>, <rustc_expand[3365769f7608aa23]::config::StripUnconfigured>::expand_cfg_attr::{closure#1}>>>::spec_extend::{closure#0}>::{closure#0}>
  14:        0x11cd3c712 - <alloc[dd1f1a43ef059f04]::vec::Vec<rustc_ast[e82592d0ad019575]::ast::Attribute> as alloc[dd1f1a43ef059f04]::vec::spec_from_iter::SpecFromIter<rustc_ast[e82592d0ad019575]::ast::Attribute, core[322641b34a183a]::iter::adapters::map::Map<alloc[dd1f1a43ef059f04]::vec::into_iter::IntoIter<(rustc_ast[e82592d0ad019575]::ast::AttrItem, rustc_span[50ad8565b2f1bbfb]::span_encoding::Span)>, <rustc_expand[3365769f7608aa23]::config::StripUnconfigured>::expand_cfg_attr::{closure#1}>>>::from_iter
  15:        0x11ce4a8c5 - <rustc_expand[3365769f7608aa23]::config::StripUnconfigured>::expand_cfg_attr
  16:        0x11cd94f3d - <rustc_ast[e82592d0ad019575]::ast::Expr as rustc_ast[e82592d0ad019575]::ast_traits::HasAttrs>::visit_attrs::<<rustc_expand[3365769f7608aa23]::expand::InvocationCollector>::expand_cfg_attr<rustc_ast[e82592d0ad019575]::ast_traits::AstNodeWrapper<rustc_ast[e82592d0ad019575]::ptr::P<rustc_ast[e82592d0ad019575]::ast::Expr>, rustc_expand[3365769f7608aa23]::expand::OptExprTag>>::{closure#0}>
  17:        0x11ce1fff0 - <rustc_expand[3365769f7608aa23]::expand::InvocationCollector as rustc_ast[e82592d0ad019575]::mut_visit::MutVisitor>::visit_expr
  18:        0x11ce1fb28 - <rustc_expand[3365769f7608aa23]::expand::InvocationCollector as rustc_ast[e82592d0ad019575]::mut_visit::MutVisitor>::visit_ty
  19:        0x11cde82ed - rustc_ast[e82592d0ad019575]::mut_visit::noop_flat_map_field_def::<rustc_expand[3365769f7608aa23]::expand::InvocationCollector>
  20:        0x11ce1b77e - <rustc_expand[3365769f7608aa23]::expand::InvocationCollector as rustc_ast[e82592d0ad019575]::mut_visit::MutVisitor>::flat_map_field_def
  21:        0x11cd2ea13 - <alloc[dd1f1a43ef059f04]::vec::Vec<rustc_ast[e82592d0ad019575]::ast::FieldDef> as rustc_data_structures[2eed7d44fa552923]::map_in_place::MapInPlace<rustc_ast[e82592d0ad019575]::ast::FieldDef>>::flat_map_in_place::<rustc_ast[e82592d0ad019575]::mut_visit::noop_visit_variant_data<rustc_expand[3365769f7608aa23]::expand::InvocationCollector>::{closure#1}, smallvec[fa9e0875fe25e8ef]::SmallVec<[rustc_ast[e82592d0ad019575]::ast::FieldDef; 1usize]>>
  22:        0x11cde6661 - rustc_ast[e82592d0ad019575]::mut_visit::noop_visit_item_kind::<rustc_expand[3365769f7608aa23]::expand::InvocationCollector>
  23:        0x11cde407c - rustc_ast[e82592d0ad019575]::mut_visit::noop_flat_map_item::<rustc_expand[3365769f7608aa23]::expand::InvocationCollector>
  24:        0x11cd64d3d - <rustc_ast[e82592d0ad019575]::ptr::P<rustc_ast[e82592d0ad019575]::ast::Item> as rustc_expand[3365769f7608aa23]::expand::InvocationCollectorNode>::wrap_flat_map_node_noop_flat_map::<<rustc_expand[3365769f7608aa23]::expand::InvocationCollector>::flat_map_node<rustc_ast[e82592d0ad019575]::ptr::P<rustc_ast[e82592d0ad019575]::ast::Item>>::{closure#0}>
  25:        0x11ce18929 - <rustc_expand[3365769f7608aa23]::expand::InvocationCollector as rustc_ast[e82592d0ad019575]::mut_visit::MutVisitor>::flat_map_item
  26:        0x11cd2936d - <alloc[dd1f1a43ef059f04]::vec::Vec<rustc_ast[e82592d0ad019575]::ptr::P<rustc_ast[e82592d0ad019575]::ast::Item>> as rustc_data_structures[2eed7d44fa552923]::map_in_place::MapInPlace<rustc_ast[e82592d0ad019575]::ptr::P<rustc_ast[e82592d0ad019575]::ast::Item>>>::flat_map_in_place::<rustc_ast[e82592d0ad019575]::mut_visit::noop_visit_item_kind<rustc_expand[3365769f7608aa23]::expand::InvocationCollector>::{closure#3}, smallvec[fa9e0875fe25e8ef]::SmallVec<[rustc_ast[e82592d0ad019575]::ptr::P<rustc_ast[e82592d0ad019575]::ast::Item>; 1usize]>>
  27:        0x11ce1f858 - <rustc_expand[3365769f7608aa23]::expand::InvocationCollector as rustc_ast[e82592d0ad019575]::mut_visit::MutVisitor>::visit_crate
  28:        0x11ce163d5 - <rustc_expand[3365769f7608aa23]::expand::MacroExpander>::collect_invocations
  29:        0x11ce11fa8 - <rustc_expand[3365769f7608aa23]::expand::MacroExpander>::fully_expand_fragment
  30:        0x11ce11c46 - <rustc_expand[3365769f7608aa23]::expand::MacroExpander>::expand_crate
  31:        0x118d60591 - <rustc_session[304ed9bc7c986c94]::session::Session>::time::<core[322641b34a183a]::result::Result<rustc_ast[e82592d0ad019575]::ast::Crate, rustc_errors[ef89fc2f19a46f90]::ErrorGuaranteed>, rustc_interface[bab52cec3fba6e57]::passes::configure_and_expand::{closure#1}>
  32:        0x118d9b025 - rustc_interface[bab52cec3fba6e57]::passes::configure_and_expand
  33:        0x118d7da51 - <rustc_interface[bab52cec3fba6e57]::queries::Queries>::expansion
  34:        0x118caf560 - rustc_span[50ad8565b2f1bbfb]::with_source_map::<core[322641b34a183a]::result::Result<(), rustc_errors[ef89fc2f19a46f90]::ErrorGuaranteed>, rustc_interface[bab52cec3fba6e57]::interface::run_compiler<core[322641b34a183a]::result::Result<(), rustc_errors[ef89fc2f19a46f90]::ErrorGuaranteed>, rustc_driver[f93b8a3886cc8b65]::run_compiler::{closure#1}>::{closure#0}::{closure#1}>
  35:        0x118c9f5bc - <scoped_tls[5d80669e9829205e]::ScopedKey<rustc_span[50ad8565b2f1bbfb]::SessionGlobals>>::set::<rustc_interface[bab52cec3fba6e57]::interface::run_compiler<core[322641b34a183a]::result::Result<(), rustc_errors[ef89fc2f19a46f90]::ErrorGuaranteed>, rustc_driver[f93b8a3886cc8b65]::run_compiler::{closure#1}>::{closure#0}, core[322641b34a183a]::result::Result<(), rustc_errors[ef89fc2f19a46f90]::ErrorGuaranteed>>
  36:        0x118c6e37a - std[38dd6138ba148cb2]::sys_common::backtrace::__rust_begin_short_backtrace::<rustc_interface[bab52cec3fba6e57]::util::run_in_thread_pool_with_globals<rustc_interface[bab52cec3fba6e57]::interface::run_compiler<core[322641b34a183a]::result::Result<(), rustc_errors[ef89fc2f19a46f90]::ErrorGuaranteed>, rustc_driver[f93b8a3886cc8b65]::run_compiler::{closure#1}>::{closure#0}, core[322641b34a183a]::result::Result<(), rustc_errors[ef89fc2f19a46f90]::ErrorGuaranteed>>::{closure#0}::{closure#0}, core[322641b34a183a]::result::Result<(), rustc_errors[ef89fc2f19a46f90]::ErrorGuaranteed>>
  37:        0x118c54b4b - <<std[38dd6138ba148cb2]::thread::Builder>::spawn_unchecked_<rustc_interface[bab52cec3fba6e57]::util::run_in_thread_pool_with_globals<rustc_interface[bab52cec3fba6e57]::interface::run_compiler<core[322641b34a183a]::result::Result<(), rustc_errors[ef89fc2f19a46f90]::ErrorGuaranteed>, rustc_driver[f93b8a3886cc8b65]::run_compiler::{closure#1}>::{closure#0}, core[322641b34a183a]::result::Result<(), rustc_errors[ef89fc2f19a46f90]::ErrorGuaranteed>>::{closure#0}::{closure#0}, core[322641b34a183a]::result::Result<(), rustc_errors[ef89fc2f19a46f90]::ErrorGuaranteed>>::{closure#1} as core[322641b34a183a]::ops::function::FnOnce<()>>::call_once::{shim:vtable#0}
  38:        0x109032967 - std::sys::unix::thread::Thread::new::thread_start::hae9a83a2ac729f3b
  39:     0x7ff80d84d4e1 - __pthread_start

error: internal compiler error: unexpected panic

note: the compiler unexpectedly panicked. this is a bug.

note: we would appreciate a bug report: https://github.com/rust-lang/rust/issues/new?labels=C-bug%2C+I-ICE%2C+T-compiler&template=ice.md

note: rustc 1.67.0-nightly (e75aab045 2022-11-09) running on x86_64-apple-darwin

query stack during panic:
end of query stack
error: aborting due to 2 previous errors

Regression

Possibly a regression in nightly-2020-10-25, perhaps from #77255.

It's hard to tell with such a cursed testcase.

Version

rustc 1.67.0-nightly (e75aab045 2022-11-09)
binary: rustc
commit-hash: e75aab045fc476f176a58c408f6b06f0e275c6e1
commit-date: 2022-11-09
host: x86_64-apple-darwin
release: 1.67.0-nightly
LLVM version: 15.0.4

[chenyukang]

@rustbot claim

[JohnTitor]

Triage: Fixed on the latest nightly (possibly related to #108297), marking as E-needs-test.

[chenyukang]

Hmm, it's weird, I assigned it to myself, but why there is no assignee right now?
@JohnTitor do we need to add a test case and close it?

[JohnTitor]

If you think other tests cover this case, feel free to close :) Otherwise a test would be helpful.