Skip to content

Commit 1cf0194

Browse files
CentrilCentril
authored
Rollup merge of rust-lang#69211 - petrochenkov:prevtok, r=Centril
parser: Simplify treatment of macro variables in `Parser::bump` Follow-up to rust-lang#69006. Token normalization for `$ident` and `$lifetime` is merged directly into `bump`. Special "unknown macro variable" diagnostic for unexpected `$`s is removed as preventing legal code from compiling (as a result `bump` also doesn't call itself recursively anymore and can't make `prev_token` inconsistent). r? @Centril
2 parents b864d23 + 950845c commit 1cf0194

File tree

13 files changed

+88
-82
lines changed

13 files changed

+88
-82
lines changed

‎src/librustc_expand/mbe/macro_parser.rs

-2
Original file line numberDiff line numberDiff line change
@@ -856,8 +856,6 @@ fn parse_nt(p: &mut Parser<'_>, sp: Span, name: Symbol) -> Nonterminal {
856856
if name == sym::tt {
857857
return token::NtTT(p.parse_token_tree());
858858
}
859-
// check at the beginning and the parser checks after each bump
860-
p.process_potential_macro_variable();
861859
match parse_nt_inner(p, sp, name) {
862860
Ok(nt) => nt,
863861
Err(mut err) => {

‎src/librustc_expand/mbe/macro_rules.rs

-1
Original file line numberDiff line numberDiff line change
@@ -267,7 +267,6 @@ fn generic_extension<'cx>(
267267
cx.current_expansion.module.mod_path.last().map(|id| id.to_string());
268268
p.last_type_ascription = cx.current_expansion.prior_type_ascription;
269269

270-
p.process_potential_macro_variable();
271270
// Let the context choose how to interpret the result.
272271
// Weird, but useful for X-macros.
273272
return Box::new(ParserAnyMacro {

‎src/librustc_parse/lib.rs

+4-3
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ use rustc_errors::{Diagnostic, FatalError, Level, PResult};
99
use rustc_session::parse::ParseSess;
1010
use rustc_span::{FileName, SourceFile, Span};
1111
use syntax::ast;
12-
use syntax::token::{self, Nonterminal};
12+
use syntax::token::{self, Nonterminal, Token};
1313
use syntax::tokenstream::{self, TokenStream, TokenTree};
1414

1515
use std::path::{Path, PathBuf};
@@ -170,8 +170,9 @@ fn maybe_source_file_to_parser(
170170
let (stream, unclosed_delims) = maybe_file_to_stream(sess, source_file, None)?;
171171
let mut parser = stream_to_parser(sess, stream, None);
172172
parser.unclosed_delims = unclosed_delims;
173-
if parser.token == token::Eof && parser.token.span.is_dummy() {
174-
parser.token.span = Span::new(end_pos, end_pos, parser.token.span.ctxt());
173+
if parser.token == token::Eof {
174+
let span = Span::new(end_pos, end_pos, parser.token.span.ctxt());
175+
parser.set_token(Token::new(token::Eof, span));
175176
}
176177

177178
Ok(parser)

‎src/librustc_parse/parser/expr.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -166,7 +166,7 @@ impl<'a> Parser<'a> {
166166
while let Some(op) = self.check_assoc_op() {
167167
// Adjust the span for interpolated LHS to point to the `$lhs` token
168168
// and not to what it refers to.
169-
let lhs_span = match self.unnormalized_prev_token().kind {
169+
let lhs_span = match self.unnormalized_prev_token.kind {
170170
TokenKind::Interpolated(..) => self.prev_span,
171171
_ => lhs.span,
172172
};
@@ -527,7 +527,7 @@ impl<'a> Parser<'a> {
527527
) -> PResult<'a, (Span, P<Expr>)> {
528528
expr.map(|e| {
529529
(
530-
match self.unnormalized_prev_token().kind {
530+
match self.unnormalized_prev_token.kind {
531531
TokenKind::Interpolated(..) => self.prev_span,
532532
_ => e.span,
533533
},

‎src/librustc_parse/parser/item.rs

+4-3
Original file line numberDiff line numberDiff line change
@@ -1374,8 +1374,9 @@ impl<'a> Parser<'a> {
13741374
}
13751375

13761376
fn report_invalid_macro_expansion_item(&self, args: &MacArgs) {
1377+
let span = args.span().expect("undelimited macro call");
13771378
let mut err = self.struct_span_err(
1378-
self.prev_span,
1379+
span,
13791380
"macros that expand to items must be delimited with braces or followed by a semicolon",
13801381
);
13811382
if self.unclosed_delims.is_empty() {
@@ -1390,14 +1391,14 @@ impl<'a> Parser<'a> {
13901391
);
13911392
} else {
13921393
err.span_suggestion(
1393-
self.prev_span,
1394+
span,
13941395
"change the delimiters to curly braces",
13951396
" { /* items */ }".to_string(),
13961397
Applicability::HasPlaceholders,
13971398
);
13981399
}
13991400
err.span_suggestion(
1400-
self.prev_span.shrink_to_hi(),
1401+
span.shrink_to_hi(),
14011402
"add a semicolon",
14021403
';'.to_string(),
14031404
Applicability::MaybeIncorrect,

‎src/librustc_parse/parser/mod.rs

+36-64
Original file line numberDiff line numberDiff line change
@@ -93,18 +93,16 @@ pub struct Parser<'a> {
9393
/// Use span from this token if you need an isolated span.
9494
pub token: Token,
9595
/// The current non-normalized token if it's different from `token`.
96-
/// Preferable use is through the `unnormalized_token()` getter.
9796
/// Use span from this token if you need to concatenate it with some neighbouring spans.
98-
unnormalized_token: Option<Token>,
97+
unnormalized_token: Token,
9998
/// The previous normalized token.
10099
/// Use span from this token if you need an isolated span.
101100
prev_token: Token,
102101
/// The previous non-normalized token if it's different from `prev_token`.
103-
/// Preferable use is through the `unnormalized_prev_token()` getter.
104102
/// Use span from this token if you need to concatenate it with some neighbouring spans.
105-
unnormalized_prev_token: Option<Token>,
106-
/// Equivalent to `unnormalized_prev_token().span`.
107-
/// FIXME: Remove in favor of `(unnormalized_)prev_token().span`.
103+
unnormalized_prev_token: Token,
104+
/// Equivalent to `unnormalized_prev_token.span`.
105+
/// FIXME: Remove in favor of `(unnormalized_)prev_token.span`.
108106
pub prev_span: Span,
109107
restrictions: Restrictions,
110108
/// Used to determine the path to externally loaded source files.
@@ -378,9 +376,9 @@ impl<'a> Parser<'a> {
378376
let mut parser = Parser {
379377
sess,
380378
token: Token::dummy(),
381-
unnormalized_token: None,
379+
unnormalized_token: Token::dummy(),
382380
prev_token: Token::dummy(),
383-
unnormalized_prev_token: None,
381+
unnormalized_prev_token: Token::dummy(),
384382
prev_span: DUMMY_SP,
385383
restrictions: Restrictions::empty(),
386384
recurse_into_file_modules,
@@ -404,7 +402,8 @@ impl<'a> Parser<'a> {
404402
subparser_name,
405403
};
406404

407-
parser.token = parser.next_tok();
405+
// Make parser point to the first token.
406+
parser.bump();
408407

409408
if let Some(directory) = directory {
410409
parser.directory = directory;
@@ -418,27 +417,18 @@ impl<'a> Parser<'a> {
418417
}
419418
}
420419

421-
parser.process_potential_macro_variable();
422420
parser
423421
}
424422

425-
fn unnormalized_token(&self) -> &Token {
426-
self.unnormalized_token.as_ref().unwrap_or(&self.token)
427-
}
428-
429-
fn unnormalized_prev_token(&self) -> &Token {
430-
self.unnormalized_prev_token.as_ref().unwrap_or(&self.prev_token)
431-
}
432-
433-
fn next_tok(&mut self) -> Token {
423+
fn next_tok(&mut self, fallback_span: Span) -> Token {
434424
let mut next = if self.desugar_doc_comments {
435425
self.token_cursor.next_desugared()
436426
} else {
437427
self.token_cursor.next()
438428
};
439429
if next.span.is_dummy() {
440430
// Tweak the location for better diagnostics, but keep syntactic context intact.
441-
next.span = self.unnormalized_token().span.with_ctxt(next.span.ctxt());
431+
next.span = fallback_span.with_ctxt(next.span.ctxt());
442432
}
443433
next
444434
}
@@ -896,6 +886,23 @@ impl<'a> Parser<'a> {
896886
self.parse_delim_comma_seq(token::Paren, f)
897887
}
898888

889+
// Interpolated identifier (`$i: ident`) and lifetime (`$l: lifetime`)
890+
// tokens are replaced with usual identifier and lifetime tokens,
891+
// so the former are never encountered during normal parsing.
892+
crate fn set_token(&mut self, token: Token) {
893+
self.unnormalized_token = token;
894+
self.token = match &self.unnormalized_token.kind {
895+
token::Interpolated(nt) => match **nt {
896+
token::NtIdent(ident, is_raw) => {
897+
Token::new(token::Ident(ident.name, is_raw), ident.span)
898+
}
899+
token::NtLifetime(ident) => Token::new(token::Lifetime(ident.name), ident.span),
900+
_ => self.unnormalized_token.clone(),
901+
},
902+
_ => self.unnormalized_token.clone(),
903+
}
904+
}
905+
899906
/// Advance the parser by one token.
900907
pub fn bump(&mut self) {
901908
if self.prev_token.kind == TokenKind::Eof {
@@ -905,16 +912,15 @@ impl<'a> Parser<'a> {
905912
}
906913

907914
// Update the current and previous tokens.
908-
let next_token = self.next_tok();
909-
self.prev_token = mem::replace(&mut self.token, next_token);
915+
self.prev_token = self.token.take();
910916
self.unnormalized_prev_token = self.unnormalized_token.take();
917+
let next_token = self.next_tok(self.unnormalized_prev_token.span);
918+
self.set_token(next_token);
911919

912920
// Update fields derived from the previous token.
913-
self.prev_span = self.unnormalized_prev_token().span;
921+
self.prev_span = self.unnormalized_prev_token.span;
914922

915923
self.expected_tokens.clear();
916-
// Check after each token.
917-
self.process_potential_macro_variable();
918924
}
919925

920926
/// Advances the parser using provided token as a next one. Use this when
@@ -924,12 +930,12 @@ impl<'a> Parser<'a> {
924930
/// Correct token kinds and spans need to be calculated instead.
925931
fn bump_with(&mut self, next: TokenKind, span: Span) {
926932
// Update the current and previous tokens.
927-
let next_token = Token::new(next, span);
928-
self.prev_token = mem::replace(&mut self.token, next_token);
933+
self.prev_token = self.token.take();
929934
self.unnormalized_prev_token = self.unnormalized_token.take();
935+
self.set_token(Token::new(next, span));
930936

931937
// Update fields derived from the previous token.
932-
self.prev_span = self.unnormalized_prev_token().span.with_hi(span.lo());
938+
self.prev_span = self.unnormalized_prev_token.span.with_hi(span.lo());
933939

934940
self.expected_tokens.clear();
935941
}
@@ -1066,39 +1072,6 @@ impl<'a> Parser<'a> {
10661072
}
10671073
}
10681074

1069-
pub fn process_potential_macro_variable(&mut self) {
1070-
let normalized_token = match self.token.kind {
1071-
token::Dollar
1072-
if self.token.span.from_expansion() && self.look_ahead(1, |t| t.is_ident()) =>
1073-
{
1074-
self.bump();
1075-
let name = match self.token.kind {
1076-
token::Ident(name, _) => name,
1077-
_ => unreachable!(),
1078-
};
1079-
let span = self.prev_span.to(self.token.span);
1080-
self.struct_span_err(span, &format!("unknown macro variable `{}`", name))
1081-
.span_label(span, "unknown macro variable")
1082-
.emit();
1083-
self.bump();
1084-
return;
1085-
}
1086-
token::Interpolated(ref nt) => {
1087-
// Interpolated identifier and lifetime tokens are replaced with usual identifier
1088-
// and lifetime tokens, so the former are never encountered during normal parsing.
1089-
match **nt {
1090-
token::NtIdent(ident, is_raw) => {
1091-
Token::new(token::Ident(ident.name, is_raw), ident.span)
1092-
}
1093-
token::NtLifetime(ident) => Token::new(token::Lifetime(ident.name), ident.span),
1094-
_ => return,
1095-
}
1096-
}
1097-
_ => return,
1098-
};
1099-
self.unnormalized_token = Some(mem::replace(&mut self.token, normalized_token));
1100-
}
1101-
11021075
/// Parses a single token tree from the input.
11031076
pub fn parse_token_tree(&mut self) -> TokenTree {
11041077
match self.token.kind {
@@ -1107,15 +1080,14 @@ impl<'a> Parser<'a> {
11071080
&mut self.token_cursor.frame,
11081081
self.token_cursor.stack.pop().unwrap(),
11091082
);
1110-
self.token.span = frame.span.entire();
1083+
self.set_token(Token::new(TokenKind::CloseDelim(frame.delim), frame.span.close));
11111084
self.bump();
11121085
TokenTree::Delimited(frame.span, frame.delim, frame.tree_cursor.stream.into())
11131086
}
11141087
token::CloseDelim(_) | token::Eof => unreachable!(),
11151088
_ => {
1116-
let token = self.token.clone();
11171089
self.bump();
1118-
TokenTree::Token(token)
1090+
TokenTree::Token(self.prev_token.clone())
11191091
}
11201092
}
11211093
}

‎src/librustc_parse/parser/path.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,7 @@ impl<'a> Parser<'a> {
134134
path
135135
});
136136

137-
let lo = self.unnormalized_token().span;
137+
let lo = self.unnormalized_token.span;
138138
let mut segments = Vec::new();
139139
let mod_sep_ctxt = self.token.span.ctxt();
140140
if self.eat(&token::ModSep) {

‎src/test/ui/issues/issue-6596-1.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
macro_rules! e {
22
($inp:ident) => (
33
$nonexistent
4-
//~^ ERROR unknown macro variable `nonexistent`
4+
//~^ ERROR expected expression, found `$`
55
);
66
}
77

‎src/test/ui/issues/issue-6596-1.stderr

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
1-
error: unknown macro variable `nonexistent`
1+
error: expected expression, found `$`
22
--> $DIR/issue-6596-1.rs:3:9
33
|
44
LL | $nonexistent
5-
| ^^^^^^^^^^^^ unknown macro variable
5+
| ^^^^^^^^^^^^ expected expression
66
...
77
LL | e!(foo);
88
| -------- in this macro invocation

‎src/test/ui/issues/issue-6596-2.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
macro_rules! g {
44
($inp:ident) => (
55
{ $inp $nonexistent }
6-
//~^ ERROR unknown macro variable `nonexistent`
6+
//~^ ERROR expected one of `!`, `.`, `::`, `;`, `?`, `{`, `}`, or an operator, found `$`
77
);
88
}
99

‎src/test/ui/issues/issue-6596-2.stderr

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
1-
error: unknown macro variable `nonexistent`
1+
error: expected one of `!`, `.`, `::`, `;`, `?`, `{`, `}`, or an operator, found `$`
22
--> $DIR/issue-6596-2.rs:5:16
33
|
44
LL | { $inp $nonexistent }
5-
| ^^^^^^^^^^^^ unknown macro variable
5+
| ^^^^^^^^^^^^ expected one of 8 possible tokens
66
...
77
LL | g!(foo);
88
| -------- in this macro invocation

‎src/test/ui/proc-macro/auxiliary/generate-dollar-ident.rs

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
// force-host
2+
// no-prefer-dynamic
3+
4+
#![feature(proc_macro_hygiene)]
5+
#![feature(proc_macro_quote)]
6+
#![crate_type = "proc-macro"]
7+
8+
extern crate proc_macro;
9+
use proc_macro::*;
10+
11+
#[proc_macro]
12+
pub fn dollar_ident(input: TokenStream) -> TokenStream {
13+
let black_hole = input.into_iter().next().unwrap();
14+
quote! {
15+
$black_hole!($$var);
16+
}
17+
}

‎src/test/ui/proc-macro/generate-dollar-ident.rs

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
// Proc macros can generate token sequence `$ IDENT`
2+
// without it being recognized as an unknown macro variable.
3+
4+
// check-pass
5+
// aux-build:generate-dollar-ident.rs
6+
7+
extern crate generate_dollar_ident;
8+
use generate_dollar_ident::*;
9+
10+
macro_rules! black_hole {
11+
($($tt:tt)*) => {};
12+
}
13+
14+
black_hole!($var);
15+
16+
dollar_ident!(black_hole);
17+
18+
fn main() {}

0 commit comments

Comments
 (0)