Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit 1fe2c03

Browse files
nnethercotennethercote
committedDec 10, 2018
Remove tokenstream::Delimited.
Because it's an extra type layer that doesn't really help; in a couple of places it actively gets in the way, and overall removing it makes the code nicer. It does, however, move `tokenstream::TokenTree` further away from the `TokenTree` in `quote.rs`. More importantly, this change reduces the size of `TokenStream` from 48 bytes to 40 bytes on x86-64, which is enough to slightly reduce instruction counts on numerous benchmarks, the best by 1.5%. Note that `open_tt` and `close_tt` have gone from being methods on `Delimited` to associated methods of `TokenTree`.
1 parent b755501 commit 1fe2c03

File tree

18 files changed

+182
-203
lines changed

18 files changed

+182
-203
lines changed
 

‎src/librustc/hir/lowering.rs

+4-6
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ use syntax::ptr::P;
7272
use syntax::source_map::{self, respan, CompilerDesugaringKind, Spanned};
7373
use syntax::std_inject;
7474
use syntax::symbol::{keywords, Symbol};
75-
use syntax::tokenstream::{Delimited, TokenStream, TokenTree};
75+
use syntax::tokenstream::{TokenStream, TokenTree};
7676
use syntax::parse::token::Token;
7777
use syntax::visit::{self, Visitor};
7878
use syntax_pos::{Span, MultiSpan};
@@ -1088,12 +1088,10 @@ impl<'a> LoweringContext<'a> {
10881088
fn lower_token_tree(&mut self, tree: TokenTree) -> TokenStream {
10891089
match tree {
10901090
TokenTree::Token(span, token) => self.lower_token(token, span),
1091-
TokenTree::Delimited(span, delimited) => TokenTree::Delimited(
1091+
TokenTree::Delimited(span, delim, tts) => TokenTree::Delimited(
10921092
span,
1093-
Delimited {
1094-
delim: delimited.delim,
1095-
tts: self.lower_token_stream(delimited.tts.into()).into(),
1096-
},
1093+
delim,
1094+
self.lower_token_stream(tts.into()).into(),
10971095
).into(),
10981096
}
10991097
}

‎src/librustc/ich/impls_syntax.rs

+3-3
Original file line numberDiff line numberDiff line change
@@ -265,10 +265,10 @@ for tokenstream::TokenTree {
265265
span.hash_stable(hcx, hasher);
266266
hash_token(token, hcx, hasher);
267267
}
268-
tokenstream::TokenTree::Delimited(span, ref delimited) => {
268+
tokenstream::TokenTree::Delimited(span, delim, ref tts) => {
269269
span.hash_stable(hcx, hasher);
270-
std_hash::Hash::hash(&delimited.delim, hasher);
271-
for sub_tt in delimited.stream().trees() {
270+
std_hash::Hash::hash(&delim, hasher);
271+
for sub_tt in tts.stream().trees() {
272272
sub_tt.hash_stable(hcx, hasher);
273273
}
274274
}

‎src/librustc_lint/builtin.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -1540,8 +1540,8 @@ impl KeywordIdents {
15401540
}
15411541
_ => {},
15421542
}
1543-
TokenTree::Delimited(_, ref delim) => {
1544-
self.check_tokens(cx, delim.tts.clone().into())
1543+
TokenTree::Delimited(_, _, tts) => {
1544+
self.check_tokens(cx, tts.stream())
15451545
},
15461546
}
15471547
}

‎src/libsyntax/ast.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -1235,7 +1235,7 @@ pub enum MacDelimiter {
12351235

12361236
impl Mac_ {
12371237
pub fn stream(&self) -> TokenStream {
1238-
self.tts.clone().into()
1238+
self.tts.stream()
12391239
}
12401240
}
12411241

‎src/libsyntax/attr/mod.rs

+8-7
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ use parse::token::{self, Token};
3434
use ptr::P;
3535
use symbol::Symbol;
3636
use ThinVec;
37-
use tokenstream::{TokenStream, TokenTree, Delimited, DelimSpan};
37+
use tokenstream::{TokenStream, TokenTree, DelimSpan};
3838
use GLOBALS;
3939

4040
use std::iter;
@@ -549,10 +549,11 @@ impl MetaItemKind {
549549
}
550550
tokens.push(item.node.tokens());
551551
}
552-
TokenTree::Delimited(DelimSpan::from_single(span), Delimited {
553-
delim: token::Paren,
554-
tts: TokenStream::concat(tokens).into(),
555-
}).into()
552+
TokenTree::Delimited(
553+
DelimSpan::from_single(span),
554+
token::Paren,
555+
TokenStream::concat(tokens).into(),
556+
).into()
556557
}
557558
}
558559
}
@@ -570,9 +571,9 @@ impl MetaItemKind {
570571
None
571572
};
572573
}
573-
Some(TokenTree::Delimited(_, ref delimited)) if delimited.delim == token::Paren => {
574+
Some(TokenTree::Delimited(_, delim, ref tts)) if delim == token::Paren => {
574575
tokens.next();
575-
delimited.stream()
576+
tts.stream()
576577
}
577578
_ => return Some(MetaItemKind::Word),
578579
};

‎src/libsyntax/ext/expand.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -622,9 +622,9 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
622622
fn extract_proc_macro_attr_input(&self, tokens: TokenStream, span: Span) -> TokenStream {
623623
let mut trees = tokens.trees();
624624
match trees.next() {
625-
Some(TokenTree::Delimited(_, delim)) => {
625+
Some(TokenTree::Delimited(_, _, tts)) => {
626626
if trees.next().is_none() {
627-
return delim.tts.into()
627+
return tts.into()
628628
}
629629
}
630630
Some(TokenTree::Token(..)) => {}

‎src/libsyntax/ext/quote.rs

+15-17
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ pub mod rt {
3636
use symbol::Symbol;
3737
use ThinVec;
3838

39-
use tokenstream::{self, DelimSpan, TokenTree, TokenStream};
39+
use tokenstream::{DelimSpan, TokenTree, TokenStream};
4040

4141
pub use parse::new_parser_from_tts;
4242
pub use syntax_pos::{BytePos, Span, DUMMY_SP, FileName};
@@ -246,9 +246,9 @@ pub mod rt {
246246
inner.push(self.tokens.clone());
247247

248248
let delim_span = DelimSpan::from_single(self.span);
249-
r.push(TokenTree::Delimited(delim_span, tokenstream::Delimited {
250-
delim: token::Bracket, tts: TokenStream::concat(inner).into()
251-
}));
249+
r.push(TokenTree::Delimited(
250+
delim_span, token::Bracket, TokenStream::concat(inner).into()
251+
));
252252
r
253253
}
254254
}
@@ -262,10 +262,9 @@ pub mod rt {
262262

263263
impl ToTokens for () {
264264
fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
265-
vec![TokenTree::Delimited(DelimSpan::dummy(), tokenstream::Delimited {
266-
delim: token::Paren,
267-
tts: TokenStream::empty().into(),
268-
})]
265+
vec![
266+
TokenTree::Delimited(DelimSpan::dummy(), token::Paren, TokenStream::empty().into())
267+
]
269268
}
270269
}
271270

@@ -382,8 +381,6 @@ pub mod rt {
382381

383382
// Replaces `Token::OpenDelim .. Token::CloseDelim` with `TokenTree::Delimited(..)`.
384383
pub fn unflatten(tts: Vec<TokenTree>) -> Vec<TokenTree> {
385-
use tokenstream::Delimited;
386-
387384
let mut results = Vec::new();
388385
let mut result = Vec::new();
389386
let mut open_span = DUMMY_SP;
@@ -395,10 +392,11 @@ pub fn unflatten(tts: Vec<TokenTree>) -> Vec<TokenTree> {
395392
}
396393
TokenTree::Token(span, token::CloseDelim(delim)) => {
397394
let delim_span = DelimSpan::from_pair(open_span, span);
398-
let tree = TokenTree::Delimited(delim_span, Delimited {
395+
let tree = TokenTree::Delimited(
396+
delim_span,
399397
delim,
400-
tts: result.into_iter().map(TokenStream::from).collect::<TokenStream>().into(),
401-
});
398+
result.into_iter().map(TokenStream::from).collect::<TokenStream>().into(),
399+
);
402400
result = results.pop().unwrap();
403401
result.push(tree);
404402
}
@@ -758,10 +756,10 @@ fn statements_mk_tt(cx: &ExtCtxt, tt: &TokenTree, quoted: bool) -> Vec<ast::Stmt
758756
vec![e_tok]);
759757
vec![cx.stmt_expr(e_push)]
760758
},
761-
TokenTree::Delimited(span, ref delimed) => {
762-
let mut stmts = statements_mk_tt(cx, &delimed.open_tt(span.open), false);
763-
stmts.extend(statements_mk_tts(cx, delimed.stream()));
764-
stmts.extend(statements_mk_tt(cx, &delimed.close_tt(span.close), false));
759+
TokenTree::Delimited(span, delim, ref tts) => {
760+
let mut stmts = statements_mk_tt(cx, &TokenTree::open_tt(span.open, delim), false);
761+
stmts.extend(statements_mk_tts(cx, tts.stream()));
762+
stmts.extend(statements_mk_tt(cx, &TokenTree::close_tt(span.close, delim), false));
765763
stmts
766764
}
767765
}

‎src/libsyntax/ext/tt/quoted.rs

+8-8
Original file line numberDiff line numberDiff line change
@@ -280,17 +280,17 @@ where
280280
// `tree` is a `$` token. Look at the next token in `trees`
281281
tokenstream::TokenTree::Token(span, token::Dollar) => match trees.next() {
282282
// `tree` is followed by a delimited set of token trees. This indicates the beginning
283-
// of a repetition sequence in the macro (e.g., `$(pat)*`).
284-
Some(tokenstream::TokenTree::Delimited(span, delimited)) => {
283+
// of a repetition sequence in the macro (e.g. `$(pat)*`).
284+
Some(tokenstream::TokenTree::Delimited(span, delim, tts)) => {
285285
// Must have `(` not `{` or `[`
286-
if delimited.delim != token::Paren {
287-
let tok = pprust::token_to_string(&token::OpenDelim(delimited.delim));
286+
if delim != token::Paren {
287+
let tok = pprust::token_to_string(&token::OpenDelim(delim));
288288
let msg = format!("expected `(`, found `{}`", tok);
289289
sess.span_diagnostic.span_err(span.entire(), &msg);
290290
}
291291
// Parse the contents of the sequence itself
292292
let sequence = parse(
293-
delimited.tts.into(),
293+
tts.into(),
294294
expect_matchers,
295295
sess,
296296
features,
@@ -354,12 +354,12 @@ where
354354

355355
// `tree` is the beginning of a delimited set of tokens (e.g., `(` or `{`). We need to
356356
// descend into the delimited set and further parse it.
357-
tokenstream::TokenTree::Delimited(span, delimited) => TokenTree::Delimited(
357+
tokenstream::TokenTree::Delimited(span, delim, tts) => TokenTree::Delimited(
358358
span,
359359
Lrc::new(Delimited {
360-
delim: delimited.delim,
360+
delim: delim,
361361
tts: parse(
362-
delimited.tts.into(),
362+
tts.into(),
363363
expect_matchers,
364364
sess,
365365
features,

‎src/libsyntax/ext/tt/transcribe.rs

+6-5
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ use fold::noop_fold_tt;
1717
use parse::token::{self, Token, NtTT};
1818
use smallvec::SmallVec;
1919
use syntax_pos::DUMMY_SP;
20-
use tokenstream::{TokenStream, TokenTree, Delimited, DelimSpan};
20+
use tokenstream::{TokenStream, TokenTree, DelimSpan};
2121

2222
use rustc_data_structures::fx::FxHashMap;
2323
use rustc_data_structures::sync::Lrc;
@@ -105,10 +105,11 @@ pub fn transcribe(cx: &ExtCtxt,
105105
if result_stack.is_empty() {
106106
return TokenStream::concat(result);
107107
}
108-
let tree = TokenTree::Delimited(span, Delimited {
109-
delim: forest.delim,
110-
tts: TokenStream::concat(result).into(),
111-
});
108+
let tree = TokenTree::Delimited(
109+
span,
110+
forest.delim,
111+
TokenStream::concat(result).into(),
112+
);
112113
result = result_stack.pop().unwrap();
113114
result.push(tree.into());
114115
}

‎src/libsyntax/fold.rs

+3-5
Original file line numberDiff line numberDiff line change
@@ -605,12 +605,10 @@ pub fn noop_fold_tt<T: Folder>(tt: TokenTree, fld: &mut T) -> TokenTree {
605605
match tt {
606606
TokenTree::Token(span, tok) =>
607607
TokenTree::Token(fld.new_span(span), fld.fold_token(tok)),
608-
TokenTree::Delimited(span, delimed) => TokenTree::Delimited(
608+
TokenTree::Delimited(span, delim, tts) => TokenTree::Delimited(
609609
DelimSpan::from_pair(fld.new_span(span.open), fld.new_span(span.close)),
610-
Delimited {
611-
tts: fld.fold_tts(delimed.stream()).into(),
612-
delim: delimed.delim,
613-
}
610+
delim,
611+
fld.fold_tts(tts.stream()).into(),
614612
),
615613
}
616614
}

‎src/libsyntax/parse/lexer/tokentrees.rs

+5-4
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
use print::pprust::token_to_string;
1212
use parse::lexer::StringReader;
1313
use parse::{token, PResult};
14-
use tokenstream::{Delimited, DelimSpan, TokenStream, TokenTree};
14+
use tokenstream::{DelimSpan, TokenStream, TokenTree};
1515

1616
impl<'a> StringReader<'a> {
1717
// Parse a stream of tokens into a list of `TokenTree`s, up to an `Eof`.
@@ -155,10 +155,11 @@ impl<'a> StringReader<'a> {
155155
_ => {}
156156
}
157157

158-
Ok(TokenTree::Delimited(delim_span, Delimited {
158+
Ok(TokenTree::Delimited(
159+
delim_span,
159160
delim,
160-
tts: tts.into(),
161-
}).into())
161+
tts.into(),
162+
).into())
162163
},
163164
token::CloseDelim(_) => {
164165
// An unexpected closing delimiter (i.e., there is no

‎src/libsyntax/parse/mod.rs

+29-32
Original file line numberDiff line numberDiff line change
@@ -764,7 +764,7 @@ mod tests {
764764
use attr::first_attr_value_str_by_name;
765765
use parse;
766766
use print::pprust::item_to_string;
767-
use tokenstream::{self, DelimSpan, TokenTree};
767+
use tokenstream::{DelimSpan, TokenTree};
768768
use util::parser_testing::string_to_stream;
769769
use util::parser_testing::{string_to_expr, string_to_item};
770770
use with_globals;
@@ -795,42 +795,41 @@ mod tests {
795795
Some(&TokenTree::Token(_, token::Ident(name_macro_rules, false))),
796796
Some(&TokenTree::Token(_, token::Not)),
797797
Some(&TokenTree::Token(_, token::Ident(name_zip, false))),
798-
Some(&TokenTree::Delimited(_, ref macro_delimed)),
798+
Some(&TokenTree::Delimited(_, macro_delim, ref macro_tts)),
799799
)
800800
if name_macro_rules.name == "macro_rules"
801801
&& name_zip.name == "zip" => {
802-
let tts = &macro_delimed.stream().trees().collect::<Vec<_>>();
802+
let tts = &macro_tts.stream().trees().collect::<Vec<_>>();
803803
match (tts.len(), tts.get(0), tts.get(1), tts.get(2)) {
804804
(
805805
3,
806-
Some(&TokenTree::Delimited(_, ref first_delimed)),
806+
Some(&TokenTree::Delimited(_, first_delim, ref first_tts)),
807807
Some(&TokenTree::Token(_, token::FatArrow)),
808-
Some(&TokenTree::Delimited(_, ref second_delimed)),
808+
Some(&TokenTree::Delimited(_, second_delim, ref second_tts)),
809809
)
810-
if macro_delimed.delim == token::Paren => {
811-
let tts = &first_delimed.stream().trees().collect::<Vec<_>>();
810+
if macro_delim == token::Paren => {
811+
let tts = &first_tts.stream().trees().collect::<Vec<_>>();
812812
match (tts.len(), tts.get(0), tts.get(1)) {
813813
(
814814
2,
815815
Some(&TokenTree::Token(_, token::Dollar)),
816816
Some(&TokenTree::Token(_, token::Ident(ident, false))),
817817
)
818-
if first_delimed.delim == token::Paren && ident.name == "a" => {},
819-
_ => panic!("value 3: {:?}", *first_delimed),
818+
if first_delim == token::Paren && ident.name == "a" => {},
819+
_ => panic!("value 3: {:?} {:?}", first_delim, first_tts),
820820
}
821-
let tts = &second_delimed.stream().trees().collect::<Vec<_>>();
821+
let tts = &second_tts.stream().trees().collect::<Vec<_>>();
822822
match (tts.len(), tts.get(0), tts.get(1)) {
823823
(
824824
2,
825825
Some(&TokenTree::Token(_, token::Dollar)),
826826
Some(&TokenTree::Token(_, token::Ident(ident, false))),
827827
)
828-
if second_delimed.delim == token::Paren
829-
&& ident.name == "a" => {},
830-
_ => panic!("value 4: {:?}", *second_delimed),
828+
if second_delim == token::Paren && ident.name == "a" => {},
829+
_ => panic!("value 4: {:?} {:?}", second_delim, second_tts),
831830
}
832831
},
833-
_ => panic!("value 2: {:?}", *macro_delimed),
832+
_ => panic!("value 2: {:?} {:?}", macro_delim, macro_tts),
834833
}
835834
},
836835
_ => panic!("value: {:?}",tts),
@@ -848,26 +847,24 @@ mod tests {
848847
TokenTree::Token(sp(3, 4), token::Ident(Ident::from_str("a"), false)).into(),
849848
TokenTree::Delimited(
850849
DelimSpan::from_pair(sp(5, 6), sp(13, 14)),
851-
tokenstream::Delimited {
852-
delim: token::DelimToken::Paren,
853-
tts: TokenStream::concat(vec![
854-
TokenTree::Token(sp(6, 7),
855-
token::Ident(Ident::from_str("b"), false)).into(),
856-
TokenTree::Token(sp(8, 9), token::Colon).into(),
857-
TokenTree::Token(sp(10, 13),
858-
token::Ident(Ident::from_str("i32"), false)).into(),
859-
]).into(),
860-
}).into(),
850+
token::DelimToken::Paren,
851+
TokenStream::concat(vec![
852+
TokenTree::Token(sp(6, 7),
853+
token::Ident(Ident::from_str("b"), false)).into(),
854+
TokenTree::Token(sp(8, 9), token::Colon).into(),
855+
TokenTree::Token(sp(10, 13),
856+
token::Ident(Ident::from_str("i32"), false)).into(),
857+
]).into(),
858+
).into(),
861859
TokenTree::Delimited(
862860
DelimSpan::from_pair(sp(15, 16), sp(20, 21)),
863-
tokenstream::Delimited {
864-
delim: token::DelimToken::Brace,
865-
tts: TokenStream::concat(vec![
866-
TokenTree::Token(sp(17, 18),
867-
token::Ident(Ident::from_str("b"), false)).into(),
868-
TokenTree::Token(sp(18, 19), token::Semi).into(),
869-
]).into(),
870-
}).into()
861+
token::DelimToken::Brace,
862+
TokenStream::concat(vec![
863+
TokenTree::Token(sp(17, 18),
864+
token::Ident(Ident::from_str("b"), false)).into(),
865+
TokenTree::Token(sp(18, 19), token::Semi).into(),
866+
]).into(),
867+
).into()
871868
]);
872869

873870
assert_eq!(tts, expected);

0 commit comments

Comments
 (0)
Please sign in to comment.