Skip to content

Commit 8ef2485

Browse files
borsbors
committed
Auto merge of #103812 - clubby789:improve-include-bytes, r=petrochenkov
Delay `include_bytes` to AST lowering Hopefully addresses #65818. This PR introduces a new `ExprKind::IncludedBytes` which stores the path and bytes of a file included with `include_bytes!()`. We can then create a literal from the bytes during AST lowering, which means we don't need to escape the bytes into valid UTF8 which is the cause of most of the overhead of embedding large binary blobs.
2 parents aa05f99 + b2da155 commit 8ef2485

File tree

19 files changed

+78
-15
lines changed

19 files changed

+78
-15
lines changed

‎compiler/rustc_ast/src/ast.rs

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1208,7 +1208,7 @@ impl Expr {
12081208
ExprKind::Tup(_) => ExprPrecedence::Tup,
12091209
ExprKind::Binary(op, ..) => ExprPrecedence::Binary(op.node),
12101210
ExprKind::Unary(..) => ExprPrecedence::Unary,
1211-
ExprKind::Lit(_) => ExprPrecedence::Lit,
1211+
ExprKind::Lit(_) | ExprKind::IncludedBytes(..) => ExprPrecedence::Lit,
12121212
ExprKind::Type(..) | ExprKind::Cast(..) => ExprPrecedence::Cast,
12131213
ExprKind::Let(..) => ExprPrecedence::Let,
12141214
ExprKind::If(..) => ExprPrecedence::If,
@@ -1446,6 +1446,12 @@ pub enum ExprKind {
14461446
/// with an optional value to be returned.
14471447
Yeet(Option<P<Expr>>),
14481448

1449+
/// Bytes included via `include_bytes!`
1450+
/// Added for optimization purposes to avoid the need to escape
1451+
/// large binary blobs - should always behave like [`ExprKind::Lit`]
1452+
/// with a `ByteStr` literal.
1453+
IncludedBytes(Lrc<[u8]>),
1454+
14491455
/// Placeholder for an expression that wasn't syntactically well formed in some way.
14501456
Err,
14511457
}

‎compiler/rustc_ast/src/mut_visit.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1428,7 +1428,7 @@ pub fn noop_visit_expr<T: MutVisitor>(
14281428
}
14291429
ExprKind::Try(expr) => vis.visit_expr(expr),
14301430
ExprKind::TryBlock(body) => vis.visit_block(body),
1431-
ExprKind::Lit(_) | ExprKind::Err => {}
1431+
ExprKind::Lit(_) | ExprKind::IncludedBytes(..) | ExprKind::Err => {}
14321432
}
14331433
vis.visit_id(id);
14341434
vis.visit_span(span);

‎compiler/rustc_ast/src/util/literal.rs

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
33
use crate::ast::{self, Lit, LitKind};
44
use crate::token::{self, Token};
5+
use rustc_data_structures::sync::Lrc;
56
use rustc_lexer::unescape::{byte_from_char, unescape_byte, unescape_char, unescape_literal, Mode};
67
use rustc_span::symbol::{kw, sym, Symbol};
78
use rustc_span::Span;
@@ -231,6 +232,13 @@ impl Lit {
231232
Lit { token_lit: kind.to_token_lit(), kind, span }
232233
}
233234

235+
/// Recovers an AST literal from a string of bytes produced by `include_bytes!`.
236+
/// This requires ASCII-escaping the string, which can result in poor performance
237+
/// for very large strings of bytes.
238+
pub fn from_included_bytes(bytes: &Lrc<[u8]>, span: Span) -> Lit {
239+
Self::from_lit_kind(LitKind::ByteStr(bytes.clone()), span)
240+
}
241+
234242
/// Losslessly convert an AST literal into a token.
235243
pub fn to_token(&self) -> Token {
236244
let kind = match self.token_lit.kind {

‎compiler/rustc_ast/src/visit.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -901,7 +901,7 @@ pub fn walk_expr<'a, V: Visitor<'a>>(visitor: &mut V, expression: &'a Expr) {
901901
}
902902
ExprKind::Try(ref subexpression) => visitor.visit_expr(subexpression),
903903
ExprKind::TryBlock(ref body) => visitor.visit_block(body),
904-
ExprKind::Lit(_) | ExprKind::Err => {}
904+
ExprKind::Lit(_) | ExprKind::IncludedBytes(..) | ExprKind::Err => {}
905905
}
906906

907907
visitor.visit_expr_post(expression)

‎compiler/rustc_ast_lowering/src/expr.rs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,10 @@ impl<'hir> LoweringContext<'_, 'hir> {
8787
ExprKind::Lit(ref l) => {
8888
hir::ExprKind::Lit(respan(self.lower_span(l.span), l.kind.clone()))
8989
}
90+
ExprKind::IncludedBytes(ref bytes) => hir::ExprKind::Lit(respan(
91+
self.lower_span(e.span),
92+
LitKind::ByteStr(bytes.clone()),
93+
)),
9094
ExprKind::Cast(ref expr, ref ty) => {
9195
let expr = self.lower_expr(expr);
9296
let ty =

‎compiler/rustc_ast_lowering/src/pat.rs

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -323,7 +323,10 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
323323
// ```
324324
fn lower_expr_within_pat(&mut self, expr: &Expr, allow_paths: bool) -> &'hir hir::Expr<'hir> {
325325
match expr.kind {
326-
ExprKind::Lit(..) | ExprKind::ConstBlock(..) | ExprKind::Err => {}
326+
ExprKind::Lit(..)
327+
| ExprKind::ConstBlock(..)
328+
| ExprKind::IncludedBytes(..)
329+
| ExprKind::Err => {}
327330
ExprKind::Path(..) if allow_paths => {}
328331
ExprKind::Unary(UnOp::Neg, ref inner) if matches!(inner.kind, ExprKind::Lit(_)) => {}
329332
_ => {

‎compiler/rustc_ast_pretty/src/pprust/state/expr.rs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -322,6 +322,10 @@ impl<'a> State<'a> {
322322
ast::ExprKind::Lit(ref lit) => {
323323
self.print_literal(lit);
324324
}
325+
ast::ExprKind::IncludedBytes(ref bytes) => {
326+
let lit = ast::Lit::from_included_bytes(bytes, expr.span);
327+
self.print_literal(&lit)
328+
}
325329
ast::ExprKind::Cast(ref expr, ref ty) => {
326330
let prec = AssocOp::As.precedence() as i8;
327331
self.print_expr_maybe_paren(expr, prec);

‎compiler/rustc_builtin_macros/src/assert/context.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -303,6 +303,7 @@ impl<'cx, 'a> Context<'cx, 'a> {
303303
| ExprKind::Field(_, _)
304304
| ExprKind::ForLoop(_, _, _, _)
305305
| ExprKind::If(_, _, _)
306+
| ExprKind::IncludedBytes(..)
306307
| ExprKind::InlineAsm(_)
307308
| ExprKind::Let(_, _, _)
308309
| ExprKind::Lit(_)

‎compiler/rustc_builtin_macros/src/concat.rs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,9 @@ pub fn expand_concat(
4343
has_errors = true;
4444
}
4545
},
46+
ast::ExprKind::IncludedBytes(..) => {
47+
cx.span_err(e.span, "cannot concatenate a byte string literal")
48+
}
4649
ast::ExprKind::Err => {
4750
has_errors = true;
4851
}

‎compiler/rustc_builtin_macros/src/concat_bytes.rs

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,16 @@ fn handle_array_element(
108108
None
109109
}
110110
},
111+
ast::ExprKind::IncludedBytes(..) => {
112+
if !*has_errors {
113+
cx.struct_span_err(expr.span, "cannot concatenate doubly nested array")
114+
.note("byte strings are treated as arrays of bytes")
115+
.help("try flattening the array")
116+
.emit();
117+
}
118+
*has_errors = true;
119+
None
120+
}
111121
_ => {
112122
missing_literals.push(expr.span);
113123
None
@@ -167,6 +177,9 @@ pub fn expand_concat_bytes(
167177
has_errors = true;
168178
}
169179
},
180+
ast::ExprKind::IncludedBytes(ref bytes) => {
181+
accumulator.extend_from_slice(bytes);
182+
}
170183
ast::ExprKind::Err => {
171184
has_errors = true;
172185
}

‎compiler/rustc_builtin_macros/src/source_util.rs

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -216,7 +216,10 @@ pub fn expand_include_bytes(
216216
}
217217
};
218218
match cx.source_map().load_binary_file(&file) {
219-
Ok(bytes) => base::MacEager::expr(cx.expr_byte_str(sp, bytes)),
219+
Ok(bytes) => {
220+
let expr = cx.expr(sp, ast::ExprKind::IncludedBytes(bytes.into()));
221+
base::MacEager::expr(expr)
222+
}
220223
Err(e) => {
221224
cx.span_err(sp, &format!("couldn't read {}: {}", file.display(), e));
222225
DummyResult::any(sp)

‎compiler/rustc_expand/src/proc_macro_server.rs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -525,6 +525,13 @@ impl server::TokenStream for Rustc<'_, '_> {
525525
ast::ExprKind::Lit(l) => {
526526
Ok(tokenstream::TokenStream::token_alone(token::Literal(l.token_lit), l.span))
527527
}
528+
ast::ExprKind::IncludedBytes(bytes) => {
529+
let lit = ast::Lit::from_included_bytes(bytes, expr.span);
530+
Ok(tokenstream::TokenStream::token_alone(
531+
token::TokenKind::Literal(lit.token_lit),
532+
expr.span,
533+
))
534+
}
528535
ast::ExprKind::Unary(ast::UnOp::Neg, e) => match &e.kind {
529536
ast::ExprKind::Lit(l) => match l.token_lit {
530537
token::Lit { kind: token::Integer | token::Float, .. } => {

‎compiler/rustc_parse/src/parser/path.rs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -631,7 +631,9 @@ impl<'a> Parser<'a> {
631631
/// - A single-segment path.
632632
pub(super) fn expr_is_valid_const_arg(&self, expr: &P<rustc_ast::Expr>) -> bool {
633633
match &expr.kind {
634-
ast::ExprKind::Block(_, _) | ast::ExprKind::Lit(_) => true,
634+
ast::ExprKind::Block(_, _)
635+
| ast::ExprKind::Lit(_)
636+
| ast::ExprKind::IncludedBytes(..) => true,
635637
ast::ExprKind::Unary(ast::UnOp::Neg, expr) => {
636638
matches!(expr.kind, ast::ExprKind::Lit(_))
637639
}

‎compiler/rustc_passes/src/hir_stats.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -560,13 +560,14 @@ impl<'v> ast_visit::Visitor<'v> for StatCollector<'v> {
560560
}
561561

562562
fn visit_expr(&mut self, e: &'v ast::Expr) {
563+
#[rustfmt::skip]
563564
record_variants!(
564565
(self, e, e.kind, Id::None, ast, Expr, ExprKind),
565566
[
566567
Box, Array, ConstBlock, Call, MethodCall, Tup, Binary, Unary, Lit, Cast, Type, Let,
567568
If, While, ForLoop, Loop, Match, Closure, Block, Async, Await, TryBlock, Assign,
568569
AssignOp, Field, Index, Range, Underscore, Path, AddrOf, Break, Continue, Ret,
569-
InlineAsm, MacCall, Struct, Repeat, Paren, Try, Yield, Yeet, Err
570+
InlineAsm, MacCall, Struct, Repeat, Paren, Try, Yield, Yeet, IncludedBytes, Err
570571
]
571572
);
572573
ast_visit::walk_expr(self, e)

‎src/test/ui/proc-macro/expand-expr.rs

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
// aux-build:expand-expr.rs
2-
2+
#![feature(concat_bytes)]
33
extern crate expand_expr;
44

55
use expand_expr::{
@@ -23,6 +23,11 @@ expand_expr_is!(
2323
concat!("contents: ", include_str!("auxiliary/included-file.txt"))
2424
);
2525

26+
expand_expr_is!(
27+
b"contents: Included file contents\n",
28+
concat_bytes!(b"contents: ", include_bytes!("auxiliary/included-file.txt"))
29+
);
30+
2631
// Correct value is checked for multiple sources.
2732
check_expand_expr_file!(file!());
2833

‎src/test/ui/proc-macro/expand-expr.stderr

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,29 @@
11
error: expected one of `.`, `?`, or an operator, found `;`
2-
--> $DIR/expand-expr.rs:101:27
2+
--> $DIR/expand-expr.rs:106:27
33
|
44
LL | expand_expr_fail!("string"; hello);
55
| ^ expected one of `.`, `?`, or an operator
66

77
error: expected expression, found `$`
8-
--> $DIR/expand-expr.rs:104:19
8+
--> $DIR/expand-expr.rs:109:19
99
|
1010
LL | expand_expr_fail!($);
1111
| ^ expected expression
1212

1313
error: expected expression, found `$`
14-
--> $DIR/expand-expr.rs:33:23
14+
--> $DIR/expand-expr.rs:38:23
1515
|
1616
LL | ($($t:tt)*) => { $($t)* };
1717
| ^^^^ expected expression
1818

1919
error: expected expression, found `$`
20-
--> $DIR/expand-expr.rs:106:28
20+
--> $DIR/expand-expr.rs:111:28
2121
|
2222
LL | expand_expr_fail!(echo_pm!($));
2323
| ^ expected expression
2424

2525
error: macro expansion ignores token `hello` and any following
26-
--> $DIR/expand-expr.rs:110:47
26+
--> $DIR/expand-expr.rs:115:47
2727
|
2828
LL | expand_expr_is!("string", echo_tts!("string"; hello));
2929
| --------------------^^^^^-- help: you might be missing a semicolon here: `;`
@@ -33,7 +33,7 @@ LL | expand_expr_is!("string", echo_tts!("string"; hello));
3333
= note: the usage of `echo_tts!` is likely invalid in expression context
3434

3535
error: macro expansion ignores token `;` and any following
36-
--> $DIR/expand-expr.rs:111:44
36+
--> $DIR/expand-expr.rs:116:44
3737
|
3838
LL | expand_expr_is!("string", echo_pm!("string"; hello));
3939
| -----------------^-------- help: you might be missing a semicolon here: `;`
@@ -43,7 +43,7 @@ LL | expand_expr_is!("string", echo_pm!("string"; hello));
4343
= note: the usage of `echo_pm!` is likely invalid in expression context
4444

4545
error: recursion limit reached while expanding `recursive_expand!`
46-
--> $DIR/expand-expr.rs:119:16
46+
--> $DIR/expand-expr.rs:124:16
4747
|
4848
LL | const _: u32 = recursive_expand!();
4949
| ^^^^^^^^^^^^^^^^^^^

‎src/tools/clippy/clippy_utils/src/sugg.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -207,6 +207,7 @@ impl<'a> Sugg<'a> {
207207
| ast::ExprKind::InlineAsm(..)
208208
| ast::ExprKind::ConstBlock(..)
209209
| ast::ExprKind::Lit(..)
210+
| ast::ExprKind::IncludedBytes(..)
210211
| ast::ExprKind::Loop(..)
211212
| ast::ExprKind::MacCall(..)
212213
| ast::ExprKind::MethodCall(..)

‎src/tools/rustfmt/src/expr.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -399,6 +399,7 @@ pub(crate) fn format_expr(
399399
}
400400
}
401401
ast::ExprKind::Underscore => Some("_".to_owned()),
402+
ast::ExprKind::IncludedBytes(..) => unreachable!(),
402403
ast::ExprKind::Err => None,
403404
};
404405

‎src/tools/rustfmt/src/utils.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -496,6 +496,7 @@ pub(crate) fn is_block_expr(context: &RewriteContext<'_>, expr: &ast::Expr, repr
496496
| ast::ExprKind::Continue(..)
497497
| ast::ExprKind::Err
498498
| ast::ExprKind::Field(..)
499+
| ast::ExprKind::IncludedBytes(..)
499500
| ast::ExprKind::InlineAsm(..)
500501
| ast::ExprKind::Let(..)
501502
| ast::ExprKind::Path(..)

0 commit comments

Comments
 (0)