proc_macro_server.rs

use crate::base::ExtCtxt;

use rustc_ast::ast;
use rustc_ast::token;
use rustc_ast::tokenstream::{self, DelimSpan, IsJoint::*, TokenStream, TreeAndJoint};
use rustc_ast::util::comments;
use rustc_ast_pretty::pprust;
use rustc_data_structures::sync::Lrc;
use rustc_errors::Diagnostic;
use rustc_parse::lexer::nfc_normalize;
use rustc_parse::{nt_to_tokenstream, parse_stream_from_source_str};
use rustc_session::parse::ParseSess;
use rustc_span::symbol::{self, kw, sym, Symbol};
use rustc_span::{BytePos, FileName, MultiSpan, Pos, SourceFile, Span};

use pm::bridge::{server, TokenTree};
use pm::{Delimiter, Level, LineColumn, Spacing};
use std::ops::Bound;
use std::{ascii, panic};

trait FromInternal<T> {
    fn from_internal(x: T) -> Self;
}

trait ToInternal<T> {
    fn to_internal(self) -> T;
}

impl FromInternal<token::DelimToken> for Delimiter {
    fn from_internal(delim: token::DelimToken) -> Delimiter {
        match delim {
            token::Paren => Delimiter::Parenthesis,
            token::Brace => Delimiter::Brace,
            token::Bracket => Delimiter::Bracket,
            token::NoDelim => Delimiter::None,
        }
    }
}

impl ToInternal<token::DelimToken> for Delimiter {
    fn to_internal(self) -> token::DelimToken {
        match self {
            Delimiter::Parenthesis => token::Paren,
            Delimiter::Brace => token::Brace,
            Delimiter::Bracket => token::Bracket,
            Delimiter::None => token::NoDelim,
        }
    }
}

impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec<Self>)>
    for TokenTree<Group, Punct, Ident, Literal>
{
    fn from_internal(
        ((tree, is_joint), sess, stack): (TreeAndJoint, &ParseSess, &mut Vec<Self>),
    ) -> Self {
        use rustc_ast::token::*;

        let joint = is_joint == Joint;
        let Token { kind, span } = match tree {
            tokenstream::TokenTree::Delimited(span, delim, tts) => {
                let delimiter = Delimiter::from_internal(delim);
                return TokenTree::Group(Group { delimiter, stream: tts, span });
            }
            tokenstream::TokenTree::Token(token) => token,
        };

        macro_rules! tt {
            ($ty:ident { $($field:ident $(: $value:expr)*),+ $(,)? }) => (
                TokenTree::$ty(self::$ty {
                    $($field $(: $value)*,)+
                    span,
                })
            );
            ($ty:ident::$method:ident($($value:expr),*)) => (
                TokenTree::$ty(self::$ty::$method($($value,)* span))
            );
        }
        macro_rules! op {
            ($a:expr) => {
                tt!(Punct::new($a, joint))
            };
            ($a:expr, $b:expr) => {{
                stack.push(tt!(Punct::new($b, joint)));
                tt!(Punct::new($a, true))
            }};
            ($a:expr, $b:expr, $c:expr) => {{
                stack.push(tt!(Punct::new($c, joint)));
                stack.push(tt!(Punct::new($b, true)));
                tt!(Punct::new($a, true))
            }};
        }

        match kind {
            Eq => op!('='),
            Lt => op!('<'),
            Le => op!('<', '='),
            EqEq => op!('=', '='),
            Ne => op!('!', '='),
            Ge => op!('>', '='),
            Gt => op!('>'),
            AndAnd => op!('&', '&'),
            OrOr => op!('|', '|'),
            Not => op!('!'),
            Tilde => op!('~'),
            BinOp(Plus) => op!('+'),
            BinOp(Minus) => op!('-'),
            BinOp(Star) => op!('*'),
            BinOp(Slash) => op!('/'),
            BinOp(Percent) => op!('%'),
            BinOp(Caret) => op!('^'),
            BinOp(And) => op!('&'),
            BinOp(Or) => op!('|'),
            BinOp(Shl) => op!('<', '<'),
            BinOp(Shr) => op!('>', '>'),
            BinOpEq(Plus) => op!('+', '='),
            BinOpEq(Minus) => op!('-', '='),
            BinOpEq(Star) => op!('*', '='),
            BinOpEq(Slash) => op!('/', '='),
            BinOpEq(Percent) => op!('%', '='),
            BinOpEq(Caret) => op!('^', '='),
            BinOpEq(And) => op!('&', '='),
            BinOpEq(Or) => op!('|', '='),
            BinOpEq(Shl) => op!('<', '<', '='),
            BinOpEq(Shr) => op!('>', '>', '='),
            At => op!('@'),
            Dot => op!('.'),
            DotDot => op!('.', '.'),
            DotDotDot => op!('.', '.', '.'),
            DotDotEq => op!('.', '.', '='),
            Comma => op!(','),
            Semi => op!(';'),
            Colon => op!(':'),
            ModSep => op!(':', ':'),
            RArrow => op!('-', '>'),
            LArrow => op!('<', '-'),
            FatArrow => op!('=', '>'),
            Pound => op!('#'),
            Dollar => op!('$'),
            Question => op!('?'),
            SingleQuote => op!('\''),

            Ident(name, false) if name == kw::DollarCrate => tt!(Ident::dollar_crate()),
            Ident(name, is_raw) => tt!(Ident::new(sess, name, is_raw)),
            Lifetime(name) => {
                let ident = symbol::Ident::new(name, span).without_first_quote();
                stack.push(tt!(Ident::new(sess, ident.name, false)));
                tt!(Punct::new('\'', true))
            }
            Literal(lit) => tt!(Literal { lit }),
            DocComment(c) => {
                let style = comments::doc_comment_style(&c.as_str());
                let stripped = comments::strip_doc_comment_decoration(&c.as_str());
                let mut escaped = String::new();
                for ch in stripped.chars() {
                    escaped.extend(ch.escape_debug());
                }
                let stream = vec![
                    Ident(sym::doc, false),
                    Eq,
                    TokenKind::lit(token::Str, Symbol::intern(&escaped), None),
                ]
                .into_iter()
                .map(|kind| tokenstream::TokenTree::token(kind, span))
                .collect();
                stack.push(TokenTree::Group(Group {
                    delimiter: Delimiter::Bracket,
                    stream,
                    span: DelimSpan::from_single(span),
                }));
                if style == ast::AttrStyle::Inner {
                    stack.push(tt!(Punct::new('!', false)));
                }
                tt!(Punct::new('#', false))
            }

            Interpolated(nt) => {
                let stream = nt_to_tokenstream(&nt, sess, span);
                TokenTree::Group(Group {
                    delimiter: Delimiter::None,
                    stream,
                    span: DelimSpan::from_single(span),
                })
            }

            OpenDelim(..) | CloseDelim(..) => unreachable!(),
            Whitespace | Comment | Shebang(..) | Unknown(..) | Eof => unreachable!(),
        }
    }
}

impl ToInternal<TokenStream> for TokenTree<Group, Punct, Ident, Literal> {
    fn to_internal(self) -> TokenStream {
        use rustc_ast::token::*;

        let (ch, joint, span) = match self {
            TokenTree::Punct(Punct { ch, joint, span }) => (ch, joint, span),
            TokenTree::Group(Group { delimiter, stream, span }) => {
                return tokenstream::TokenTree::Delimited(span, delimiter.to_internal(), stream)
                    .into();
            }
            TokenTree::Ident(self::Ident { sym, is_raw, span }) => {
                return tokenstream::TokenTree::token(Ident(sym, is_raw), span).into();
            }
            TokenTree::Literal(self::Literal {
                lit: token::Lit { kind: token::Integer, symbol, suffix },
                span,
            }) if symbol.as_str().starts_with('-') => {
                let minus = BinOp(BinOpToken::Minus);
                let symbol = Symbol::intern(&symbol.as_str()[1..]);
                let integer = TokenKind::lit(token::Integer, symbol, suffix);
                let a = tokenstream::TokenTree::token(minus, span);
                let b = tokenstream::TokenTree::token(integer, span);
                return vec![a, b].into_iter().collect();
            }
            TokenTree::Literal(self::Literal {
                lit: token::Lit { kind: token::Float, symbol, suffix },
                span,
            }) if symbol.as_str().starts_with('-') => {
                let minus = BinOp(BinOpToken::Minus);
                let symbol = Symbol::intern(&symbol.as_str()[1..]);
                let float = TokenKind::lit(token::Float, symbol, suffix);
                let a = tokenstream::TokenTree::token(minus, span);
                let b = tokenstream::TokenTree::token(float, span);
                return vec![a, b].into_iter().collect();
            }
            TokenTree::Literal(self::Literal { lit, span }) => {
                return tokenstream::TokenTree::token(Literal(lit), span).into();
            }
        };

        let kind = match ch {
            '=' => Eq,
            '<' => Lt,
            '>' => Gt,
            '!' => Not,
            '~' => Tilde,
            '+' => BinOp(Plus),
            '-' => BinOp(Minus),
            '*' => BinOp(Star),
            '/' => BinOp(Slash),
            '%' => BinOp(Percent),
            '^' => BinOp(Caret),
            '&' => BinOp(And),
            '|' => BinOp(Or),
            '@' => At,
            '.' => Dot,
            ',' => Comma,
            ';' => Semi,
            ':' => Colon,
            '#' => Pound,
            '$' => Dollar,
            '?' => Question,
            '\'' => SingleQuote,
            _ => unreachable!(),
        };

        let tree = tokenstream::TokenTree::token(kind, span);
        TokenStream::new(vec![(tree, if joint { Joint } else { NonJoint })])
    }
}

impl ToInternal<rustc_errors::Level> for Level {
    fn to_internal(self) -> rustc_errors::Level {
        match self {
            Level::Error => rustc_errors::Level::Error,
            Level::Warning => rustc_errors::Level::Warning,
            Level::Note => rustc_errors::Level::Note,
            Level::Help => rustc_errors::Level::Help,
            _ => unreachable!("unknown proc_macro::Level variant: {:?}", self),
        }
    }
}

#[derive(Clone)]
pub struct TokenStreamIter {
    cursor: tokenstream::Cursor,
    stack: Vec<TokenTree<Group, Punct, Ident, Literal>>,
}

#[derive(Clone)]
pub struct Group {
    delimiter: Delimiter,
    stream: TokenStream,
    span: DelimSpan,
}

#[derive(Copy, Clone, PartialEq, Eq, Hash)]
pub struct Punct {
    ch: char,
    // NB. not using `Spacing` here because it doesn't implement `Hash`.
    joint: bool,
    span: Span,
}

impl Punct {
    fn new(ch: char, joint: bool, span: Span) -> Punct {
        const LEGAL_CHARS: &[char] = &[
            '=', '<', '>', '!', '~', '+', '-', '*', '/', '%', '^', '&', '|', '@', '.', ',', ';',
            ':', '#', '$', '?', '\'',
        ];
        if !LEGAL_CHARS.contains(&ch) {
            panic!("unsupported character `{:?}`", ch)
        }
        Punct { ch, joint, span }
    }
}

#[derive(Copy, Clone, PartialEq, Eq, Hash)]
pub struct Ident {
    sym: Symbol,
    is_raw: bool,
    span: Span,
}

impl Ident {
    fn is_valid(string: &str) -> bool {
        let mut chars = string.chars();
        if let Some(start) = chars.next() {
            rustc_lexer::is_id_start(start) && chars.all(rustc_lexer::is_id_continue)
        } else {
            false
        }
    }
    fn new(sess: &ParseSess, sym: Symbol, is_raw: bool, span: Span) -> Ident {
        let sym = nfc_normalize(&sym.as_str());
        let string = sym.as_str();
        if !Self::is_valid(&string) {
            panic!("`{:?}` is not a valid identifier", string)
        }
        if is_raw && !sym.can_be_raw() {
            panic!("`{}` cannot be a raw identifier", string);
        }
        sess.symbol_gallery.insert(sym, span);
        Ident { sym, is_raw, span }
    }
    fn dollar_crate(span: Span) -> Ident {
        // `$crate` is accepted as an ident only if it comes from the compiler.
        Ident { sym: kw::DollarCrate, is_raw: false, span }
    }
}

// FIXME(eddyb) `Literal` should not expose internal `Debug` impls.
#[derive(Clone, Debug)]
pub struct Literal {
    lit: token::Lit,
    span: Span,
}

pub(crate) struct Rustc<'a> {
    sess: &'a ParseSess,
    def_site: Span,
    call_site: Span,
    mixed_site: Span,
}

impl<'a> Rustc<'a> {
    pub fn new(cx: &'a ExtCtxt<'_>) -> Self {
        let expn_data = cx.current_expansion.id.expn_data();
        Rustc {
            sess: cx.parse_sess,
            def_site: cx.with_def_site_ctxt(expn_data.def_site),
            call_site: cx.with_call_site_ctxt(expn_data.call_site),
            mixed_site: cx.with_mixed_site_ctxt(expn_data.call_site),
        }
    }

    fn lit(&mut self, kind: token::LitKind, symbol: Symbol, suffix: Option<Symbol>) -> Literal {
        Literal { lit: token::Lit::new(kind, symbol, suffix), span: server::Span::call_site(self) }
    }
}

impl server::Types for Rustc<'_> {
    type TokenStream = TokenStream;
    type TokenStreamBuilder = tokenstream::TokenStreamBuilder;
    type TokenStreamIter = TokenStreamIter;
    type Group = Group;
    type Punct = Punct;
    type Ident = Ident;
    type Literal = Literal;
    type SourceFile = Lrc<SourceFile>;
    type MultiSpan = Vec<Span>;
    type Diagnostic = Diagnostic;
    type Span = Span;
}

impl server::TokenStream for Rustc<'_> {
    fn new(&mut self) -> Self::TokenStream {
        TokenStream::default()
    }
    fn is_empty(&mut self, stream: &Self::TokenStream) -> bool {
        stream.is_empty()
    }
    fn from_str(&mut self, src: &str) -> Self::TokenStream {
        parse_stream_from_source_str(
            FileName::proc_macro_source_code(src),
            src.to_string(),
            self.sess,
            Some(self.call_site),
        )
    }
    fn to_string(&mut self, stream: &Self::TokenStream) -> String {
        pprust::tts_to_string(stream.clone())
    }
    fn from_token_tree(
        &mut self,
        tree: TokenTree<Self::Group, Self::Punct, Self::Ident, Self::Literal>,
    ) -> Self::TokenStream {
        tree.to_internal()
    }
    fn into_iter(&mut self, stream: Self::TokenStream) -> Self::TokenStreamIter {
        TokenStreamIter { cursor: stream.trees(), stack: vec![] }
    }
}

impl server::TokenStreamBuilder for Rustc<'_> {
    fn new(&mut self) -> Self::TokenStreamBuilder {
        tokenstream::TokenStreamBuilder::new()
    }
    fn push(&mut self, builder: &mut Self::TokenStreamBuilder, stream: Self::TokenStream) {
        builder.push(stream);
    }
    fn build(&mut self, builder: Self::TokenStreamBuilder) -> Self::TokenStream {
        builder.build()
    }
}

impl server::TokenStreamIter for Rustc<'_> {
    fn next(
        &mut self,
        iter: &mut Self::TokenStreamIter,
    ) -> Option<TokenTree<Self::Group, Self::Punct, Self::Ident, Self::Literal>> {
        loop {
            let tree = iter.stack.pop().or_else(|| {
                let next = iter.cursor.next_with_joint()?;
                Some(TokenTree::from_internal((next, self.sess, &mut iter.stack)))
            })?;
            // HACK: The condition "dummy span + group with empty delimiter" represents an AST
            // fragment approximately converted into a token stream. This may happen, for
            // example, with inputs to proc macro attributes, including derives. Such "groups"
            // need to flattened during iteration over stream's token trees.
            // Eventually this needs to be removed in favor of keeping original token trees
            // and not doing the roundtrip through AST.
            if let TokenTree::Group(ref group) = tree {
                if group.delimiter == Delimiter::None && group.span.entire().is_dummy() {
                    iter.cursor.append(group.stream.clone());
                    continue;
                }
            }
            return Some(tree);
        }
    }
}

impl server::Group for Rustc<'_> {
    fn new(&mut self, delimiter: Delimiter, stream: Self::TokenStream) -> Self::Group {
        Group { delimiter, stream, span: DelimSpan::from_single(server::Span::call_site(self)) }
    }
    fn delimiter(&mut self, group: &Self::Group) -> Delimiter {
        group.delimiter
    }
    fn stream(&mut self, group: &Self::Group) -> Self::TokenStream {
        group.stream.clone()
    }
    fn span(&mut self, group: &Self::Group) -> Self::Span {
        group.span.entire()
    }
    fn span_open(&mut self, group: &Self::Group) -> Self::Span {
        group.span.open
    }
    fn span_close(&mut self, group: &Self::Group) -> Self::Span {
        group.span.close
    }
    fn set_span(&mut self, group: &mut Self::Group, span: Self::Span) {
        group.span = DelimSpan::from_single(span);
    }
}

impl server::Punct for Rustc<'_> {
    fn new(&mut self, ch: char, spacing: Spacing) -> Self::Punct {
        Punct::new(ch, spacing == Spacing::Joint, server::Span::call_site(self))
    }
    fn as_char(&mut self, punct: Self::Punct) -> char {
        punct.ch
    }
    fn spacing(&mut self, punct: Self::Punct) -> Spacing {
        if punct.joint { Spacing::Joint } else { Spacing::Alone }
    }
    fn span(&mut self, punct: Self::Punct) -> Self::Span {
        punct.span
    }
    fn with_span(&mut self, punct: Self::Punct, span: Self::Span) -> Self::Punct {
        Punct { span, ..punct }
    }
}

impl server::Ident for Rustc<'_> {
    fn new(&mut self, string: &str, span: Self::Span, is_raw: bool) -> Self::Ident {
        Ident::new(self.sess, Symbol::intern(string), is_raw, span)
    }
    fn span(&mut self, ident: Self::Ident) -> Self::Span {
        ident.span
    }
    fn with_span(&mut self, ident: Self::Ident, span: Self::Span) -> Self::Ident {
        Ident { span, ..ident }
    }
}

impl server::Literal for Rustc<'_> {
    fn debug_kind(&mut self, literal: &Self::Literal) -> String {
        format!("{:?}", literal.lit.kind)
    }
    fn symbol(&mut self, literal: &Self::Literal) -> String {
        literal.lit.symbol.to_string()
    }
    fn suffix(&mut self, literal: &Self::Literal) -> Option<String> {
        literal.lit.suffix.as_ref().map(Symbol::to_string)
    }
    fn integer(&mut self, n: &str) -> Self::Literal {
        self.lit(token::Integer, Symbol::intern(n), None)
    }
    fn typed_integer(&mut self, n: &str, kind: &str) -> Self::Literal {
        self.lit(token::Integer, Symbol::intern(n), Some(Symbol::intern(kind)))
    }
    fn float(&mut self, n: &str) -> Self::Literal {
        self.lit(token::Float, Symbol::intern(n), None)
    }
    fn f32(&mut self, n: &str) -> Self::Literal {
        self.lit(token::Float, Symbol::intern(n), Some(sym::f32))
    }
    fn f64(&mut self, n: &str) -> Self::Literal {
        self.lit(token::Float, Symbol::intern(n), Some(sym::f64))
    }
    fn string(&mut self, string: &str) -> Self::Literal {
        let mut escaped = String::new();
        for ch in string.chars() {
            escaped.extend(ch.escape_debug());
        }
        self.lit(token::Str, Symbol::intern(&escaped), None)
    }
    fn character(&mut self, ch: char) -> Self::Literal {
        let mut escaped = String::new();
        escaped.extend(ch.escape_unicode());
        self.lit(token::Char, Symbol::intern(&escaped), None)
    }
    fn byte_string(&mut self, bytes: &[u8]) -> Self::Literal {
        let string = bytes
            .iter()
            .cloned()
            .flat_map(ascii::escape_default)
            .map(Into::<char>::into)
            .collect::<String>();
        self.lit(token::ByteStr, Symbol::intern(&string), None)
    }
    fn span(&mut self, literal: &Self::Literal) -> Self::Span {
        literal.span
    }
    fn set_span(&mut self, literal: &mut Self::Literal, span: Self::Span) {
        literal.span = span;
    }
    fn subspan(
        &mut self,
        literal: &Self::Literal,
        start: Bound<usize>,
        end: Bound<usize>,
    ) -> Option<Self::Span> {
        let span = literal.span;
        let length = span.hi().to_usize() - span.lo().to_usize();

        let start = match start {
            Bound::Included(lo) => lo,
            Bound::Excluded(lo) => lo + 1,
            Bound::Unbounded => 0,
        };

        let end = match end {
            Bound::Included(hi) => hi + 1,
            Bound::Excluded(hi) => hi,
            Bound::Unbounded => length,
        };

        // Bounds check the values, preventing addition overflow and OOB spans.
        if start > u32::max_value() as usize
            || end > u32::max_value() as usize
            || (u32::max_value() - start as u32) < span.lo().to_u32()
            || (u32::max_value() - end as u32) < span.lo().to_u32()
            || start >= end
            || end > length
        {
            return None;
        }

        let new_lo = span.lo() + BytePos::from_usize(start);
        let new_hi = span.lo() + BytePos::from_usize(end);
        Some(span.with_lo(new_lo).with_hi(new_hi))
    }
}

impl server::SourceFile for Rustc<'_> {
    fn eq(&mut self, file1: &Self::SourceFile, file2: &Self::SourceFile) -> bool {
        Lrc::ptr_eq(file1, file2)
    }
    fn path(&mut self, file: &Self::SourceFile) -> String {
        match file.name {
            FileName::Real(ref name) => name
                .local_path()
                .to_str()
                .expect("non-UTF8 file path in `proc_macro::SourceFile::path`")
                .to_string(),
            _ => file.name.to_string(),
        }
    }
    fn is_real(&mut self, file: &Self::SourceFile) -> bool {
        file.is_real_file()
    }
}

impl server::MultiSpan for Rustc<'_> {
    fn new(&mut self) -> Self::MultiSpan {
        vec![]
    }
    fn push(&mut self, spans: &mut Self::MultiSpan, span: Self::Span) {
        spans.push(span)
    }
}

impl server::Diagnostic for Rustc<'_> {
    fn new(&mut self, level: Level, msg: &str, spans: Self::MultiSpan) -> Self::Diagnostic {
        let mut diag = Diagnostic::new(level.to_internal(), msg);
        diag.set_span(MultiSpan::from_spans(spans));
        diag
    }
    fn sub(
        &mut self,
        diag: &mut Self::Diagnostic,
        level: Level,
        msg: &str,
        spans: Self::MultiSpan,
    ) {
        diag.sub(level.to_internal(), msg, MultiSpan::from_spans(spans), None);
    }
    fn emit(&mut self, diag: Self::Diagnostic) {
        self.sess.span_diagnostic.emit_diagnostic(&diag);
    }
}

impl server::Span for Rustc<'_> {
    fn debug(&mut self, span: Self::Span) -> String {
        format!("{:?} bytes({}..{})", span.ctxt(), span.lo().0, span.hi().0)
    }
    fn def_site(&mut self) -> Self::Span {
        self.def_site
    }
    fn call_site(&mut self) -> Self::Span {
        self.call_site
    }
    fn mixed_site(&mut self) -> Self::Span {
        self.mixed_site
    }
    fn source_file(&mut self, span: Self::Span) -> Self::SourceFile {
        self.sess.source_map().lookup_char_pos(span.lo()).file
    }
    fn parent(&mut self, span: Self::Span) -> Option<Self::Span> {
        span.parent()
    }
    fn source(&mut self, span: Self::Span) -> Self::Span {
        span.source_callsite()
    }
    fn start(&mut self, span: Self::Span) -> LineColumn {
        let loc = self.sess.source_map().lookup_char_pos(span.lo());
        LineColumn { line: loc.line, column: loc.col.to_usize() }
    }
    fn end(&mut self, span: Self::Span) -> LineColumn {
        let loc = self.sess.source_map().lookup_char_pos(span.hi());
        LineColumn { line: loc.line, column: loc.col.to_usize() }
    }
    fn join(&mut self, first: Self::Span, second: Self::Span) -> Option<Self::Span> {
        let self_loc = self.sess.source_map().lookup_char_pos(first.lo());
        let other_loc = self.sess.source_map().lookup_char_pos(second.lo());

        if self_loc.file.name != other_loc.file.name {
            return None;
        }

        Some(first.to(second))
    }
    fn resolved_at(&mut self, span: Self::Span, at: Self::Span) -> Self::Span {
        span.with_ctxt(at.ctxt())
    }
    fn source_text(&mut self, span: Self::Span) -> Option<String> {
        self.sess.source_map().span_to_snippet(span).ok()
    }
}