Gate things on feature=parser all over the library

Caveats:
- `dialect` is completely disabled which is not the final goal.
- `parser/mod.rs` and `tokenizer/mod.rs` should be split so that
  the number of annotations is greatly reduced and applied at the
  file level instead of struct/function level.

Author: felipecrv

Diff for: Cargo.toml

@@ -65,3 +65,11 @@ pretty_assertions = "1"
 [package.metadata.docs.rs]
 # Document these features on docs.rs
 features = ["parser", "serde", "visitor"]
+
+[[example]]
+name = "cli"
+required-features = ["parser"]
+
+[[example]]
+name = "parse_select"
+required-features = ["parser"]

Diff for: src/ast/helpers/stmt_create_table.rs

@@ -588,6 +588,7 @@ impl TryFrom<Statement> for CreateTableBuilder {
 
 /// Helper return type when parsing configuration for a `CREATE TABLE` statement.
 #[derive(Default)]
+#[cfg(feature = "parser")]
 pub(crate) struct CreateTableConfiguration {
     pub partition_by: Option<Box<Expr>>,
     pub cluster_by: Option<WrappedCollection<Vec<Ident>>>,

Diff for: src/ast/spans.rs

@@ -59,10 +59,13 @@ fn union_spans<I: Iterator<Item = Span>>(iter: I) -> Span {
 ///
 /// # Example
 /// ```
+/// # #[cfg(feature = "parser")]
 /// # use sqlparser::parser::{Parser, ParserError};
 /// # use sqlparser::ast::Spanned;
+/// # #[cfg(feature = "parser")]
 /// # use sqlparser::dialect::GenericDialect;
 /// # use sqlparser::tokenizer::Location;
+/// # #[cfg(feature = "parser")]
 /// # fn main() -> Result<(), ParserError> {
 /// let dialect = GenericDialect {};
 /// let sql = r#"SELECT *
@@ -78,6 +81,9 @@ fn union_spans<I: Iterator<Item = Span>>(iter: I) -> Span {
 /// assert_eq!(span.end, Location::new(2, 15));
 /// # Ok(())
 /// # }
+/// #
+/// # #[cfg(not(feature = "parser"))]
+/// # fn main() {}
 /// ```
 ///
 pub trait Spanned {
@@ -2167,6 +2173,7 @@ impl Spanned for TableObject {
 }
 
 #[cfg(test)]
+#[cfg(feature = "parser")]
 pub mod tests {
     use crate::dialect::{Dialect, GenericDialect, SnowflakeDialect};
     use crate::parser::Parser;

Diff for: src/lib.rs

@@ -32,18 +32,23 @@
 //! # Example parsing SQL text
 //!
 //! ```
+//! # #[cfg(feature = "parser")]
 //! use sqlparser::dialect::GenericDialect;
+//! # #[cfg(feature = "parser")]
 //! use sqlparser::parser::Parser;
 //!
+//! # #[cfg(feature = "parser")]
 //! let dialect = GenericDialect {}; // or AnsiDialect
 //!
 //! let sql = "SELECT a, b, 123, myfunc(b) \
 //!            FROM table_1 \
 //!            WHERE a > b AND b < 100 \
 //!            ORDER BY a DESC, b";
 //!
+//! # #[cfg(feature = "parser")]
 //! let ast = Parser::parse_sql(&dialect, sql).unwrap();
 //!
+//! # #[cfg(feature = "parser")]
 //! println!("AST: {:?}", ast);
 //! ```
 //!
@@ -54,13 +59,17 @@
 //! useful for tools that analyze and manipulate SQL.
 //!
 //! ```
+//! # #[cfg(feature = "parser")]
 //! # use sqlparser::dialect::GenericDialect;
+//! # #[cfg(feature = "parser")]
 //! # use sqlparser::parser::Parser;
 //! let sql = "SELECT a FROM table_1";
 //!
+//! # #[cfg(feature = "parser")]
 //! // parse to a Vec<Statement>
 //! let ast = Parser::parse_sql(&GenericDialect, sql).unwrap();
 //!
+//! # #[cfg(feature = "parser")]
 //! // The original SQL text can be generated from the AST
 //! assert_eq!(ast[0].to_string(), sql);
 //! ```
@@ -141,13 +150,17 @@ extern crate pretty_assertions;
 
 pub mod ast;
 #[macro_use]
+#[cfg(feature = "parser")]
 pub mod dialect;
 pub mod keywords;
 pub mod parser;
 pub mod tokenizer;
 
-#[doc(hidden)]
 // This is required to make utilities accessible by both the crate-internal
 // unit-tests and by the integration tests <https://stackoverflow.com/a/44541071/1026>
-// External users are not supposed to rely on this module.
+// External users are not supposed to rely on these modules.
+#[doc(hidden)]
+#[cfg(feature = "parser")]
+pub mod test_dialect_utils;
+#[doc(hidden)]
 pub mod test_utils;

Diff for: src/parser/mod.rs

@@ -20,25 +20,36 @@ use alloc::{
     vec,
     vec::Vec,
 };
+#[allow(unused_imports)]
 use core::{
     fmt::{self, Display},
     str::FromStr,
 };
+#[allow(unused_imports)]
 use helpers::attached_token::AttachedToken;
 
+#[allow(unused_imports)]
 use log::debug;
 
+#[cfg(feature = "parser")]
 use recursion::RecursionCounter;
+#[allow(unused_imports)]
 use IsLateral::*;
+#[allow(unused_imports)]
 use IsOptional::*;
 
+#[cfg(feature = "parser")]
 use crate::ast::helpers::stmt_create_table::{CreateTableBuilder, CreateTableConfiguration};
+#[allow(unused_imports)]
 use crate::ast::Statement::CreatePolicy;
 use crate::ast::*;
+#[cfg(feature = "parser")]
 use crate::dialect::*;
+#[allow(unused_imports)]
 use crate::keywords::{Keyword, ALL_KEYWORDS};
 use crate::tokenizer::*;
 
+#[cfg(feature = "parser")]
 mod alter;
 
 #[derive(Debug, Clone, PartialEq, Eq)]
@@ -49,13 +60,15 @@ pub enum ParserError {
 }
 
 // Use `Parser::expected` instead, if possible
+#[allow(unused_macros)]
 macro_rules! parser_err {
     ($MSG:expr, $loc:expr) => {
         Err(ParserError::ParserError(format!("{}{}", $MSG, $loc)))
     };
 }
 
 #[cfg(feature = "std")]
+#[cfg(feature = "parser")]
 /// Implementation [`RecursionCounter`] if std is available
 mod recursion {
     use std::cell::Cell;
@@ -184,9 +197,11 @@ impl fmt::Display for ParserError {
 impl std::error::Error for ParserError {}
 
 // By default, allow expressions up to this deep before erroring
+#[allow(unused)]
 const DEFAULT_REMAINING_DEPTH: usize = 50;
 
 // A constant EOF token that can be referenced.
+#[allow(unused)]
 const EOF_TOKEN: TokenWithSpan = TokenWithSpan {
     token: Token::EOF,
     span: Span {
@@ -207,8 +222,10 @@ const EOF_TOKEN: TokenWithSpan = TokenWithSpan {
 /// child type.
 ///
 /// See [Parser::parse_data_type] for details
+#[cfg(feature = "parser")]
 struct MatchedTrailingBracket(bool);
 
+#[cfg(feature = "parser")]
 impl From<bool> for MatchedTrailingBracket {
     fn from(value: bool) -> Self {
         Self(value)
@@ -264,6 +281,7 @@ impl ParserOptions {
 }
 
 #[derive(Copy, Clone)]
+#[allow(unused)]
 enum ParserState {
     /// The default state of the parser.
     Normal,
@@ -309,8 +327,7 @@ enum ParserState {
 ///    "foo"
 ///   ]
 /// ```
-///
-///
+#[cfg(feature = "parser")]
 pub struct Parser<'a> {
     /// The tokens
     tokens: Vec<TokenWithSpan>,
@@ -328,6 +345,7 @@ pub struct Parser<'a> {
     recursion_counter: RecursionCounter,
 }
 
+#[cfg(feature = "parser")]
 impl<'a> Parser<'a> {
     /// Create a parser for a [`Dialect`]
     ///

Diff for: src/tokenizer/mod.rs

@@ -40,13 +40,13 @@ use serde::{Deserialize, Serialize};
 #[cfg(feature = "visitor")]
 use sqlparser_derive::{Visit, VisitMut};
 
-use crate::dialect::Dialect;
+#[cfg(feature = "parser")]
 use crate::dialect::{
-    BigQueryDialect, DuckDbDialect, GenericDialect, MySqlDialect, PostgreSqlDialect,
-    SnowflakeDialect,
+    BigQueryDialect, Dialect, DuckDbDialect, GenericDialect, HiveDialect, MySqlDialect,
+    PostgreSqlDialect, SnowflakeDialect,
 };
 use crate::keywords::{Keyword, ALL_KEYWORDS, ALL_KEYWORDS_INDEX};
-use crate::{ast::DollarQuotedString, dialect::HiveDialect};
+use crate::ast::DollarQuotedString;
 
 /// SQL Token enumeration
 #[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)]
@@ -690,12 +690,14 @@ impl fmt::Display for TokenizerError {
 #[cfg(feature = "std")]
 impl std::error::Error for TokenizerError {}
 
+#[allow(unused)]
 struct State<'a> {
     peekable: Peekable<Chars<'a>>,
     pub line: u64,
     pub col: u64,
 }
 
+#[allow(unused)]
 impl State<'_> {
     /// return the next character and advance the stream
     pub fn next(&mut self) -> Option<char> {
@@ -727,6 +729,7 @@ impl State<'_> {
 }
 
 /// Represents how many quote characters enclose a string literal.
+#[allow(unused)]
 #[derive(Copy, Clone)]
 enum NumStringQuoteChars {
     /// e.g. `"abc"`, `'abc'`, `r'abc'`
@@ -736,6 +739,7 @@ enum NumStringQuoteChars {
 }
 
 /// Settings for tokenizing a quoted string literal.
+#[allow(unused)]
 struct TokenizeQuotedStringSettings {
     /// The character used to quote the string.
     quote_style: char,
@@ -753,6 +757,7 @@ struct TokenizeQuotedStringSettings {
 }
 
 /// SQL Tokenizer
+#[cfg(feature = "parser")]
 pub struct Tokenizer<'a> {
     dialect: &'a dyn Dialect,
     query: &'a str,
@@ -761,6 +766,7 @@ pub struct Tokenizer<'a> {
     unescape: bool,
 }
 
+#[cfg(feature = "parser")]
 impl<'a> Tokenizer<'a> {
     /// Create a new SQL tokenizer for the specified SQL statement
     ///
@@ -1948,6 +1954,7 @@ impl<'a> Tokenizer<'a> {
 /// Read from `chars` until `predicate` returns `false` or EOF is hit.
 /// Return the characters read as String, and keep the first non-matching
 /// char available as `chars.next()`.
+#[cfg(feature = "parser")]
 fn peeking_take_while(chars: &mut State, mut predicate: impl FnMut(char) -> bool) -> String {
     let mut s = String::new();
     while let Some(&ch) = chars.peek() {
@@ -1962,6 +1969,7 @@ fn peeking_take_while(chars: &mut State, mut predicate: impl FnMut(char) -> bool
 }
 
 /// Same as peeking_take_while, but also passes the next character to the predicate.
+#[cfg(feature = "parser")]
 fn peeking_next_take_while(
     chars: &mut State,
     mut predicate: impl FnMut(char, Option<char>) -> bool,
@@ -1979,14 +1987,17 @@ fn peeking_next_take_while(
     s
 }
 
+#[cfg(feature = "parser")]
 fn unescape_single_quoted_string(chars: &mut State<'_>) -> Option<String> {
     Unescape::new(chars).unescape()
 }
 
+#[cfg(feature = "parser")]
 struct Unescape<'a: 'b, 'b> {
     chars: &'b mut State<'a>,
 }
 
+#[cfg(feature = "parser")]
 impl<'a: 'b, 'b> Unescape<'a, 'b> {
     fn new(chars: &'b mut State<'a>) -> Self {
         Self { chars }
@@ -2127,6 +2138,7 @@ impl<'a: 'b, 'b> Unescape<'a, 'b> {
     }
 }
 
+#[cfg(feature = "parser")]
 fn unescape_unicode_single_quoted_string(chars: &mut State<'_>) -> Result<String, TokenizerError> {
     let mut unescaped = String::new();
     chars.next(); // consume the opening quote
@@ -2162,6 +2174,7 @@ fn unescape_unicode_single_quoted_string(chars: &mut State<'_>) -> Result<String
     })
 }
 
+#[cfg(feature = "parser")]
 fn take_char_from_hex_digits(
     chars: &mut State<'_>,
     max_digits: usize,