feat: select from stage with files/pattern.

src/query/ast/src/ast/format/ast_format.rs

@@ -2199,15 +2199,19 @@ impl<'ast> Visitor<'ast> for AstFormatVisitor {
             TableReference::Stage {
                 span: _,
                 location,
-                files,
+                options,
                 alias,
             } => {
                 let mut children = Vec::new();
-                if !files.is_empty() {
+                if let Some(files) = &options.files {
                     let files = files.join(",");
                     let files = format!("files = {}", files);
                     children.push(FormatTreeNode::new(AstFormatContext::new(files)))
                 }
+                if let Some(pattern) = &options.pattern {
+                    let pattern = format!("patter = {}", pattern);
+                    children.push(FormatTreeNode::new(AstFormatContext::new(pattern)))
+                }
                 let stage_name = format!("Stage {:?}", location);
                 let format_ctx = if let Some(alias) = alias {
                     AstFormatContext::with_children_alias(

src/query/ast/src/ast/format/syntax/query.rs

@@ -331,15 +331,21 @@ pub(crate) fn pretty_table(table: TableReference) -> RcDoc<'static> {
         TableReference::Stage {
             span: _,
             location,
-            files,
+            options,
             alias,
         } => RcDoc::text(location.to_string())
-            .append(if files.is_empty() {
-                RcDoc::nil()
-            } else {
+            .append(if let Some(files) = options.files {
                 let files = files.join(",");
                 let files = format!("FILES {}", files);
                 RcDoc::text(files)
+            } else {
+                RcDoc::nil()
+            })
+            .append(if let Some(pattern) = options.pattern {
+                let pattern = format!("Pattern {pattern}");
+                RcDoc::text(pattern)
+            } else {
+                RcDoc::nil()
             })
             .append(if let Some(a) = alias {
                 RcDoc::text(format!(" AS {a}"))

src/query/ast/src/ast/query.rs

@@ -22,6 +22,7 @@ use crate::ast::write_period_separated_list;
 use crate::ast::Expr;
 use crate::ast::FileLocation;
 use crate::ast::Identifier;
+use crate::ast::SelectStageOptions;
 
 /// Root node of a query tree
 #[derive(Debug, Clone, PartialEq)]
@@ -181,7 +182,7 @@ pub enum TableReference {
     Stage {
         span: Span,
         location: FileLocation,
-        files: Vec<String>,
+        options: SelectStageOptions,
         alias: Option<TableAlias>,
     },
 }
@@ -367,14 +368,17 @@ impl Display for TableReference {
             TableReference::Stage {
                 span: _,
                 location,
-                files,
+                options,
                 alias,
             } => {
                 write!(f, "({location})")?;
-                if !files.is_empty() {
+                if let Some(files) = &options.files {
                     let files = files.join(",");
                     write!(f, " FILES {files}")?;
                 }
+                if let Some(pattern) = &options.pattern {
+                    write!(f, " PATTERN {pattern}")?;
+                }
                 if let Some(alias) = alias {
                     write!(f, " AS {alias}")?;
                 }

src/query/ast/src/ast/statements/stage.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use std::collections::BTreeMap;
+use std::default::Default;
 use std::fmt::Display;
 use std::fmt::Formatter;
 
@@ -72,3 +73,28 @@ impl Display for CreateStageStmt {
         Ok(())
     }
 }
+
+#[derive(Debug, Clone, PartialEq)]
+pub enum SelectStageOption {
+    Files(Vec<String>),
+    Pattern(String),
+}
+
+#[derive(Debug, Clone, PartialEq, Default)]
+pub struct SelectStageOptions {
+    pub files: Option<Vec<String>>,
+    pub pattern: Option<String>,
+}
+
+impl SelectStageOptions {
+    pub fn from(opts: Vec<SelectStageOption>) -> Self {
+        let mut options: SelectStageOptions = Default::default();
+        for opt in opts.into_iter() {
+            match opt {
+                SelectStageOption::Files(v) => options.files = Some(v),
+                SelectStageOption::Pattern(v) => options.pattern = Some(v),
+            }
+        }
+        options
+    }
+}

src/query/ast/src/parser/query.rs

@@ -21,6 +21,7 @@ use pratt::Associativity;
 use pratt::PrattParser;
 use pratt::Precedence;
 
+use super::stage::select_stage_option;
 use super::stage::stage_location;
 use super::stage::uri_location;
 use crate::ast::*;
@@ -283,7 +284,7 @@ pub enum TableReferenceElement {
     Group(TableReference),
     Stage {
         location: FileLocation,
-        files: Vec<String>,
+        options: Vec<SelectStageOption>,
         alias: Option<TableAlias>,
     },
 }
@@ -368,12 +369,18 @@ pub fn table_reference_element(i: Input) -> IResult<WithSpan<TableReferenceEleme
 
     let aliased_stage = map(
         rule! {
-            (#stage_location | #uri_location) ~ ( FILES ~ "=" ~ "(" ~ #comma_separated_list0(literal_string) ~ ")")? ~ #table_alias?
+            (#stage_location | #uri_location) ~  ("(" ~ ( #select_stage_option )* ~")")? ~ #table_alias?
         },
-        |(location, files, alias)| TableReferenceElement::Stage {
-            location,
-            alias,
-            files: files.map(|v| v.3).unwrap_or_default(),
+        |(location, options, alias)| {
+            let options = match options {
+                None => vec![],
+                Some((_, v, _)) => v,
+            };
+            TableReferenceElement::Stage {
+                location,
+                alias,
+                options,
+            }
         },
     );
 
@@ -459,14 +466,17 @@ impl<'a, I: Iterator<Item = WithSpan<'a, TableReferenceElement>>> PrattParser<I>
             },
             TableReferenceElement::Stage {
                 location,
-                files,
+                options,
                 alias,
-            } => TableReference::Stage {
-                span: transform_span(input.span.0),
-                location,
-                files,
-                alias,
-            },
+            } => {
+                let options = SelectStageOptions::from(options);
+                TableReference::Stage {
+                    span: transform_span(input.span.0),
+                    location,
+                    options,
+                    alias,
+                }
+            }
             _ => unreachable!(),
         };
         Ok(table_ref)

src/query/ast/src/parser/stage.rs

@@ -13,9 +13,11 @@
 // limitations under the License.
 use std::collections::BTreeMap;
 
+use nom::branch::alt;
 use nom::combinator::map;
 use url::Url;
 
+use crate::ast::SelectStageOption;
 use crate::ast::StageLocation;
 use crate::ast::UriLocation;
 use crate::input::Input;
@@ -200,3 +202,16 @@ pub fn uri_location(i: Input) -> IResult<UriLocation> {
         },
     )(i)
 }
+
+pub fn select_stage_option(i: Input) -> IResult<SelectStageOption> {
+    alt((
+        map(
+            rule! { FILES ~ "=>" ~ "(" ~ #comma_separated_list0(literal_string) ~ ")" },
+            |(_, _, _, files, _)| SelectStageOption::Files(files),
+        ),
+        map(
+            rule! { PATTERN ~ "=>" ~ #literal_string },
+            |(_, _, pattern)| SelectStageOption::Pattern(pattern),
+        ),
+    ))(i)
+}

src/query/sql/src/planner/binder/table.rs

@@ -319,7 +319,7 @@ impl Binder {
             TableReference::Stage {
                 span: _,
                 location,
-                files: _,
+                options,
                 alias,
             } => {
                 let (user_stage_info, path) = match location.clone() {
@@ -346,8 +346,8 @@ impl Binder {
                 ) {
                     let files_info = StageFilesInfo {
                         path,
-                        pattern: None,
-                        files: None,
+                        pattern: options.pattern.clone(),
+                        files: options.files.clone(),
                     };
                     let read_options = ParquetReadOptions::default();
 

tests/suites/1_stateful/08_select_stage/08_00_parquet/08_00_00_basic.sh

@@ -13,7 +13,7 @@ echo "create stage s1 FILE_FORMAT = (type = PARQUET);" | $MYSQL_CLIENT_CONNECT
 echo "copy into @s1 from t1;" | $MYSQL_CLIENT_CONNECT
 echo "select * from @s1;" | $MYSQL_CLIENT_CONNECT
 
-DATADIR_PATH="/tmp/00_0004_select_stage"
+DATADIR_PATH="/tmp/08_00_00"
 rm -rf ${DATADIR_PATH}
 DATADIR="fs://$DATADIR_PATH/"
 echo "copy into '${DATADIR}' from t1 FILE_FORMAT = (type = PARQUET);" | $MYSQL_CLIENT_CONNECT

tests/suites/1_stateful/08_select_stage/08_00_parquet/08_00_02_select_files.result

@@ -0,0 +1,10 @@
+1	(1,'a')
+2	(3,'b')
+3	(3,'c')
+---
+1	(1,'a')
+2	(3,'b')
+3	(3,'c')
+1	(1,'a')
+2	(3,'b')
+3	(3,'c')

tests/suites/1_stateful/08_select_stage/08_00_parquet/08_00_02_select_files.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+. "$CURDIR"/../../../../shell_env.sh
+
+
+DATADIR_PATH="/tmp/08_00_02"
+rm -rf ${DATADIR_PATH}
+mkdir ${DATADIR_PATH}
+DATADIR="fs://$DATADIR_PATH/"
+
+echo "drop stage if exists s2;" | $MYSQL_CLIENT_CONNECT
+echo "create stage s2 url = '${DATADIR}' FILE_FORMAT = (type = PARQUET);"  | $MYSQL_CLIENT_CONNECT
+
+
+cp "$CURDIR"/../../../../data/tuple.parquet ${DATADIR_PATH}/
+cp "$CURDIR"/../../../../data/sample.csv ${DATADIR_PATH}/
+
+echo "select * from @s2 (files => ('tuple.parquet'));" | $MYSQL_CLIENT_CONNECT
+
+echo "---"
+
+cp "$CURDIR"/../../../../data/tuple.parquet ${DATADIR_PATH}/tuple2.parquet
+echo "select * from @s2 (pattern => '.*parquet');" | $MYSQL_CLIENT_CONNECT