Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Copy into support optimize hint #11614

Merged
merged 7 commits into from
Jun 1, 2023
Merged

feat: Copy into support optimize hint #11614

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
34f0147
fix: init
jun0315 May 29, 2023
89e3927
fix: ut
jun0315 May 29, 2023
d635105
fix: ut
jun0315 May 30, 2023
79a67b4
test: add copy into
jun0315 May 30, 2023
17cea15
Merge branch 'main' into set_var
jun0315 May 31, 2023
80c68f8
test: rm set_var test
jun0315 May 31, 2023
cc69b86
test: add copy test
jun0315 Jun 1, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions src/query/ast/src/ast/statements/copy.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ use url::Url;

use crate::ast::write_quoted_comma_separated_list;
use crate::ast::write_space_separated_map;
use crate::ast::Hint;
use crate::ast::Identifier;
use crate::ast::Query;
use crate::parser::unescape::escape_at_string;
Expand All @@ -38,6 +39,7 @@ use crate::parser::unescape::escape_at_string;
/// ```
#[derive(Debug, Clone, PartialEq)]
pub struct CopyStmt {
pub hints: Option<Hint>,
pub src: CopyUnit,
pub dst: CopyUnit,
pub files: Option<Vec<String>>,
Expand Down Expand Up @@ -79,6 +81,9 @@ impl CopyStmt {
impl Display for CopyStmt {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(f, "COPY")?;
if let Some(hints) = &self.hints {
write!(f, "{} ", hints)?;
}
write!(f, " INTO {}", self.dst)?;
write!(f, " FROM {}", self.src)?;

Expand Down
4 changes: 3 additions & 1 deletion src/query/ast/src/parser/statement.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -966,12 +966,14 @@ pub fn statement(i: Input) -> IResult<StatementMsg> {
let copy_into = map(
rule! {
COPY
~ #hint?
~ INTO ~ #copy_unit
~ FROM ~ #copy_unit
~ ( #copy_option )*
},
|(_, _, dst, _, src, opts)| {
|(_, opt_hints, _, dst, _, src, opts)| {
let mut copy_stmt = CopyStmt {
hints: opt_hints,
src,
dst,
files: Default::default(),
Expand Down
1 change: 1 addition & 0 deletions src/query/ast/src/parser/token.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ impl<'a> Iterator for Tokenizer<'a> {
| TokenKind::REPLACE
| TokenKind::UPDATE
| TokenKind::DELETE
| TokenKind::COPY
)
) && kind == TokenKind::HintPrefix
{
Expand Down
14 changes: 14 additions & 0 deletions src/query/ast/tests/it/testdata/statement.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7743,6 +7743,7 @@ COPY INTO mytable FROM @~/mybucket/data.csv FILE_FORMAT = ( field_delimiter = ',
---------- AST ------------
Copy(
CopyStmt {
hints: None,
src: StageLocation(
StageLocation {
name: "~",
Expand Down Expand Up @@ -7800,6 +7801,7 @@ COPY INTO mytable FROM 's3://mybucket/data.csv' FILE_FORMAT = ( field_delimiter
---------- AST ------------
Copy(
CopyStmt {
hints: None,
src: UriLocation(
UriLocation {
protocol: "s3",
Expand Down Expand Up @@ -7865,6 +7867,7 @@ COPY INTO mytable FROM 's3://mybucket/data.csv' CONNECTION = ( endpoint_url='htt
---------- AST ------------
Copy(
CopyStmt {
hints: None,
src: UriLocation(
UriLocation {
protocol: "s3",
Expand Down Expand Up @@ -7932,6 +7935,7 @@ COPY INTO mytable FROM 's3://mybucket/data.csv' CONNECTION = ( endpoint_url='htt
---------- AST ------------
Copy(
CopyStmt {
hints: None,
src: UriLocation(
UriLocation {
protocol: "s3",
Expand Down Expand Up @@ -7988,6 +7992,7 @@ COPY INTO mytable FROM 'https://127.0.0.1:9900/' SINGLE = false PURGE = false FO
---------- AST ------------
Copy(
CopyStmt {
hints: None,
src: UriLocation(
UriLocation {
protocol: "https",
Expand Down Expand Up @@ -8037,6 +8042,7 @@ COPY INTO mytable FROM 'https://127.0.0.1/' SINGLE = false PURGE = false FORCE =
---------- AST ------------
Copy(
CopyStmt {
hints: None,
src: UriLocation(
UriLocation {
protocol: "https",
Expand Down Expand Up @@ -8094,6 +8100,7 @@ COPY INTO mytable FROM @my_stage/ FILE_FORMAT = ( field_delimiter = ',' record_d
---------- AST ------------
Copy(
CopyStmt {
hints: None,
src: StageLocation(
StageLocation {
name: "my_stage",
Expand Down Expand Up @@ -8150,6 +8157,7 @@ COPY INTO 's3://mybucket/data.csv' FROM mytable FILE_FORMAT = ( field_delimiter
---------- AST ------------
Copy(
CopyStmt {
hints: None,
src: Table {
catalog: None,
database: None,
Expand Down Expand Up @@ -8212,6 +8220,7 @@ COPY INTO @my_stage/ FROM mytable FILE_FORMAT = ( field_delimiter = ',' record_d
---------- AST ------------
Copy(
CopyStmt {
hints: None,
src: Table {
catalog: None,
database: None,
Expand Down Expand Up @@ -8272,6 +8281,7 @@ COPY INTO mytable FROM 's3://mybucket/data.csv' CONNECTION = ( aws_key_id='acces
---------- AST ------------
Copy(
CopyStmt {
hints: None,
src: UriLocation(
UriLocation {
protocol: "s3",
Expand Down Expand Up @@ -8337,6 +8347,7 @@ COPY INTO mytable FROM @external_stage/path/to/file.csv FILE_FORMAT = ( field_de
---------- AST ------------
Copy(
CopyStmt {
hints: None,
src: StageLocation(
StageLocation {
name: "external_stage",
Expand Down Expand Up @@ -8393,6 +8404,7 @@ COPY INTO mytable FROM @external_stage/path/to/dir/ FILE_FORMAT = ( field_delimi
---------- AST ------------
Copy(
CopyStmt {
hints: None,
src: StageLocation(
StageLocation {
name: "external_stage",
Expand Down Expand Up @@ -8449,6 +8461,7 @@ COPY INTO mytable FROM @external_stage/path/to/file.csv FILE_FORMAT = ( field_de
---------- AST ------------
Copy(
CopyStmt {
hints: None,
src: StageLocation(
StageLocation {
name: "external_stage",
Expand Down Expand Up @@ -8506,6 +8519,7 @@ COPY INTO mytable FROM 'fs:///path/to/data.csv' FILE_FORMAT = ( field_delimiter
---------- AST ------------
Copy(
CopyStmt {
hints: None,
src: UriLocation(
UriLocation {
protocol: "fs",
Expand Down
9 changes: 8 additions & 1 deletion src/query/sql/src/planner/binder/binder.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,14 @@ impl<'a> Binder {
self.bind_show_table_functions(bind_context, limit).await?
}

Statement::Copy(stmt) => self.bind_copy(bind_context, stmt).await?,
Statement::Copy(stmt) => {
if let Some(hints) = &stmt.hints {
if let Some(e) = self.opt_hints_set_var(bind_context, hints).await.err() {
warn!("In Copy resolve optimize hints {:?} failed, err: {:?}", hints, e);
}
}
self.bind_copy(bind_context, stmt).await?
},

Statement::ShowMetrics => {
self.bind_rewrite_to_query(
Expand Down
4 changes: 2 additions & 2 deletions tests/suites/0_stateless/05_hints/05_0001_set_var.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
. "$CURDIR"/../../../shell_env.sh

# mariadb mysql client has some bug, please use mysql offical client
# mariadb mysql client has some bug, please use mysql official client
# mysql --version
# mysql Ver 8.0.32-0ubuntu0.20.04.2 for Linux on x86_64 ((Ubuntu))
echo "select * from (select /*+SET_VAR(timezone='Asia/Shanghai')*/ name, value value from system.settings where name in ('timezone') union all (select /*+SET_VAR(timezone='America/Los_Angeles')*/ 'x','x')) order by name desc;" | $MYSQL_CLIENT_CONNECT
Expand Down Expand Up @@ -33,4 +33,4 @@ echo "set timezone='America/Toronto'; select /*+SET_VAR(timezone='Asia/Shanghai'
echo "create table set_var.t(c1 timestamp)" | $MYSQL_CLIENT_CONNECT
# Toronto and Shanghai time diff is 13 hours.
echo "set timezone='America/Toronto'; insert /*+SET_VAR(timezone='Asia/Shanghai') */ into set_var.t values('2022-02-02 03:00:00'); select /*+SET_VAR(timezone='Asia/Shanghai') */ * from set_var.t; select * from set_var.t;" | $MYSQL_CLIENT_CONNECT
echo "drop database set_var;" | $MYSQL_CLIENT_CONNECT
echo "drop database set_var;" | $MYSQL_CLIENT_CONNECT