Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(frontend): deprecate CREATE MATERIALIZED SOURCE syntax #7281

Merged
merged 16 commits into from
Jan 18, 2023
Merged

feat(frontend): deprecate CREATE MATERIALIZED SOURCE syntax #7281

Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
dadca92
deprecate CREATE MATERIALIZED SOURCE syntax
xx01cyx Jan 10, 2023
d2ed46c
Merge remote-tracking branch 'origin/main' into cyx/deprecate-create-…
xx01cyx Jan 11, 2023
13848d8
fix ci
xx01cyx Jan 11, 2023
fcda273
fix ci
xx01cyx Jan 12, 2023
aaec00b
merge main and resolve conflicts
xx01cyx Jan 12, 2023
501ab79
fix build
xx01cyx Jan 12, 2023
0e9e2bd
fix planner test
xx01cyx Jan 12, 2023
3833426
resolve table before source when checking duplication
xx01cyx Jan 12, 2023
9d41680
fix typo in invalid operation
xx01cyx Jan 13, 2023
e2034d4
Merge remote-tracking branch 'origin/main' into cyx/deprecate-create-…
xx01cyx Jan 13, 2023
9aa11a8
add back the test case removed before
xx01cyx Jan 13, 2023
faa10bd
remove match statement & remove materialized source from planner test
xx01cyx Jan 16, 2023
e7215b7
merge main and resolve conflicts
xx01cyx Jan 18, 2023
fa1a13c
Merge branch 'main' into cyx/deprecate-create-materialized-source
mergify[bot] Jan 18, 2023
9f496db
Merge branch 'main' into cyx/deprecate-create-materialized-source
mergify[bot] Jan 18, 2023
c471f60
Merge branch 'main' into cyx/deprecate-create-materialized-source
mergify[bot] Jan 18, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 12 additions & 12 deletions e2e_test/ch_benchmark/create_sources.slt.part
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
statement ok
CREATE MATERIALIZED SOURCE customer (
CREATE TABLE customer (
c_id INTEGER,
c_d_id INTEGER,
c_w_id INTEGER,
Expand Down Expand Up @@ -31,7 +31,7 @@ CREATE MATERIALIZED SOURCE customer (
) ROW FORMAT DEBEZIUM_JSON;

statement ok
CREATE MATERIALIZED SOURCE district (
CREATE TABLE district (
d_id INTEGER,
d_w_id INTEGER,
d_name VARCHAR,
Expand All @@ -52,7 +52,7 @@ CREATE MATERIALIZED SOURCE district (
) ROW FORMAT DEBEZIUM_JSON;

statement ok
CREATE MATERIALIZED SOURCE history (
CREATE TABLE history (
h_c_id INTEGER,
h_c_d_id INTEGER,
h_c_w_id INTEGER,
Expand All @@ -70,7 +70,7 @@ CREATE MATERIALIZED SOURCE history (
) ROW FORMAT DEBEZIUM_JSON;

statement ok
CREATE MATERIALIZED SOURCE item (
CREATE TABLE item (
i_id INTEGER,
i_im_id INTEGER,
i_name VARCHAR,
Expand All @@ -85,7 +85,7 @@ CREATE MATERIALIZED SOURCE item (
) ROW FORMAT DEBEZIUM_JSON;

statement ok
CREATE MATERIALIZED SOURCE neworder (
CREATE TABLE neworder (
no_o_id INTEGER,
no_d_id INTEGER,
no_w_id INTEGER,
Expand All @@ -98,7 +98,7 @@ CREATE MATERIALIZED SOURCE neworder (
) ROW FORMAT DEBEZIUM_JSON;

statement ok
CREATE MATERIALIZED SOURCE orderline (
CREATE TABLE orderline (
ol_o_id INTEGER,
ol_d_id INTEGER,
ol_w_id INTEGER,
Expand All @@ -118,7 +118,7 @@ CREATE MATERIALIZED SOURCE orderline (
) ROW FORMAT DEBEZIUM_JSON;

statement ok
CREATE MATERIALIZED SOURCE orders (
CREATE TABLE orders (
o_id INTEGER,
o_d_id INTEGER,
o_w_id INTEGER,
Expand All @@ -136,7 +136,7 @@ CREATE MATERIALIZED SOURCE orders (
) ROW FORMAT DEBEZIUM_JSON;

statement ok
CREATE MATERIALIZED SOURCE stock (
CREATE TABLE stock (
s_i_id INTEGER,
s_w_id INTEGER,
s_quantity INTEGER,
Expand Down Expand Up @@ -164,7 +164,7 @@ CREATE MATERIALIZED SOURCE stock (
) ROW FORMAT DEBEZIUM_JSON;

statement ok
CREATE MATERIALIZED SOURCE warehouse (
CREATE TABLE warehouse (
w_id INTEGER,
w_name VARCHAR,
w_street_1 VARCHAR,
Expand All @@ -183,7 +183,7 @@ CREATE MATERIALIZED SOURCE warehouse (
) ROW FORMAT DEBEZIUM_JSON;

statement ok
CREATE MATERIALIZED SOURCE supplier (
CREATE TABLE supplier (
SU_SUPPKEY SMALLINT,
SU_NAME VARCHAR,
SU_ADDRESS VARCHAR,
Expand All @@ -200,7 +200,7 @@ CREATE MATERIALIZED SOURCE supplier (
) ROW FORMAT DEBEZIUM_JSON;

statement ok
CREATE MATERIALIZED SOURCE region (
CREATE TABLE region (
R_REGIONKEY INTEGER,
R_NAME VARCHAR,
R_COMMENT VARCHAR,
Expand All @@ -213,7 +213,7 @@ CREATE MATERIALIZED SOURCE region (
) ROW FORMAT DEBEZIUM_JSON;

statement ok
CREATE MATERIALIZED SOURCE nation (
CREATE TABLE nation (
N_NATIONKEY INTEGER,
N_NAME VARCHAR,
N_REGIONKEY INTEGER,
Expand Down
24 changes: 12 additions & 12 deletions e2e_test/ch_benchmark/drop_sources.slt.part
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,35 +1,35 @@
statement ok
DROP SOURCE customer;
DROP TABLE customer;

statement ok
DROP SOURCE district;
DROP TABLE district;

statement ok
DROP SOURCE history;
DROP TABLE history;

statement ok
DROP SOURCE item;
DROP TABLE item;

statement ok
DROP SOURCE neworder;
DROP TABLE neworder;

statement ok
DROP SOURCE orderline;
DROP TABLE orderline;

statement ok
DROP SOURCE orders;
DROP TABLE orders;

statement ok
DROP SOURCE stock;
DROP TABLE stock;

statement ok
DROP SOURCE warehouse;
DROP TABLE warehouse;

statement ok
DROP SOURCE supplier;
DROP TABLE supplier;

statement ok
DROP SOURCE region;
DROP TABLE region;

statement ok
DROP SOURCE nation;
DROP TABLE nation;
39 changes: 17 additions & 22 deletions e2e_test/ddl/invalid_operation.slt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,6 @@ drop sink not_exists;
statement error view not found
drop view not_exists;

statement error Use `DROP SOURCE`
drop materialized source not_exists;

# 2. drop relation in non-existent schema

statement error schema not found
Expand Down Expand Up @@ -94,8 +91,7 @@ drop table mv;
statement error Use `DROP MATERIALIZED VIEW`
drop index mv;

# FIXME: improve the error message
statement error not found
statement error Use `DROP MATERIALIZED VIEW`
drop source mv;

# FIXME: improve the error message
Expand All @@ -116,8 +112,7 @@ drop table idx;
statement error Use `DROP INDEX`
drop materialized view idx;

# FIXME: improve the error message
statement error not found
statement error Use `DROP INDEX`
drop source idx;

# FIXME: improve the error message
Expand All @@ -128,9 +123,9 @@ drop sink idx;
statement error not found
drop view idx;

# 4.4 materialized source
# 4.4 table with connector
statement ok
create materialized source msrc (v int) with (
create table msrc (v int) with (
connector = 'datagen',
fields.v.kind = 'sequence',
fields.v.start = '1',
Expand All @@ -139,15 +134,15 @@ create materialized source msrc (v int) with (
datagen.split.num = '1'
) row format json;

statement error Use `DROP SOURCE`
drop table msrc;

statement error Use `DROP SOURCE`
statement error Use `DROP TABLE`
drop materialized view msrc;

statement error Use `DROP SOURCE`
statement error Use `DROP TABLE`
drop index msrc;

statement error Use `DROP TABLE`
drop source msrc;

# FIXME: improve the error message
statement error not found
drop sink msrc;
Expand All @@ -156,7 +151,7 @@ drop sink msrc;
statement error not found
drop view msrc;

# 4.5 non-materialized source
# 4.5 source
statement ok
create source src (v int) with (
connector = 'datagen',
Expand Down Expand Up @@ -295,17 +290,17 @@ update idx set v = 2;
statement error cannot change index
delete from idx;

# 6.4 materialized source
statement error cannot change materialized source
# 6.4 table with connector
statement ok
insert into msrc values (1);

statement error cannot change materialized source
statement ok
update msrc set v = 2;

statement error cannot change materialized source
statement ok
delete from msrc;

# 6.5 non-materialized source
# 6.5 source
# FIXME: improve the error message
statement error not found
insert into src values (1);
Expand Down Expand Up @@ -354,7 +349,7 @@ create table mv(v int);
statement error index.*exists
create table idx(v int);

statement error source.*exists
statement error table.*exists
create table msrc(v int);

statement error source.*exists
Expand All @@ -375,7 +370,7 @@ statement ok
drop sink sink;

statement ok
drop source msrc;
drop table msrc;

statement ok
drop source src;
Expand Down
4 changes: 0 additions & 4 deletions e2e_test/ddl/show.slt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,10 +78,6 @@ show materialized views;
----
mv3

query T
show materialized sources;
----

query T
show sources;
----
Expand Down
20 changes: 10 additions & 10 deletions e2e_test/source/basic/datagen.slt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
statement ok
create materialized source s1 (v1 int, v2 float) with (
create table s1 (v1 int, v2 float) with (
connector = 'datagen',
fields.v1.kind = 'sequence',
fields.v1.start = '1',
Expand Down Expand Up @@ -33,10 +33,10 @@ select v1, v2 from s1 where v1 is not null limit 15;
9 19

statement ok
drop source s1;
drop table s1;

statement ok
create materialized source s1 (v1 int) with (
create table s1 (v1 int) with (
connector = 'datagen',
fields.v1.kind = 'sequence',
fields.v1.start = '1',
Expand Down Expand Up @@ -66,10 +66,10 @@ select v1 from s1 order by v1 limit 10;
10

statement ok
drop source s1;
drop table s1;

statement ok
create materialized source s1 (v1 struct<v2 int>) with (
create table s1 (v1 struct<v2 int>) with (
connector = 'datagen',
fields.v1.v2.kind = 'sequence',
fields.v1.v2.start = '1',
Expand Down Expand Up @@ -99,10 +99,10 @@ select (v1).v2 from s1 order by v1 limit 10;
10

statement ok
drop source s1;
drop table s1;

statement ok
create materialized source s1 (v1 struct<v2 int>, t1 timestamp, c1 varchar) with (
create table s1 (v1 struct<v2 int>, t1 timestamp, c1 varchar) with (
connector = 'datagen',
fields.v1.v2.kind = 'random',
fields.v1.v2.min = '1',
Expand Down Expand Up @@ -136,10 +136,10 @@ select l.len, count(*) > 2*10 from (select length(s1.c1) as len from s1) as l gr
100 t

statement ok
drop source s1;
drop table s1;

statement ok
create materialized source s1 (v1 struct<v2 int> [], c1 varchar []) with (
create table s1 (v1 struct<v2 int> [], c1 varchar []) with (
connector = 'datagen',
fields.v1.length = '2',
fields.v1._.v2.kind = 'random',
Expand All @@ -166,4 +166,4 @@ select count(*) > 10 from s1;
t

statement ok
drop source s1;
drop table s1;
12 changes: 8 additions & 4 deletions e2e_test/source/basic/ddl.slt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
statement ok
create materialized source s with (
create table s with (
connector = 'kafka',
topic = 'kafka_1_partition_topic',
properties.bootstrap.server = '127.0.0.1:29092'
) row format json;

statement ok
drop source s
drop table s

query T
show sources
Expand Down Expand Up @@ -54,12 +54,15 @@ statement ok
drop source s

statement ok
create materialized source s with (
create table s with (
connector = 'kafka',
topic = 'kafka_1_partition_topic',
properties.bootstrap.server = '127.0.0.1:29092'
) row format json;

# TODO: `show sources` should display connectors created by `create source` only.
# We should introduce `show connectors` to display all connectors created via both
# `create source` and `create table`.
Comment on lines +63 to +65
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🤔 In which case a user may want to show connectors? (IIUC connectors cannot be used to do anything)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe users should be able to view all created connectors. 🤔 cc. @st1page @yezizp2012

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think user may only care about which connector is used by some table and SHOW CREATE TABLE is already enough for it. 🤔

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

agree. no need for show connectors

query T
show sources
----
Expand All @@ -68,6 +71,7 @@ s
query T
show tables
----
s

statement ok
drop source s
drop table s
Loading