From 6834e7015171e0ea7f8c2c1e4b1d50c96e29db54 Mon Sep 17 00:00:00 2001 From: Leonard Xu Date: Thu, 25 Jul 2024 19:33:09 +0800 Subject: [PATCH] [docs][minor] Add transform piece for pipeline example --- README.md | 22 +++++++++---------- .../docs/core-concept/data-pipeline.md | 11 ++++++++++ .../docs/core-concept/data-pipeline.md | 18 +++++++++++++++ 3 files changed, 40 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index c44d9be6ed7..426aa96daea 100644 --- a/README.md +++ b/README.md @@ -53,19 +53,19 @@ full database synchronization, sharding table synchronization, schema evolution password: pass transform: - - source-table: adb.web_order01 - projection: \*, UPPER(product_name) as product_name - filter: id > 10 AND order_id > 100 - description: project fields and filter - - source-table: adb.web_order02 - projection: \*, UPPER(product_name) as product_name - filter: id > 20 AND order_id > 200 - description: project fields and filter + - source-table: adb.web_order01 + projection: \*, UPPER(product_name) as product_name + filter: id > 10 AND order_id > 100 + description: project fields and filter + - source-table: adb.web_order02 + projection: \*, UPPER(product_name) as product_name + filter: id > 20 AND order_id > 200 + description: project fields and filter route: - - source-table: adb.web_order\.* - sink-table: adb.ods_web_orders - description: sync sharding tables to one destination table + - source-table: adb.web_order\.* + sink-table: adb.ods_web_orders + description: sync sharding tables to one destination table pipeline: name: MySQL to Doris Pipeline diff --git a/docs/content.zh/docs/core-concept/data-pipeline.md b/docs/content.zh/docs/core-concept/data-pipeline.md index b9745e8823e..4de5df3fb82 100644 --- a/docs/content.zh/docs/core-concept/data-pipeline.md +++ b/docs/content.zh/docs/core-concept/data-pipeline.md @@ -79,6 +79,17 @@ We could use following yaml file to define a complicated Data Pipeline describin fenodes: 127.0.0.1:8030 username: root password: "" + + transform: + - source-table: adb.web_order01 + projection: \*, UPPER(product_name) as product_name + filter: id > 10 AND order_id > 100 + description: project fields and filter + - source-table: adb.web_order02 + projection: \*, UPPER(product_name) as product_name + filter: id > 20 AND order_id > 200 + description: project fields and filter + route: - source-table: app_db.orders sink-table: ods_db.ods_orders diff --git a/docs/content/docs/core-concept/data-pipeline.md b/docs/content/docs/core-concept/data-pipeline.md index b9745e8823e..8914b721d78 100644 --- a/docs/content/docs/core-concept/data-pipeline.md +++ b/docs/content/docs/core-concept/data-pipeline.md @@ -57,6 +57,24 @@ We could use following yaml file to define a concise Data Pipeline describing sy username: root password: "" + transform: + - source-table: adb.web_order01 + projection: \*, UPPER(product_name) as product_name + filter: id > 10 AND order_id > 100 + description: project fields and filter + - source-table: adb.web_order02 + projection: \*, UPPER(product_name) as product_name + filter: id > 20 AND order_id > 200 + description: project fields and filter + + route: + - source-table: app_db.orders + sink-table: ods_db.ods_orders + - source-table: app_db.shipments + sink-table: ods_db.ods_shipments + - source-table: app_db.products + sink-table: ods_db.ods_products + pipeline: name: Sync MySQL Database to Doris parallelism: 2