chore(risedev): add external_labels and remote_write in prometeus con…

…fig (#3825)

* add remote write in prometheus config

* fix config

* add env PROMETHEUS_REMOTE_WRITE

* cargo fmt

* cargo fmt

* update risedev yml

* fix risedev yml

* fix risedev yml

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

risedev.yml

@@ -83,6 +83,23 @@ risedev:
     - use: kafka
       persist-data: true
 
+  full-benchmark:
+    - use: minio
+    - use: etcd
+    - use: meta-node
+    - use: compute-node
+    - use: frontend
+    - use: compactor
+    - use: prometheus
+      remote-write: true
+      remote-write-region: "ap-southeast-1"
+      remote-write-url: "https://aps-workspaces.ap-southeast-1.amazonaws.com/workspaces/ws-f3841dad-6a5c-420f-8f62-8f66487f512a/api/v1/remote_write"
+    - use: grafana
+    - use: zookeeper
+      persist-data: true
+    - use: kafka
+      persist-data: true
+
   ##########################################
   ### Configuration used for ./risedev p ###
   ##########################################
@@ -433,6 +450,15 @@ template:
     # Id of this instance
     id: prometheus
 
+    # If `remote_write` is true, this Prometheus instance will push metrics to remote instance
+    remote-write: false
+
+    # AWS region of remote write
+    remote-write-region: ""
+
+    # Remote write url of this instance
+    remote-write-url: ""
+
     # Compute-nodes used by this Prometheus instance
     provide-compute-node: "compute-node*"
 

src/risedevtool/Cargo.toml

@@ -7,6 +7,7 @@ version = "0.1.10"
 [dependencies]
 anyhow = "1"
 bytes = "1"
+chrono = { version = "0.4", default-features = false, features = ["clock", "std"] }
 clap = { version = "3", features = ["derive"] }
 console = "0.15"
 dialoguer = "0.10"

src/risedevtool/src/config_gen/prometheus_gen.rs

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use chrono::prelude::*;
 use itertools::Itertools;
 
 use crate::PrometheusConfig;
@@ -70,11 +71,36 @@ impl PrometheusGen {
             .map(|node| format!("\"{}:{}\"", node.address, 9644))
             .join(",");
 
+        let now = Local::now().format("%Y%m%d-%H%M%S");
+
+        let remote_write = if config.remote_write {
+            let remote_write_region = &config.remote_write_region;
+            let remote_write_url = &config.remote_write_url;
+            format!(
+                r#"
+remote_write:
+  -
+    url: {remote_write_url}
+    queue_config:
+        max_samples_per_send: 1000
+        max_shards: 200
+        capacity: 2500
+    sigv4:
+         region: {remote_write_region}
+"#,
+            )
+        } else {
+            String::from("")
+        };
+
         format!(
             r#"# --- THIS FILE IS AUTO GENERATED BY RISEDEV ---
 global:
   scrape_interval: 1s
   evaluation_interval: 5s
+  external_labels:
+    rw_cluster: {now}
+{remote_write}
 
 scrape_configs:
   - job_name: prometheus

src/risedevtool/src/service_config.rs

@@ -155,6 +155,10 @@ pub struct PrometheusConfig {
     pub port: u16,
     pub listen_address: String,
 
+    pub remote_write: bool,
+    pub remote_write_region: String,
+    pub remote_write_url: String,
+
     pub provide_compute_node: Option<Vec<ComputeNodeConfig>>,
     pub provide_meta_node: Option<Vec<MetaNodeConfig>>,
     pub provide_minio: Option<Vec<MinioConfig>>,