fix LR ovr predict(FederatedAI#4659)

fix fate-test data cli(FederatedAI#5008)
edit bq examples & add pipeline testsuite(FederatedAI#5008)

Signed-off-by: Yu Wu <yolandawu131@gmail.com>

Author: nemirorox

doc/api/fate_test.md

@@ -0,0 +1,91 @@
+# FATE Test Tutorial
+
+A collection of useful tools to running FATE tests and [:file_folder:examples](../../examples).
+
+## quick start
+
+1. install
+
+    ```bash
+    pip install -e python/fate_test
+    ```
+2. edit default fate\_test\_config.yaml
+
+   ```bash
+   # edit priority config file with system default editor
+   # filling some field according to comments
+   fate_test config edit
+   ```
+
+3. configure FATE-Flow Commandline server setting
+
+    ```bash
+    # configure FATE-Flow Commandline server setting
+    flow init --port 9380 --ip 127.0.0.1
+    ```
+
+4. run some fate\_test suite
+
+   ```bash
+   fate_test suite -i <path contains *testsuite.yaml>
+   ```
+
+5. run some fate\_test benchmark quality
+
+   ```bash
+   fate_test benchmark-quality -i <path contains *benchmark.yaml>
+   ```
+
+6. run some fate\_test benchmark performance
+
+   ```bash
+   fate_test benchmark-quality -i <path contains *performance.yaml>
+   ```
+
+7useful logs or exception will be saved to logs dir with namespace
+shown in last step
+
+## command types
+
+- [suite](../api/fate_test.md#testsuite): used for running [testsuites](../api/fate_test.md#testsuite-configuration),
+  collection of FATE jobs
+
+  ```bash
+  fate_test suite -i <path contains *testsuite.yaml>
+  ```
+
+- [data](../api/fate_test.md#data): used for upload, delete, and generate dataset
+
+    - [upload/delete data](../api/fate_test.md#data-command-options) command:
+
+      ```bash
+      fate_test data [upload|delete] -i <path1 contains *testsuite.yaml | *benchmark.yaml>
+      ```
+    - [upload example data of min_test/all_examples](../api/fate_test.md#data-command-options) command:
+
+      ```bash
+      fate_test data upload -t min_test
+      fate_test data upload -t all_examples
+      ```
+
+    - [generate data](../api/fate_test.md#generate-command-options) command:
+
+      ```bash
+      fate_test data generate -i <path1 contains *testsuite.yaml | *benchmark.yaml>
+      ```
+
+- [benchmark-quality](../api/fate_test.md#benchmark-quality): used for comparing modeling quality between FATE
+  and other machine learning systems, as specified
+  in [benchmark job configuration](../api/fate_test.md#benchmark-job-configuration)
+
+  ```bash
+  fate_test bq -i <path contains *benchmark.yaml>
+  ```
+
+- [benchmark-performance](../api/fate_test.md#benchmark-performance): used for checking FATE algorithm performance; user
+  should first generate and upload data before running performance testsuite
+
+  ```bash
+  fate_test data generate -i <path contains *performance.yaml> -ng 10000 -fg 10 -fh 10 -m 1.0 --upload-data
+  fate_test performance -i <path contains *performance.yaml> --skip-data
+  ```

doc/tutorial/fate_test_tutorial.md

@@ -17,7 +17,7 @@
 import argparse
 
 from fate_client.pipeline import FateFlowPipeline
-from fate_client.pipeline.components.fate import CoordinatedLR, Intersection
+from fate_client.pipeline.components.fate import CoordinatedLR, PSI
 from fate_client.pipeline.components.fate import Evaluation
 from fate_client.pipeline.interface import DataWarehouseChannel
 from fate_client.pipeline.utils import test_utils
@@ -48,11 +48,11 @@ def main(config="../../config.yaml", param="./lr_config.yaml", namespace=""):
     if config.timeout:
         pipeline.conf.set("timeout", config.timeout)
 
-    intersect_0 = Intersection("intersect_0", method="raw")
-    intersect_0.guest.component_setting(input_data=DataWarehouseChannel(name=guest_train_data["name"],
-                                                                        namespace=guest_train_data["namespace"]))
-    intersect_0.hosts[0].component_setting(input_data=DataWarehouseChannel(name=host_train_data["name"],
-                                                                           namespace=host_train_data["namespace"]))
+    psi_0 = PSI("psi_0")
+    psi_0.guest.component_setting(input_data=DataWarehouseChannel(name=guest_train_data["name"],
+                                                                  namespace=guest_train_data["namespace"]))
+    psi_0.hosts[0].component_setting(input_data=DataWarehouseChannel(name=host_train_data["name"],
+                                                                     namespace=host_train_data["namespace"]))
 
     lr_param = {
     }
@@ -68,10 +68,10 @@ def main(config="../../config.yaml", param="./lr_config.yaml", namespace=""):
     }
     lr_param.update(config_param)
     lr_0 = CoordinatedLR("lr_0",
-                         train_data=intersect_0.outputs["output_data"],
+                         train_data=psi_0.outputs["output_data"],
                          **lr_param)
     lr_1 = CoordinatedLR("lr_1",
-                         test_data=intersect_0.outputs["output_data"],
+                         test_data=psi_0.outputs["output_data"],
                          input_model=lr_0.outputs["output_model"])
 
     evaluation_0 = Evaluation("evaluation_0",
@@ -80,7 +80,7 @@ def main(config="../../config.yaml", param="./lr_config.yaml", namespace=""):
                               metrics=["auc", "binary_precision", "binary_accuracy", "binary_recall"],
                               input_data=lr_0.outputs["train_output_data"])
 
-    pipeline.add_task(intersect_0)
+    pipeline.add_task(psi_0)
     pipeline.add_task(lr_0)
     pipeline.add_task(lr_1)
     pipeline.add_task(evaluation_0)

examples/benchmark_performance/coordinated_lr/test_lr_sid.py

@@ -2,7 +2,7 @@ data_guest: "default_credit_hetero_guest"
 data_host: "default_credit_hetero_host"
 idx: "id"
 label_name: "y"
-epochs: 22
+epochs: 30
 init_param:
   fit_intercept: True
   method: "zeros"
@@ -17,6 +17,6 @@ optimizer:
   penalty: "L2"
   alpha: 0.001
   optimizer_params:
-    lr: 0.15
+    lr: 0.21
 batch_size: 3200
 early_stop: "diff"

examples/benchmark_quality/lr/default_credit_config.yaml

@@ -3,7 +3,7 @@ data_host: "epsilon_5k_hetero_host"
 idx: "id"
 label_name: "y"
 epochs: 8
-batch_size: 2500
+batch_size: 2200
 init_param:
   fit_intercept: True
   method: "random"

examples/benchmark_quality/lr/epsilon_5k_config.yaml

@@ -2,20 +2,20 @@ data_guest: "give_credit_hetero_guest"
 data_host: "give_credit_hetero_host"
 idx: "id"
 label_name: "y"
-epochs: 6
+epochs: 12
 init_param:
   fit_intercept: True
   method: "zeros"
 learning_rate_scheduler:
   method: "linear"
   scheduler_params:
-    factor: 0.7
+    start_factor: 0.71
     total_iters: 1000
 optimizer:
-  method: "adam"
+  method: "rmsprop"
   penalty: "L2"
-  alpha: 10
+  alpha: 0.01
   optimizer_params:
-    lr: 0.2
+    lr: 0.29
 batch_size: 5500
 early_stop: "diff"

examples/benchmark_quality/lr/give_credit_config.yaml

@@ -179,15 +179,15 @@ data:
     table_name: vehicle_scale_hetero_host
     namespace: experiment
     role: host_0
-hetero_lr-binary-0-breast:
-  local:
-    script: "./sklearn-lr-binary.py"
-    conf: "./breast_lr_sklearn_config.yaml"
-  FATE-hetero-lr:
-    script: "./pipeline-lr-binary.py"
-    conf: "./breast_config.yaml"
-  compare_setting:
-    relative_tol: 0.01
+#hetero_lr-binary-0-breast:
+#  local:
+#    script: "./sklearn-lr-binary.py"
+#    conf: "./breast_lr_sklearn_config.yaml"
+#  FATE-hetero-lr:
+#    script: "./pipeline-lr-binary.py"
+#    conf: "./breast_config.yaml"
+#  compare_setting:
+#    relative_tol: 0.01
 hetero_lr-binary-1-default-credit:
   local:
     script: "./sklearn-lr-binary.py"
@@ -197,15 +197,15 @@ hetero_lr-binary-1-default-credit:
     conf: "./default_credit_config.yaml"
   compare_setting:
     relative_tol: 0.01
-hetero_lr-binary-2-epsilon-5k:
-  local:
-    script: "./sklearn-lr-binary.py"
-    conf: "./epsilon_5k_lr_sklearn_config.yaml"
-  FATE-hetero-lr:
-    script: "./pipeline-lr-binary.py"
-    conf: "./epsilon_5k_config.yaml"
-  compare_setting:
-    relative_tol: 0.01
+#hetero_lr-binary-2-epsilon-5k:
+#  local:
+#    script: "./sklearn-lr-binary.py"
+#    conf: "./epsilon_5k_lr_sklearn_config.yaml"
+#  FATE-hetero-lr:
+#    script: "./pipeline-lr-binary.py"
+#    conf: "./epsilon_5k_config.yaml"
+#  compare_setting:
+#    relative_tol: 0.01
 hetero_lr-binary-3-give-credit:
   local:
     script: "./sklearn-lr-binary.py"

examples/benchmark_quality/lr/lr_benchmark.yaml

@@ -71,6 +71,7 @@ def main(config="../../config.yaml", param="./vehicle_config.yaml", namespace=""
                          input_model=lr_0.outputs["output_model"])
 
     evaluation_0 = Evaluation('evaluation_0',
+                              input_data=lr_0.outputs["train_output_data"],
                               metrics=['multi_recall', 'multi_accuracy', 'multi_precision'])
     pipeline.add_task(psi_0)
     pipeline.add_task(lr_0)

examples/benchmark_quality/lr/pipeline-lr-multi.py

@@ -42,10 +42,10 @@ def main(config="../../config.yaml", param="./vehicle_lr_sklearn_config.yaml"):
 
     config_param = {
         "penalty": param["penalty"],
-        "max_iter": param["max_iter"],
+        "max_iter": param["epochs"],
         "alpha": param["alpha"],
         "learning_rate": "optimal",
-        "eta0": param["learning_rate"],
+        "eta0": param["eta0"],
         "random_state": 105
     }
 

examples/benchmark_quality/lr/sklearn-lr-multi.py

@@ -0,0 +1,60 @@
+data:
+  - file: examples/data/motor_hetero_guest.csv
+    meta:
+      delimiter: ","
+      dtype: float64
+      input_format: dense
+      label_type: int64
+      label_name: motor_speed
+      match_id_name: id
+      match_id_range: 0
+      tag_value_delimiter: ":"
+      tag_with_value: false
+      weight_type: float64
+    partitions: 4
+    head: true
+    extend_sid: true
+    table_name: motor_hetero_guest
+    namespace: experiment
+    role: guest_0
+  - file: examples/data/motor_hetero_host.csv
+    meta:
+      delimiter: ","
+      dtype: float64
+      input_format: dense
+      match_id_name: id
+      match_id_range: 0
+      tag_value_delimiter: ":"
+      tag_with_value: false
+      weight_type: float64
+    partitions: 4
+    head: true
+    extend_sid: true
+    table_name: motor_hetero_host
+    namespace: experiment
+    role: host_0
+  - file: examples/data/motor_hetero_host.csv
+    meta:
+      delimiter: ","
+      dtype: float64
+      input_format: dense
+      match_id_name: id
+      match_id_range: 0
+      tag_value_delimiter: ":"
+      tag_with_value: false
+      weight_type: float64
+    partitions: 4
+    head: true
+    extend_sid: true
+    table_name: motor_hetero_host
+    namespace: experiment
+    role: host_1
+tasks:
+  normal-linr:
+    script: test_linr.py
+  linr-cv:
+    script: test_linr_cv.py
+  linr-warm-start:
+    script: test_linr_warm_start.py
+  linr-multi-host:
+    script: test_linr_multi_host.py