error for load_partition_data_mnist()

.github/workflows/test_mlops_cli.yml

@@ -0,0 +1,39 @@
+# This is a basic workflow to help you get started with Actions
+
+name: MLOps CLI Test
+
+# Controls when the workflow will run
+on:
+  # Triggers the workflow on push or pull request events but only for the master branch
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  # This workflow contains a single job called "mlops-cli-test"
+  mlops-cli-test:
+    # The type of runner that the job will run on
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+        working-directory: python
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v3
+
+      # Runs a single command using the runners shell
+      - name: Run test cases for fedml-build, mlops-build and mlops-login
+        run: cd build_tools/github/; conda info; source ./test_mlops_cli.sh
+
+      # Runs a single command using the runners shell
+      #- name: Run logout to exit mlops cli
+      #  run: cd build_tools/github/; conda info; source ./test_mlops_cli.sh logout
+      #  continue-on-error: true

.gitignore

@@ -2,6 +2,7 @@
 .idea
 .idea/*
 .vscode/*
+*.pkl
 
 *.pyc
 ./*/*.pyc
@@ -28,7 +29,7 @@ python/dist
 python/FedML.egg-info
 
 doc/deploy
-doc/en/_build
+doc/en/_build/doctrees
 
 
 *.h5
@@ -79,17 +80,24 @@ test/fedml_user_code/simulation_mpi/data/mnist/MNIST/download_and_unzip.sh
 test/fedml_user_code/simulation_mpi/data/mnist/__MACOSX/MNIST/._train
 test/fedml_user_code/simulation_mpi/data/mnist/__MACOSX/MNIST/._test
 test/fedml_user_code/simulation_mpi/data/mnist/__MACOSX/MNIST/._.DS_Store
-test/fedml_user_code/simulation_sp/data/mnist/__MACOSX/MNIST/._.DS_Store
-test/fedml_user_code/simulation_sp/data/mnist/__MACOSX/MNIST/._test
-test/fedml_user_code/simulation_sp/data/mnist/__MACOSX/MNIST/._train
-test/fedml_user_code/simulation_sp/data/mnist/MNIST/download_and_unzip.sh
-test/fedml_user_code/simulation_sp/data/mnist/MNIST/README.md
-test/fedml_user_code/simulation_sp/data/mnist/MNIST/stats.py
-test/fedml_user_code/simulation_sp/data/mnist/MNIST/stats.sh
-test/fedml_user_code/simulation_mpi/data/mnist/__MACOSX/MNIST/._.DS_Store
-test/fedml_user_code/simulation_mpi/data/mnist/__MACOSX/MNIST/._test
-test/fedml_user_code/simulation_mpi/data/mnist/__MACOSX/MNIST/._train
-test/fedml_user_code/simulation_mpi/data/mnist/MNIST/download_and_unzip.sh
-test/fedml_user_code/simulation_mpi/data/mnist/MNIST/README.md
-test/fedml_user_code/simulation_mpi/data/mnist/MNIST/stats.py
-test/fedml_user_code/simulation_mpi/data/mnist/MNIST/stats.sh
+
+app/fedgraphnn/social_networks_graph_clf/data/TUDataset/*
+app/fedgraphnn/social_networks_graph_clf/data/TUDataset/collab.pickle
+app/fedgraphnn/social_networks_graph_clf/x_hist.png
+
+app/fedgraphnn/subgraph_relation_pred/data/FB15k-237/*.dict
+app/fedgraphnn/subgraph_relation_pred/data/FB15k-237/*.txt
+app/fedgraphnn/subgraph_relation_pred/data/wn18rr/*.dict
+app/fedgraphnn/subgraph_relation_pred/data/wn18rr/*.txt
+app/fedgraphnn/subgraph_relation_pred/data/YAGO3-10/*.dict
+app/fedgraphnn/subgraph_relation_pred/data/YAGO3-10/*.txt
+
+app/fedgraphnn/subgraph_link_pred/data/FB15k-237/*.dict
+app/fedgraphnn/subgraph_link_pred/data/FB15k-237/*.txt
+app/fedgraphnn/subgraph_link_pred/data/wn18rr/*.dict
+app/fedgraphnn/subgraph_link_pred/data/wn18rr/*.txt
+app/fedgraphnn/subgraph_link_pred/data/YAGO3-10/*.dict
+app/fedgraphnn/subgraph_link_pred/data/YAGO3-10/*.txt
+
+app/fedgraphnn/ego_networks_link_pred/data/ego-networks
+app/fedgraphnn/ego_networks_node_clf/data/ego-networks

app/fedcv/.gitignore

@@ -0,0 +1,4 @@
+*.zip
+__MACOSX/
+__pycache__/
+*.tmp

app/fedcv/README.md

@@ -0,0 +1,11 @@
+## FedCV
+
+```bash
+mkdir fedcv
+cd fedcv
+git init
+git remote add -f origin https://github.com/FedML-AI/FedML.git
+git config core.sparsecheckout true
+echo "app/fedcv" >> .git/info/sparse-checkout
+git pull origin master
+```

app/fedcv/image_classification/README.md

@@ -0,0 +1,20 @@
+# FedCV for Image Classification
+
+## Model
+
+## Dataset
+
+## How to use
+
+### Linux
+
+```sh
+sudo chmod +x run_image_classification.sh
+./run_image_classification.sh
+```
+
+### Windows
+
+```bat
+mpiexec -np 5 python run_image_classification.py --cf config\fedml_config.yaml
+```

app/fedcv/image_classification/config/fedml_config.yaml

@@ -0,0 +1,46 @@
+common_args:
+  training_type: "simulation"
+  random_seed: 0
+
+data_args:
+  dataset: "MNIST"
+  data_cache_dir: "./data/"
+  partition_method: "hetero"
+  partition_alpha: 0.5
+
+model_args:
+  model: "densenet"
+  image_size: [112, 112]
+  class_num: 10
+
+train_args:
+  federated_optimizer: "FedAvg"
+  client_id_list: "[]"
+  client_num_in_total: 4
+  client_num_per_round: 4
+  comm_round: 100
+  epochs: 5
+  batch_size: 64
+  client_optimizer: sgd
+  lr: 0.03
+  weight_decay: 0.001
+
+validation_args:
+  frequency_of_the_test: 5
+
+device_args:
+  worker_num: 4
+  using_gpu: false
+  gpu_mapping_file: config/gpu_mapping.yaml
+  gpu_mapping_key: mapping_default
+
+comm_args:
+  backend: "MPI"
+  is_mobile: 0
+
+tracking_args:
+  log_file_dir: ./log
+  enable_wandb: false
+  wandb_key: ee0b5f53d949c84cee7decbe7a629e63fb2f8408
+  wandb_project: fedml
+  wandb_name: fedml_torch_image_classification

app/fedcv/image_classification/config/gpu_mapping.yaml

@@ -0,0 +1,61 @@
+# Please check "GPU_MAPPING.md" to see how to define the topology
+# You can define a cluster containing multiple GPUs within multiple machines by defining `gpu_mapping.yaml` as follows:
+
+# config_cluster0:
+#     host_name_node0: [num_of_processes_on_GPU0, num_of_processes_on_GPU1, num_of_processes_on_GPU2, num_of_processes_on_GPU3, ..., num_of_processes_on_GPU_n]
+#     host_name_node1: [num_of_processes_on_GPU0, num_of_processes_on_GPU1, num_of_processes_on_GPU2, num_of_processes_on_GPU3, ..., num_of_processes_on_GPU_n]
+#     host_name_node_m: [num_of_processes_on_GPU0, num_of_processes_on_GPU1, num_of_processes_on_GPU2, num_of_processes_on_GPU3, ..., num_of_processes_on_GPU_n]
+
+
+# this is used for 10 clients and 1 server training within a single machine which has 4 GPUs
+mapping_default:
+    ChaoyangHe-GPU-RTX2080Tix4: [3, 3, 3, 2]
+
+# this is used for 4 clients and 1 server training within a single machine which has 4 GPUs
+mapping_config1_5:
+    host1: [2, 1, 1, 1]
+
+# this is used for 10 clients and 1 server training within a single machine which has 4 GPUs
+mapping_config2_11:
+    host1: [3, 3, 3, 2]
+
+# this is used for 10 clients and 1 server training within a single machine which has 8 GPUs
+mapping_config3_11:
+    host1: [2, 2, 2, 1, 1, 1, 1, 1]
+
+# this is used for 4 clients and 1 server training within a single machine which has 8 GPUs, but you hope to skip the GPU device ID.
+mapping_config4_5:
+    host1: [1, 0, 0, 1, 1, 0, 1, 1]
+
+# this is used for 4 clients and 1 server training using 6 machines, each machine has 2 GPUs inside, but you hope to use the second GPU.
+mapping_config5_6:
+    host1: [0, 1]
+    host2: [0, 1]
+    host3: [0, 1]
+    host4: [0, 1]
+    host5: [0, 1]
+# this is used for 4 clients and 1 server training using 2 machines, each machine has 2 GPUs inside, but you hope to use the second GPU.
+mapping_config5_2:
+    gpu-worker2: [1,1]
+    gpu-worker1: [2,1]
+
+# this is used for 10 clients and 1 server training using 4 machines, each machine has 2 GPUs inside, but you hope to use the second GPU.
+mapping_config5_4:
+    gpu-worker2: [1,1]
+    gpu-worker1: [2,1]
+    gpu-worker3: [3,1]
+    gpu-worker4: [1,1]
+
+# for grpc GPU mapping
+mapping_FedML_gRPC:
+    hostname_node_server: [1]
+    hostname_node_1: [1, 0, 0, 0]
+    hostname_node_2: [1, 0, 0, 0]
+
+# for torch RPC GPU mapping
+mapping_FedML_tRPC:
+    lambda-server1: [0, 0, 0, 0, 2, 2, 1, 1]
+    lambda-server2: [2, 1, 1, 1, 0, 0, 0, 0]
+
+#mapping_FedML_tRPC:
+#    lambda-server1: [0, 0, 0, 0, 3, 3, 3, 2]

app/fedcv/image_classification/data/FederatedEMNIST/README.md

@@ -0,0 +1,29 @@
+# Introduction
+
+Federated EMNIST dataset extends MNIST dataset with upper and lower case English characters. It is suggested by "Adaptive Federated Optimization" (https://arxiv.org/abs/2003.00295, Google AI).
+
+In order to support more frameworks like PyTorch, we loaded data from TensorFlow Federated (TFF) [emnist load_data API](https://www.tensorflow.org/federated/api_docs/python/tff/simulation/datasets/emnist/load_data) and saved it on [google drive]( https://drive.google.com/drive/folders/1S377qFHM_q_o1hE7-ODlmtRT6iV6N7AT?usp=sharing) in h5 format. 
+
+# Prepare FederatedEMNIST Dataset
+
+You can run the following script to download the dataset:
+
+```
+sh download_federatedEMNIST.sh
+```
+
+By default, it will download EMNIST dataset which includes digits and characters.  TFF also supports getting only the digits data from EMNIST dataset. So we generated another digits only dataset from it. To obtain the digits only dataset, use the following command to download it:
+
+```
+sh download_federatedEMNIST.sh digit
+```
+
+
+# Statistics of FederatedEMNIST Dataset
+
+Data partition is the same as [TFF](https://www.tensorflow.org/federated/api_docs/python/tff/simulation/datasets/emnist), with the following statistics.  
+
+| DATASET   | TRAIN CLIENTS | TRAIN EXAMPLES | TEST CLIENTS | TEST EXAMPLES |
+| --------- | ------------- | -------------- | ------------ | ------------- |
+| EMNIST-62 | 3,400         | 671,585        | 3,400        | 77,483        |
+